diff --git a/.github/workflows/backendtests.yml b/.github/workflows/backendtests.yml index 68a0fb9a7..4d61a8748 100644 --- a/.github/workflows/backendtests.yml +++ b/.github/workflows/backendtests.yml @@ -17,6 +17,7 @@ jobs: box_secrets_available: "${{ steps.check_box.outputs.available }}" filen_secrets_available: "${{ steps.check_filen.outputs.available }}" filejump_secrets_available: "${{ steps.check_filejump.outputs.available }}" + onedrive_secrets_available: "${{ steps.check_onedrive.outputs.available }}" steps: - id: check_pcloud name: Check pCloud secrets @@ -156,7 +157,7 @@ jobs: name: Check Filejump secrets shell: bash run: | - echo "Starting Filen.io secrets check..." + echo "Starting Filejump secrets check..." if [[ -n "${{ secrets.TESTCREDENTIAL_FILEJUMP_FOLDER }}" ]] && \ [[ -n "${{ secrets.TESTCREDENTIAL_FILEJUMP_TOKEN }}" ]]; then echo "All Filejump secrets found" @@ -166,6 +167,20 @@ jobs: echo "available=false" >> $GITHUB_OUTPUT fi echo "Filejump check completed" + - id: check_onedrive + name: Check OneDrive secrets + shell: bash + run: | + echo "Starting OneDrive secrets check..." + if [[ -n "${{ secrets.TESTCREDENTIAL_ONEDRIVE_FOLDER }}" ]] && \ + [[ -n "${{ secrets.TESTCREDENTIAL_ONEDRIVE_AUTHID }}" ]]; then + echo "All OneDrive secrets found" + echo "available=true" >> $GITHUB_OUTPUT + else + echo "Missing some OneDrive secrets" + echo "available=false" >> $GITHUB_OUTPUT + fi + echo "OneDrive check completed" - name: Debug Output shell: bash run: | @@ -178,6 +193,8 @@ jobs: echo "B2 Available: ${{ steps.check_b2.outputs.available }}" echo "Box.com Available: ${{ steps.check_box.outputs.available }}" echo "Filen.io Available: ${{ steps.check_filen.outputs.available }}" + echo "Filejump Available: ${{ steps.check_filejump.outputs.available }}" + echo "OneDrive Available: ${{ steps.check_onedrive.outputs.available }}" test_ftp: needs: check_secrets if: needs.check_secrets.outputs.testcontainers_secrets_available == 'true' @@ -662,3 +679,39 @@ jobs: dotnet test --no-build --filter="ClassName~Filejump.FilejumpTests" --logger:"console;verbosity=detailed" LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + test_onedrive: + needs: check_secrets + if: needs.check_secrets.outputs.onedrive_secrets_available == 'true' + name: OneDrive Tests + runs-on: "${{ matrix.os }}" + strategy: + max-parallel: 1 + fail-fast: false + matrix: + os: + - ubuntu-latest + - windows-latest + - macos-latest + steps: + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.x + - name: Checkout source + uses: actions/checkout@v4 + - name: Restore NuGet dependencies + run: >- + dotnet restore + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + - name: Build project + run: >- + dotnet build --no-restore + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + - name: Run OneDrive tests + env: + TESTCREDENTIAL_ONEDRIVE_FOLDER: "${{ secrets.TESTCREDENTIAL_ONEDRIVE_FOLDER }}" + TESTCREDENTIAL_ONEDRIVE_AUTHID: "${{ secrets.TESTCREDENTIAL_ONEDRIVE_AUTHID }}" + run: >- + dotnet test --no-build --filter="ClassName~OneDrive.OneDriveTests" + --logger:"console;verbosity=detailed" + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln diff --git a/Duplicati.sln b/Duplicati.sln index 40d5d5bb1..ff0f27dae 100644 --- a/Duplicati.sln +++ b/Duplicati.sln @@ -21,8 +21,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Duplicati.CommandLine", "Du EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Duplicati.Library.Backend.WEBDAV", "Duplicati\Library\Backend\WEBDAV\Duplicati.Library.Backend.WEBDAV.csproj", "{BAE27510-8B5D-44B2-B33E-372A98908041}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Duplicati.Library.Backend.CloudFiles", "Duplicati\Library\Backend\CloudFiles\Duplicati.Library.Backend.CloudFiles.csproj", "{1BFAE226-8364-4086-825C-BB83F6F3EE4C}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Duplicati.CommandLine.BackendTester", "Duplicati\CommandLine\BackendTester\Duplicati.CommandLine.BackendTester.csproj", "{E7280DCA-7776-4A73-B9B5-41FD77FC8799}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Duplicati.Library.Interface", "Duplicati\Library\Interface\Duplicati.Library.Interface.csproj", "{C5899F45-B0FF-483C-9D38-24A9FCAAB237}" @@ -234,10 +232,6 @@ Global {BAE27510-8B5D-44B2-B33E-372A98908041}.Debug|Any CPU.Build.0 = Debug|Any CPU {BAE27510-8B5D-44B2-B33E-372A98908041}.Release|Any CPU.ActiveCfg = Release|Any CPU {BAE27510-8B5D-44B2-B33E-372A98908041}.Release|Any CPU.Build.0 = Release|Any CPU - {1BFAE226-8364-4086-825C-BB83F6F3EE4C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {1BFAE226-8364-4086-825C-BB83F6F3EE4C}.Debug|Any CPU.Build.0 = Debug|Any CPU - {1BFAE226-8364-4086-825C-BB83F6F3EE4C}.Release|Any CPU.ActiveCfg = Release|Any CPU - {1BFAE226-8364-4086-825C-BB83F6F3EE4C}.Release|Any CPU.Build.0 = Release|Any CPU {E7280DCA-7776-4A73-B9B5-41FD77FC8799}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {E7280DCA-7776-4A73-B9B5-41FD77FC8799}.Debug|Any CPU.Build.0 = Debug|Any CPU {E7280DCA-7776-4A73-B9B5-41FD77FC8799}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -553,7 +547,6 @@ Global {C03F6DFD-805A-4BE0-9338-64870ADDB4A2} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4} {81765A64-3661-4E3E-B850-2F6F87A51F74} = {D19A38DD-68F1-4EF5-BF5F-8966CE0D9A5B} {BAE27510-8B5D-44B2-B33E-372A98908041} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4} - {1BFAE226-8364-4086-825C-BB83F6F3EE4C} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4} {E7280DCA-7776-4A73-B9B5-41FD77FC8799} = {D19A38DD-68F1-4EF5-BF5F-8966CE0D9A5B} {0CA86ECF-5BEC-4909-B4F6-110A03B30B92} = {566EBBDA-19A4-4056-A615-D901D57D2439} {C0270709-2A40-43B5-8CF1-69581B9FA2A1} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4} diff --git a/Duplicati/Agent/Program.cs b/Duplicati/Agent/Program.cs index 53df9f5c4..d5d0351c5 100644 --- a/Duplicati/Agent/Program.cs +++ b/Duplicati/Agent/Program.cs @@ -30,9 +30,9 @@ using Duplicati.Library.Interface; using Duplicati.Library.Logging; using Duplicati.Library.Main; using Duplicati.Library.RemoteControl; -using Duplicati.Library.RestAPI; using Duplicati.Library.Utility; using Duplicati.Server; +using Duplicati.WebserverCore.Abstractions; using Duplicati.WebserverCore.Services; using Microsoft.Extensions.DependencyInjection; @@ -225,6 +225,9 @@ public static class Program if (string.IsNullOrWhiteSpace(agentConfig.SettingsEncryptionKey)) agentConfig = agentConfig with { SettingsEncryptionKey = Environment.GetEnvironmentVariable(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME) }; + // Prepare the application settings for the server + var applicationSettings = new ApplicationSettings(); + // Apply the secret provider, if present if (!string.IsNullOrWhiteSpace(agentConfig.SecretProvider)) { @@ -245,7 +248,7 @@ public static class Program options["secret-provider-cache"] = agentConfig.SecretProviderCache.ToString(); options["secret-provider-pattern"] = agentConfig.SecretProviderPattern; - FIXMEGlobal.SecretProvider = await SecretProviderHelper.ApplySecretProviderAsync([], [], options, TempFolder.SystemTempPath, null, CancellationToken.None); + applicationSettings.SecretProvider = await SecretProviderHelper.ApplySecretProviderAsync([], [], options, TempFolder.SystemTempPath, null, CancellationToken.None); // Apply the secret provider to the agent configuration foreach (var prop in agentProps) @@ -299,7 +302,7 @@ public static class Program } var t = await Task.WhenAny( - Task.Run(() => StartLocalServer(agentConfig, settings, cts.Token)), + Task.Run(() => StartLocalServer(agentConfig, applicationSettings, settings, cts.Token)), KeepRemoteConnection.Start( settings.ServerUrl, settings.JWT, @@ -307,7 +310,7 @@ public static class Program settings.ServerCertificates, cts.Token, OnConnect, - m => ReKey(m, agentConfig), + m => ReKey(applicationSettings, m, agentConfig), OnControl, OnMessage ) @@ -320,16 +323,16 @@ public static class Program } private static Task> OnConnect(Dictionary metadata) - => FIXMEGlobal.Provider.GetRequiredService().OnConnect(metadata); + => Server.Program.DuplicatiWebserver.Provider.GetRequiredService().OnConnect(metadata); private static Task OnControl(KeepRemoteConnection.ControlMessage message) - => FIXMEGlobal.Provider.GetRequiredService().OnControl(message); + => Server.Program.DuplicatiWebserver.Provider.GetRequiredService().OnControl(message); private static Task OnMessage(KeepRemoteConnection.CommandMessage message) - => FIXMEGlobal.Provider.GetRequiredService().OnMessage(message); + => Server.Program.DuplicatiWebserver.Provider.GetRequiredService().OnMessage(message); - private static Task ReKey(ClaimedClientData keydata, CommandLineArguments agentConfig) + private static Task ReKey(IApplicationSettings applicationSettings, ClaimedClientData keydata, CommandLineArguments agentConfig) { // ReKey is handled here because we store the config outside of the database Log.WriteMessage(LogMessageType.Verbose, LogTag, "ReKey", "Rekeying the settings"); @@ -340,7 +343,7 @@ public static class Program settings = settings with { ServerCertificates = keydata.ServerCertificates }; // Only allow re-keying if the settings encryption key is not set - if (!FIXMEGlobal.SettingsEncryptionKeyProvidedExternally) + if (!applicationSettings.SettingsEncryptionKeyProvidedExternally) { if (!string.IsNullOrWhiteSpace(keydata.LocalEncryptionKey) && settings.SettingsEncryptionKey != keydata.LocalEncryptionKey) { @@ -358,14 +361,15 @@ public static class Program /// /// Runs the server, restarting on crashes /// + /// The application settings for the server /// The commandline arguments passed to the server /// The cancellation token to use for the process /// An awaitable task - private static async Task RunServer(string[] args, CancellationToken cancellationToken) + private static async Task RunServer(IApplicationSettings applicationSettings, string[] args, CancellationToken cancellationToken) { cancellationToken.Register(() => { - Server.Program.ApplicationExitEvent.Set(); + applicationSettings.ApplicationExitEvent.Set(); }); var lastRestart = DateTime.Now; @@ -374,7 +378,7 @@ public static class Program try { lastRestart = DateTime.Now; - Server.Program.Main(args); + Server.Program.Main(applicationSettings, args); } catch (Exception ex) { @@ -445,10 +449,11 @@ public static class Program /// Starts the local webserver in locked down mode, restarting on crashes /// /// The agent configuration + /// The application settings for the server /// The settings for the agent /// The cancellation token that stops the server /// An awaitable task - private static async Task StartLocalServer(CommandLineArguments agentConfig, Settings settings, CancellationToken cancellationToken) + private static async Task StartLocalServer(CommandLineArguments agentConfig, IApplicationSettings applicationSettings, Settings settings, CancellationToken cancellationToken) { // TODO: Look into pipes for Kestrel to prevent network access @@ -487,10 +492,10 @@ public static class Program .ToArray(); // Set the global origin - FIXMEGlobal.Origin = "Agent"; + applicationSettings.Origin = "Agent"; // Start the server - await RunServer(args, cancellationToken); + await RunServer(applicationSettings, args, cancellationToken); } } diff --git a/Duplicati/CommandLine/ServerUtil/Connection.cs b/Duplicati/CommandLine/ServerUtil/Connection.cs index 809cba5de..0df198122 100644 --- a/Duplicati/CommandLine/ServerUtil/Connection.cs +++ b/Duplicati/CommandLine/ServerUtil/Connection.cs @@ -24,6 +24,7 @@ using System.Net.Security; using System.Text.Json; using Duplicati.Library.AutoUpdater; using Duplicati.WebserverCore.Middlewares; +using Duplicati.WebserverCore.Services; namespace Duplicati.CommandLine.ServerUtil; @@ -141,7 +142,6 @@ public class Connection /// The connection public static async Task Connect(Settings settings, bool obtainRefreshToken = false, OutputInterceptor? console = null) { - if (console != null) console.AppendConsoleMessage($"Connecting to {settings.HostUrl}..."); else @@ -201,7 +201,7 @@ public class Connection if (File.Exists(Path.Combine(DataFolderManager.GetDataFolder(DataFolderManager.AccessMode.ProbeOnly), DataFolderManager.SERVER_DATABASE_FILENAME))) { string? cfg = null; - using (var connection = Server.Program.GetDatabaseConnection(opts, true)) + using (var connection = Server.Program.GetDatabaseConnection(new ApplicationSettings(), opts, true)) { cfg = connection.ApplicationSettings.JWTConfig; if (settings.HostUrl.Scheme == "https" && connection.ApplicationSettings.ServerSSLCertificate != null && trustedCertificateHashes.Count == 0) diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/HostedInstanceKeeper.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/HostedInstanceKeeper.cs index 2eea00385..d8e5f6550 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/HostedInstanceKeeper.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/HostedInstanceKeeper.cs @@ -21,6 +21,7 @@ using System; using System.Linq; +using Duplicati.WebserverCore.Abstractions; namespace Duplicati.GUI.TrayIcon { @@ -32,9 +33,11 @@ namespace Duplicati.GUI.TrayIcon private readonly System.Threading.Thread m_runner; private Exception m_runnerException = null; public Action InstanceShutdown; + private readonly IApplicationSettings applicationSettings; - public HostedInstanceKeeper(string[] args) + public HostedInstanceKeeper(IApplicationSettings applicationSettings, string[] args) { + this.applicationSettings = applicationSettings; m_runner = new System.Threading.Thread(_ => { try @@ -44,13 +47,13 @@ namespace Duplicati.GUI.TrayIcon if (args == null || !args.Any(x => x.Trim().StartsWith("--" + Server.WebServerLoader.OPTION_PORT + "=", StringComparison.OrdinalIgnoreCase))) args = (args ?? new string[0]).Union(new string[] { "--" + Server.WebServerLoader.OPTION_PORT + "=8200,8300,8400,8500,8600,8700,8800,8900,8989" }).ToArray(); - Server.Program.Main(args); + Server.Program.Main(applicationSettings, args); } catch (Exception ex) { m_runnerException = ex; Server.Program.ServerStartedEvent?.Set(); - Server.Program.ApplicationExitEvent?.Set(); + applicationSettings.ApplicationExitEvent?.Set(); } finally { @@ -85,7 +88,7 @@ namespace Duplicati.GUI.TrayIcon { try { - Server.Program.ApplicationExitEvent.Set(); + applicationSettings.ApplicationExitEvent.Set(); if (!m_runner.Join(TimeSpan.FromSeconds(10))) { m_runner.Interrupt(); diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/HttpServerConnection.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/HttpServerConnection.cs index 0f3ada47a..d0b9f497c 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/HttpServerConnection.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/HttpServerConnection.cs @@ -334,9 +334,10 @@ namespace Duplicati.GUI.TrayIcon private async Task ObtainAccessTokenAsync() { // If we host the server, issue the access token from the service - if (FIXMEGlobal.IsServerStarted && m_passwordSource == Program.PasswordSource.HostedServer) + var sp = Server.Program.DuplicatiWebserver?.Provider; + if (sp != null && m_passwordSource == Program.PasswordSource.HostedServer) { - var provider = FIXMEGlobal.Provider.GetRequiredService(); + var provider = sp.GetRequiredService(); m_accesstoken = provider.CreateAccessToken("trayicon", provider.TemporaryFamilyId); return; } @@ -412,12 +413,13 @@ namespace Duplicati.GUI.TrayIcon string signinjwt = null; // If we host the server, issue the token from the service - if (FIXMEGlobal.IsServerStarted && m_passwordSource == Program.PasswordSource.HostedServer) + var sp = Server.Program.DuplicatiWebserver?.Provider; + if (sp != null && m_passwordSource == Program.PasswordSource.HostedServer) { - if (FIXMEGlobal.DataConnection.ApplicationSettings.DisableSigninTokens) + if (sp.GetRequiredService().ApplicationSettings.DisableSigninTokens) return null; - signinjwt = FIXMEGlobal.Provider.GetRequiredService().CreateSigninToken("trayicon"); + signinjwt = sp.GetRequiredService().CreateSigninToken("trayicon"); } // If we have database access, grab the issuer key from the db and issue a token diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs index 6fccabd6c..b16ccfad0 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs @@ -30,9 +30,11 @@ using CoCoL; using Duplicati.CommandLine; using Duplicati.Library.AutoUpdater; using Duplicati.Library.Interface; -using Duplicati.Library.RestAPI; using Duplicati.Library.Utility; using Duplicati.Server; +using Duplicati.Server.Database; +using Duplicati.WebserverCore.Services; +using Microsoft.Extensions.DependencyInjection; using Uri = System.Uri; namespace Duplicati.GUI.TrayIcon @@ -144,12 +146,13 @@ namespace Duplicati.GUI.TrayIcon try { // Tell the hosted server it was started by the TrayIcon - FIXMEGlobal.Origin = "Tray icon"; + var applicationSettings = new ApplicationSettings(); + applicationSettings.Origin = "Tray icon"; passwordSource = PasswordSource.HostedServer; // Ignore TrayIcon specific settings foreach (var c in BasicSupportedCommands.Select(x => x.Name)) Server.Program.ValidationIgnoredOptions.Add(c); - hosted = new HostedInstanceKeeper(_args); + hosted = new HostedInstanceKeeper(applicationSettings, _args); } catch (Exception ex) { @@ -164,17 +167,18 @@ namespace Duplicati.GUI.TrayIcon // We have a hosted server, if this is the first run, // we should open the main page - openui = Server.Program.IsFirstRun || Server.Program.ServerPortChanged; + var connection = Server.Program.DuplicatiWebserver.Provider.GetRequiredService(); + openui = connection.ApplicationSettings.IsFirstRun || connection.ApplicationSettings.ServerPortChanged; - var scheme = Server.Program.DataConnection.ApplicationSettings.UseHTTPS ? "https" : "http"; + var scheme = connection.ApplicationSettings.UseHTTPS ? "https" : "http"; serverURL = new UriBuilder(serverURL) { - Port = Server.Program.ServerPort, + Port = Server.Program.DuplicatiWebserver.Port, Scheme = scheme }.Uri; - if (Server.Program.DataConnection.ApplicationSettings.UseHTTPS && string.IsNullOrWhiteSpace(acceptedHostCertificate)) - acceptedHostCertificate = Server.Program.DataConnection.ApplicationSettings.ServerSSLCertificate?.FirstOrDefault(x => x.HasPrivateKey)?.GetCertHashString(); + if (connection.ApplicationSettings.UseHTTPS && string.IsNullOrWhiteSpace(acceptedHostCertificate)) + acceptedHostCertificate = connection.ApplicationSettings.ServerSSLCertificate?.FirstOrDefault(x => x.HasPrivateKey)?.GetCertHashString(); } else if (Utility.ParseBoolOption(options, READCONFIGFROMDB_OPTION)) @@ -182,7 +186,7 @@ namespace Duplicati.GUI.TrayIcon if (File.Exists(Path.Combine(DataFolderManager.GetDataFolder(DataFolderManager.AccessMode.ReadWritePermissionSet), DataFolderManager.SERVER_DATABASE_FILENAME))) { passwordSource = PasswordSource.Database; - databaseConnection = Server.Program.GetDatabaseConnection(options, true); + databaseConnection = Server.Program.GetDatabaseConnection(new ApplicationSettings(), options, true); if (databaseConnection != null) { @@ -267,8 +271,9 @@ No password provided, unable to connect to server, exiting"); } tk.ShowUrlInWindow(t.Result); - Server.Program.IsFirstRun = false; - Server.Program.ServerPortChanged = false; + var connection = Server.Program.DuplicatiWebserver.Provider.GetRequiredService(); + connection.ApplicationSettings.IsFirstRun = false; + connection.ApplicationSettings.ServerPortChanged = false; }); } diff --git a/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs b/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs index d77909b41..72b1aaa9e 100644 --- a/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs +++ b/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs @@ -198,9 +198,14 @@ namespace Duplicati.Library.Backend.AliyunOSS var client = GetClient(); try { + var metadata = new ObjectMetadata + { + ContentLength = stream.Length + }; + using var timeoutStream = stream.ObserveReadTimeout(_timeouts.ReadWriteTimeout, false); var objectResult = await Task.Factory.FromAsync( - (cb, state) => client.BeginPutObject(bucketName, objectName, timeoutStream, cb, state), + (cb, state) => client.BeginPutObject(bucketName, objectName, timeoutStream, metadata, cb, state), client.EndPutObject, null).ConfigureAwait(false); diff --git a/Duplicati/Library/Backend/Backblaze/B2.cs b/Duplicati/Library/Backend/Backblaze/B2.cs index 63c7a04c0..967c41a68 100644 --- a/Duplicati/Library/Backend/Backblaze/B2.cs +++ b/Duplicati/Library/Backend/Backblaze/B2.cs @@ -310,43 +310,8 @@ public class B2 : IStreamingBackend public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { - TempFile? tmp = null; - - var measure = stream; - while (measure is OverrideableStream os && - os.GetType().GetField("m_basestream", BindingFlags.NonPublic | BindingFlags.Instance)?.GetValue(os) is Stream baseStream) - { - measure = baseStream; - } - - if (measure == null) - throw new Exception($"Unable to unwrap stream from: {stream.GetType()}"); - - string sha1; - if (measure.CanSeek) - { - var p = measure.Position; - - // Compute the hash - using (var hashalg = HashFactory.CreateHasher("SHA1")) - sha1 = Utility.Utility.ByteArrayAsHexString(hashalg.ComputeHash(measure)); - - measure.Position = p; - } - else - { - // No seeking possible, use a temp file - tmp = new TempFile(); - await using (var sr = File.OpenWrite(tmp)) - using (var hasher = HashFactory.CreateHasher("SHA1")) - await using (var hc = new HashCalculatingStream(measure, hasher)) - { - await Utility.Utility.CopyStreamAsync(hc, sr, cancelToken).ConfigureAwait(false); - sha1 = hc.GetFinalHashString(); - } - - stream = File.OpenRead(tmp); - } + (stream, var sha1, var tmp) = await Utility.Utility.CalculateThrottledStreamHash(stream, "SHA1", cancelToken).ConfigureAwait(false); + using var _ = tmp; if (_filecache == null) await RebuildFileCache(cancelToken).ConfigureAwait(false); @@ -400,10 +365,6 @@ public class B2 : IStreamingBackend throw; } - finally - { - tmp?.Dispose(); - } } /// diff --git a/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs b/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs deleted file mode 100644 index 956cbd1f3..000000000 --- a/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs +++ /dev/null @@ -1,450 +0,0 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -using Duplicati.Library.Common.IO; -using Duplicati.Library.Interface; -using Duplicati.Library.Utility; -using Duplicati.Library.Utility.Options; -using System.Net; -using System.Runtime.CompilerServices; -using System.Security.Cryptography; - -namespace Duplicati.Library.Backend -{ - // ReSharper disable once UnusedMember.Global - // This class is instantiated dynamically in the BackendLoader. - public class CloudFiles : IBackend, IStreamingBackend - { - /// - /// The log tag for this class - /// - private static readonly string LOGTAG = Logging.Log.LogTagFromType(); - - public const string AUTH_URL_US = "https://identity.api.rackspacecloud.com/auth"; - public const string AUTH_URL_UK = "https://lon.auth.api.rackspacecloud.com/v1.0"; - private const string DUMMY_HOSTNAME = "api.mosso.com"; - - private const string AUTH_USERNAME_OPTION = "cloudfiles-username"; - private const string AUTH_PASSWORD_OPTION = "cloudfiles-accesskey"; - - private const int ITEM_LIST_LIMIT = 1000; - private readonly string m_username; - private readonly string m_password; - private readonly string m_path; - - private string? m_storageUrl = null; - private string? m_authToken = null; - private readonly string m_authUrl; - private readonly TimeoutOptionsHelper.Timeouts m_timeouts; - - // ReSharper disable once UnusedMember.Global - // This constructor is needed by the BackendLoader. - public CloudFiles() - { - m_username = null!; - m_password = null!; - m_path = null!; - m_authUrl = null!; - m_timeouts = null!; - } - - // ReSharper disable once UnusedMember.Global - // This constructor is needed by the BackendLoader. - public CloudFiles(string url, Dictionary options) - { - var uri = new Utility.Uri(url); - var auth = AuthOptionsHelper.ParseWithAlias(options, uri, AUTH_USERNAME_OPTION, AUTH_PASSWORD_OPTION); - - if (!auth.HasUsername) - throw new UserInformationException(Strings.CloudFiles.NoUserIDError, "CloudFilesNoUserID"); - if (auth.HasPassword) - throw new UserInformationException(Strings.CloudFiles.NoAPIKeyError, "CloudFilesNoApiKey"); - - (m_username, m_password) = auth.GetCredentials(); - - //Fallback to the previous format - if (url.Contains(DUMMY_HOSTNAME)) - { - Logging.Log.WriteWarningMessage(LOGTAG, "CloudFilesDeprecatedFormat", null, Strings.CloudFiles.DeprecatedFormatWarning(DUMMY_HOSTNAME)); - var u = new System.Uri(url); - - if (!string.IsNullOrEmpty(u.UserInfo)) - { - if (u.UserInfo.IndexOf(":", StringComparison.Ordinal) >= 0) - { - m_username = u.UserInfo.Substring(0, u.UserInfo.IndexOf(":", StringComparison.Ordinal)); - m_password = u.UserInfo.Substring(u.UserInfo.IndexOf(":", StringComparison.Ordinal) + 1); - } - else - { - m_username = u.UserInfo; - } - } - - //We use the api.mosso.com hostname. - //This allows the use of containers that have names that are not valid hostnames, - // such as container names with spaces in them - if (u.Host.Equals(DUMMY_HOSTNAME)) - m_path = Library.Utility.Uri.UrlDecode(u.PathAndQuery); - else - m_path = u.Host + Library.Utility.Uri.UrlDecode(u.PathAndQuery); - } - else - { - m_path = uri.HostAndPath; - } - - if (m_path.EndsWith("/", StringComparison.Ordinal)) - m_path = m_path.Substring(0, m_path.Length - 1); - if (!m_path.StartsWith("/", StringComparison.Ordinal)) - m_path = "/" + m_path; - - var authUrl = options.GetValueOrDefault("cloudfiles-authentication-url"); - if (string.IsNullOrEmpty(authUrl)) - authUrl = Utility.Utility.ParseBoolOption(options, "cloudfiles-uk-account") ? AUTH_URL_UK : AUTH_URL_US; - m_authUrl = authUrl; - m_timeouts = TimeoutOptionsHelper.Parse(options); - } - - #region IBackend Members - - public string DisplayName - { - get { return Strings.CloudFiles.DisplayName; } - } - - public string ProtocolKey - { - get { return "cloudfiles"; } - } - - public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) - { - var extraUrl = "?format=xml&limit=" + ITEM_LIST_LIMIT.ToString(); - var markerUrl = ""; - - bool repeat; - do - { - var doc = new System.Xml.XmlDocument(); - var req = await CreateRequest("", extraUrl + markerUrl, cancelToken).ConfigureAwait(false); - - try - { - await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ => - { - var areq = new Utility.AsyncHttpRequest(req); - using (var resp = (HttpWebResponse)areq.GetResponse()) - using (var s = areq.GetResponseStream()) - doc.Load(s); - }).ConfigureAwait(false); - } - catch (WebException wex) - { - if (markerUrl == "") //Only check on first iteration - if (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound) - throw new FolderMissingException(wex); - - //Other error, just re-throw - throw; - } - - var lst = doc.SelectNodes("container/object"); - - //Perhaps the folder does not exist? - //The response should be 404 from the server, but it is not :( - if (lst == null || lst.Count == 0 && markerUrl == "") //Only on first iteration - { - try { await CreateFolderAsync(cancelToken).ConfigureAwait(false); } - catch { } //Ignore - } - - if (lst == null) - yield break; - - var lastItemName = ""; - foreach (System.Xml.XmlNode n in lst) - { - var name = n["name"]?.InnerText; - if (!long.TryParse(n["bytes"]?.InnerText, out var size)) - size = -1; - if (!DateTime.TryParse(n["last_modified"]?.InnerText, out var mod)) - mod = new DateTime(); - - lastItemName = name; - yield return new FileEntry(name, size, mod, mod); - } - - repeat = lst.Count == ITEM_LIST_LIMIT; - - if (repeat) - markerUrl = "&marker=" + Library.Utility.Uri.UrlEncode(lastItemName); - - } while (repeat); - } - - public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) - { - using (var fs = File.OpenRead(filename)) - await PutAsync(remotename, fs, cancelToken); - } - - public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken) - { - using (var fs = File.Create(filename)) - await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false); - } - - public async Task DeleteAsync(string remotename, CancellationToken cancelToken) - { - try - { - var req = await CreateRequest("/" + remotename, "", cancelToken).ConfigureAwait(false); - await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => - { - req.Method = "DELETE"; - var areq = new AsyncHttpRequest(req); - using (var resp = (HttpWebResponse)areq.GetResponse()) - { - if (resp.StatusCode == HttpStatusCode.NotFound) - throw new FileMissingException(); - - if ((int)resp.StatusCode >= 300) - throw new WebException(Strings.CloudFiles.FileDeleteError, null, WebExceptionStatus.ProtocolError, resp); - else - using (areq.GetResponseStream()) - { } - } - }).ConfigureAwait(false); - } - catch (WebException wex) - { - if (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound) - throw new FileMissingException(wex); - else - throw; - } - } - - public IList SupportedCommands => - [ - new CommandLineArgument(AUTH_USERNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.CloudFiles.DescriptionUsernameShort, Strings.CloudFiles.DescriptionUsernameLong, null, [AuthOptionsHelper.AuthUsernameOption] ), - new CommandLineArgument(AUTH_PASSWORD_OPTION, CommandLineArgument.ArgumentType.Password, Strings.CloudFiles.DescriptionPasswordShort, Strings.CloudFiles.DescriptionPasswordLong, null, [AuthOptionsHelper.AuthPasswordOption]), - new CommandLineArgument("cloudfiles-uk-account", CommandLineArgument.ArgumentType.Boolean, Strings.CloudFiles.DescriptionUKAccountShort, Strings.CloudFiles.DescriptionUKAccountLong("cloudfiles-authentication-url", AUTH_URL_UK)), - new CommandLineArgument("cloudfiles-authentication-url", CommandLineArgument.ArgumentType.String, Strings.CloudFiles.DescriptionAuthenticationURLShort, Strings.CloudFiles.DescriptionAuthenticationURLLong_v2("cloudfiles-uk-account"), AUTH_URL_US), - .. TimeoutOptionsHelper.GetOptions(), - ]; - - public string Description - { - get { return Strings.CloudFiles.Description_v2; } - } - - #endregion - - #region IBackend_v2 Members - - public Task TestAsync(CancellationToken cancelToken) => - //The "Folder not found" is not detectable :( - this.TestListAsync(cancelToken); - - public async Task CreateFolderAsync(CancellationToken cancelToken) - { - var createReq = await CreateRequest("", "", cancelToken).ConfigureAwait(false); - await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => - { - createReq.Method = "PUT"; - var areq = new AsyncHttpRequest(createReq); - using (var resp = (HttpWebResponse)areq.GetResponse()) - { } - }).ConfigureAwait(false); - } - - #endregion - - #region IDisposable Members - - public void Dispose() - { - } - - #endregion - - #region IStreamingBackend Members - - public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult( - new string?[] { - new System.Uri(m_authUrl).Host, - string.IsNullOrWhiteSpace(m_storageUrl) ? null : new System.Uri(m_storageUrl).Host - } - .WhereNotNullOrWhiteSpace() - .ToArray() - ); - - public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) - { - var req = await CreateRequest("/" + remotename, "", cancelToken).ConfigureAwait(false); - req.Method = "GET"; - - var areq = new AsyncHttpRequest(req); - using (var resp = areq.GetResponse()) - using (var s = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponseStream()).ConfigureAwait(false)) - using (var timeoutStream = s.ObserveReadTimeout(m_timeouts.ReadWriteTimeout)) - using (var hasher = MD5.Create()) - using (var mds = new HashCalculatingStream(timeoutStream, hasher)) - { - var md5Hash = resp.Headers["ETag"]; - await Utility.Utility.CopyStreamAsync(mds, stream, true, cancelToken).ConfigureAwait(false); - - if (!string.Equals(mds.GetFinalHashString(), md5Hash, StringComparison.OrdinalIgnoreCase)) - throw new Exception(Strings.CloudFiles.ETagVerificationError); - } - } - - public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) - { - var req = await CreateRequest("/" + remotename, "", cancelToken).ConfigureAwait(false); - req.Method = "PUT"; - req.ContentType = "application/octet-stream"; - - try { req.ContentLength = stream.Length; } - catch { } - - // TODO: When reviewing this, lets build a common method to unwrap the stream passed from - // BackendManager.PutOperation, so we can use the same logic to compute hashes in all backends - - //If we can pre-calculate the MD5 hash before transmission, do so - /*if (stream.CanSeek) - { - System.Security.Cryptography.MD5 md5 = System.Security.Cryptography.MD5.Create(); - req.Headers["ETag"] = Core.Utility.ByteArrayAsHexString(md5.ComputeHash(stream)).ToLower(System.Globalization.CultureInfo.InvariantCulture); - stream.Seek(0, System.IO.SeekOrigin.Begin); - - using (System.IO.Stream s = req.GetRequestStream()) - Core.Utility.CopyStream(stream, s); - - //Reset the timeout to the default value of 100 seconds to - // avoid blocking the GetResponse() call - req.Timeout = 100000; - - //The server handles the eTag verification for us, and gives an error if the hash was a mismatch - using (HttpWebResponse resp = (HttpWebResponse)req.GetResponse()) - if ((int)resp.StatusCode >= 300) - throw new WebException(Strings.CloudFiles.FileUploadError, null, WebExceptionStatus.ProtocolError, resp); - - } - else //Otherwise use a client-side calculation - */ - //TODO: We cannot use the local MD5 calculation, because that could involve a throttled read, - // and may invoke various events - { - string? fileHash = null; - - long streamLen = -1; - try { streamLen = stream.Length; } - catch { } - - var areq = new AsyncHttpRequest(req); - using (var s = areq.GetRequestStream(streamLen)) - using (var timeoutStream = s.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout)) - using (var hasher = MD5.Create()) - using (var mds = new HashCalculatingStream(timeoutStream, hasher)) - { - await Utility.Utility.CopyStreamAsync(stream, mds, tryRewindSource: true, cancelToken: cancelToken); - fileHash = mds.GetFinalHashString(); - } - - string? md5Hash = null; - - //We need to verify the eTag locally - try - { - using (HttpWebResponse resp = (HttpWebResponse)areq.GetResponse()) - if ((int)resp.StatusCode >= 300) - throw new WebException(Strings.CloudFiles.FileUploadError, null, WebExceptionStatus.ProtocolError, resp); - else - md5Hash = resp.Headers["ETag"]; - } - catch (WebException wex) - { - //Catch 404 and turn it into a FolderNotFound error - if (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound) - throw new FolderMissingException(wex); - - //Other error, just re-throw - throw; - } - - - if (md5Hash == null || !string.Equals(md5Hash, fileHash, StringComparison.OrdinalIgnoreCase)) - { - //Remove the broken file - try { await DeleteAsync(remotename, cancelToken); } - catch { } - - throw new Exception(Strings.CloudFiles.ETagVerificationError); - } - } - } - - #endregion - - private async Task CreateRequest(string remotename, string query, CancellationToken cancelToken) - { - //If this is the first call, get an authentication token - if (string.IsNullOrEmpty(m_authToken) || string.IsNullOrEmpty(m_storageUrl)) - { - await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => - { - var authReq = (HttpWebRequest)HttpWebRequest.Create(m_authUrl); - authReq.Headers.Add("X-Auth-User", m_username); - authReq.Headers.Add("X-Auth-Key", m_password); - authReq.Method = "GET"; - - var areq = new AsyncHttpRequest(authReq); - using (var resp = areq.GetResponse()) - { - m_storageUrl = resp.Headers["X-Storage-Url"]; - m_authToken = resp.Headers["X-Auth-Token"]; - } - - if (string.IsNullOrEmpty(m_authToken) || string.IsNullOrEmpty(m_storageUrl)) - throw new Exception(Strings.CloudFiles.UnexpectedResponseError); - }).ConfigureAwait(false); - } - - var req = (HttpWebRequest)HttpWebRequest.Create(m_storageUrl + UrlEncode(m_path + remotename) + query); - req.Headers.Add("X-Auth-Token", UrlEncode(m_authToken!)); - - req.UserAgent = "Duplicati CloudFiles Backend v" + System.Reflection.Assembly.GetExecutingAssembly().GetName().Version; - req.KeepAlive = false; - req.PreAuthenticate = true; - req.AllowWriteStreamBuffering = false; - - return req; - } - - private static string UrlEncode(string value) - { - return Utility.Uri.UrlEncode(value).Replace("+", "%20").Replace("%2f", "/"); - } - } -} diff --git a/Duplicati/Library/Backend/CloudFiles/Duplicati.Library.Backend.CloudFiles.csproj b/Duplicati/Library/Backend/CloudFiles/Duplicati.Library.Backend.CloudFiles.csproj deleted file mode 100644 index 9f355da46..000000000 --- a/Duplicati/Library/Backend/CloudFiles/Duplicati.Library.Backend.CloudFiles.csproj +++ /dev/null @@ -1,25 +0,0 @@ - - - - net8.0 - Library - Duplicati.Library.Backend - Copyright © 2025 Team Duplicati, MIT license - enable - enable - - - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - diff --git a/Duplicati/Library/Backend/CloudFiles/Strings.cs b/Duplicati/Library/Backend/CloudFiles/Strings.cs deleted file mode 100644 index acb5dc405..000000000 --- a/Duplicati/Library/Backend/CloudFiles/Strings.cs +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. -using Duplicati.Library.Localization.Short; -namespace Duplicati.Library.Backend.Strings -{ - internal static class CloudFiles - { - public static string Description_v2 { get { return LC.L(@"This backend can read and write data to CloudFiles. Allowed format is ""cloudfiles://container/folder""."); } } - public static string DisplayName { get { return LC.L(@"Rackspace CloudFiles"); } } - public static string DescriptionAuthenticationURLLong_v2(string optionname) { return LC.L(@"CloudFiles use different servers for authentication based on where the account resides. Use this option to set an alternate authentication URL. This option overrides --{0}.", optionname); } - public static string DescriptionAuthenticationURLShort { get { return LC.L(@"Provide another authentication URL"); } } - public static string DescriptionPasswordLong { get { return LC.L(@"The API Access Key used to authenticate with CloudFiles."); } } - public static string DescriptionPasswordShort { get { return LC.L(@"Supply the access key used to connect to the server"); } } - public static string DescriptionUKAccountLong(string optionname, string optionvalue) { return LC.L(@"Duplicati will assume that the credentials given are for a US account. Use this option if the account is a UK based account. Note that this is equivalent to setting --{0}={1}.", optionname, optionvalue); } - public static string DescriptionUKAccountShort { get { return LC.L(@"Use a UK account"); } } - public static string DescriptionUsernameLong { get { return LC.L(@"The username used to authenticate with CloudFiles."); } } - public static string DescriptionUsernameShort { get { return LC.L(@"Supply the username used to authenticate with CloudFiles"); } } - public static string ETagVerificationError { get { return LC.L(@"MD5 Hash (ETag) verification failed"); } } - public static string FileDeleteError { get { return LC.L(@"Failed to delete file"); } } - public static string FileUploadError { get { return LC.L(@"Failed to upload file"); } } - public static string NoAPIKeyError { get { return LC.L(@"No CloudFiles API Access Key given"); } } - public static string NoUserIDError { get { return LC.L(@"No CloudFiles userID given"); } } - public static string UnexpectedResponseError { get { return LC.L(@"Unexpected CloudFiles response. Perhaps the API has changed?"); } } - public static string DeprecatedFormatWarning(string hostname) { return LC.L(@$"The URL is using the marker hostname {hostname} which is no longer supported. Please use the new format ""cloudfiles://container/folder"" instead."); } - } -} diff --git a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs index 10b2705dd..29a51ba17 100644 --- a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs +++ b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs @@ -216,7 +216,7 @@ namespace Duplicati.Library.Backend var (accessKeyId, accessKeySecret) = _auth.GetCredentials(); var host = await GetRegionEndpointAsync("https://api.idrivee2.com/api/service/get_region_end_point/" + accessKeyId, cancellationToken).ConfigureAwait(false); - _s3Client = new S3AwsClient(accessKeyId, accessKeySecret, null, host, null, true, false, _timeouts, _options); + _s3Client = new S3AwsClient(accessKeyId, accessKeySecret, null, host, null, true, false, false, _timeouts, _options); return _s3Client; } diff --git a/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs b/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs index ec4e5f30f..805544398 100644 --- a/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs +++ b/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs @@ -489,108 +489,71 @@ namespace Duplicati.Library.Backend // we specify the file size in a custom header. And if the stream is not seek-able we are not able // to use stream.Length, so we are back at square one. - // TODO: This unwrapping code is shared with CloudFiles - TempFile? tmpFile = null; - Stream? baseStream = stream; - while (baseStream is OverrideableStream) - baseStream = typeof(OverrideableStream).GetField("m_basestream", System.Reflection.BindingFlags.DeclaredOnly | System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic)?.GetValue(baseStream) as System.IO.Stream; - if (baseStream == null) - throw new Exception(string.Format("Unable to unwrap stream from: {0}", stream.GetType())); - string md5Hash; - if (baseStream.CanSeek) - { - var originalPosition = baseStream.Position; - using (var md5 = MD5.Create()) - md5Hash = Utility.Utility.ByteArrayAsHexString(md5.ComputeHash(baseStream)); - baseStream.Position = originalPosition; - } - else - { - // No seeking possible, use a temp file - tmpFile = new TempFile(); - using (var os = File.OpenWrite(tmpFile)) - using (var hasher = MD5.Create()) - using (var md5 = new HashCalculatingStream(baseStream, hasher)) - { - await Utility.Utility.CopyStreamAsync(md5, os, true, cancelToken).ConfigureAwait(false); - md5Hash = md5.GetFinalHashString(); - } - stream = File.OpenRead(tmpFile); - } - try - { - // Create request, with query parameter, and a few custom headers. - // NB: If we wanted to we could send the same POST request as below but without the file contents - // and with "cphash=[md5Hash]" as the only query parameter. Then we will get an HTTP 200 (OK) response - // if an identical file already exists, and we can skip uploading the new file. We will get - // HTTP 404 (Not Found) if file does not exists or it exists with a different hash, in which - // case we must send a new request to upload the new content. - var fileSize = stream.Length; - var req = CreateRequest(WebRequestMethods.Http.Post, remotename, "umode=nomultipart", true); - req.Headers.Add("JMd5", md5Hash); // Not required, but it will make the server verify the content and mark the file as corrupt if there is a mismatch. - req.Headers.Add("JSize", fileSize.ToString()); // Required, and used to mark file as incomplete if we upload something be the total size of the original file! - // File time stamp headers: Since we are working with a stream here we do not know the local file's timestamps, - // and then we can just omit the JCreated and JModified and let the server automatically set the current time. - //req.Headers.Add("JCreated", timeCreated); - //req.Headers.Add("JModified", timeModified); - req.ContentType = "application/octet-stream"; - req.ContentLength = fileSize; + (stream, var md5Hash, var tmp) = await Utility.Utility.CalculateThrottledStreamHash(stream, "MD5", cancelToken).ConfigureAwait(false); + using var _ = tmp; - // Write post data request - var areq = new AsyncHttpRequest(req); - using (var rs = areq.GetRequestStream()) - using (var ts = rs.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout)) - await Utility.Utility.CopyStreamAsync(stream, rs, true, cancelToken).ConfigureAwait(false); - // Send request, and check response - using (var resp = (HttpWebResponse)areq.GetResponse()) - { - if (resp.StatusCode != HttpStatusCode.Created) - throw new WebException(Strings.Jottacloud.FileUploadError, null, WebExceptionStatus.ProtocolError, resp); + // Create request, with query parameter, and a few custom headers. + // NB: If we wanted to we could send the same POST request as below but without the file contents + // and with "cphash=[md5Hash]" as the only query parameter. Then we will get an HTTP 200 (OK) response + // if an identical file already exists, and we can skip uploading the new file. We will get + // HTTP 404 (Not Found) if file does not exists or it exists with a different hash, in which + // case we must send a new request to upload the new content. + var fileSize = stream.Length; + var req = CreateRequest(WebRequestMethods.Http.Post, remotename, "umode=nomultipart", true); + req.Headers.Add("JMd5", md5Hash); // Not required, but it will make the server verify the content and mark the file as corrupt if there is a mismatch. + req.Headers.Add("JSize", fileSize.ToString()); // Required, and used to mark file as incomplete if we upload something be the total size of the original file! + // File time stamp headers: Since we are working with a stream here we do not know the local file's timestamps, + // and then we can just omit the JCreated and JModified and let the server automatically set the current time. + //req.Headers.Add("JCreated", timeCreated); + //req.Headers.Add("JModified", timeModified); + req.ContentType = "application/octet-stream"; + req.ContentLength = fileSize; - // Request seems to be successful, but we must verify the response XML content to be sure that the file - // was correctly uploaded: The server will verify the JSize header and mark the file as incomplete if - // there was mismatch, and it will verify the JMd5 header and mark the file as corrupt if there was a hash - // mismatch. The returned XML contains a file element, and if upload was error free it contains a single - // child element "currentRevision", which has a "state" child element with the string "COMPLETED". - // If there was a problem we should have a "latestRevision" child element, and this will have state with - // value "INCOMPLETE" or "CORRUPT". If the file was new or had no previous complete versions the latestRevision - // will be the only child, but if not there may also be a "currentRevision" representing the previous - // complete version - and then we need to detect the case where our upload failed but there was an existing - // complete version! - using (var rs = areq.GetResponseStream()) + // Write post data request + var areq = new AsyncHttpRequest(req); + using (var rs = areq.GetRequestStream()) + using (var ts = rs.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout)) + await Utility.Utility.CopyStreamAsync(stream, rs, true, cancelToken).ConfigureAwait(false); + // Send request, and check response + using (var resp = (HttpWebResponse)areq.GetResponse()) + { + if (resp.StatusCode != HttpStatusCode.Created) + throw new WebException(Strings.Jottacloud.FileUploadError, null, WebExceptionStatus.ProtocolError, resp); + + // Request seems to be successful, but we must verify the response XML content to be sure that the file + // was correctly uploaded: The server will verify the JSize header and mark the file as incomplete if + // there was mismatch, and it will verify the JMd5 header and mark the file as corrupt if there was a hash + // mismatch. The returned XML contains a file element, and if upload was error free it contains a single + // child element "currentRevision", which has a "state" child element with the string "COMPLETED". + // If there was a problem we should have a "latestRevision" child element, and this will have state with + // value "INCOMPLETE" or "CORRUPT". If the file was new or had no previous complete versions the latestRevision + // will be the only child, but if not there may also be a "currentRevision" representing the previous + // complete version - and then we need to detect the case where our upload failed but there was an existing + // complete version! + using (var rs = areq.GetResponseStream()) + { + var doc = new System.Xml.XmlDocument(); + try { doc.Load(rs); } + catch (System.Xml.XmlException) { - var doc = new System.Xml.XmlDocument(); - try { doc.Load(rs); } - catch (System.Xml.XmlException) - { - throw new WebException(Strings.Jottacloud.FileUploadError, WebExceptionStatus.ProtocolError); - } - bool uploadCompletedSuccessfully = false; - var xFile = doc["file"]; - if (xFile != null) - { - var xRevState = xFile.SelectSingleNode("latestRevision"); - if (xRevState == null) - { - xRevState = xFile.SelectSingleNode("currentRevision/state"); - if (xRevState != null) - uploadCompletedSuccessfully = xRevState.InnerText == "COMPLETED"; // Success: There is no "latestRevision", only a "currentRevision" (and it specifies the file is complete, but I think it always will). - } - } - if (!uploadCompletedSuccessfully) // Report error (and we just let the incomplete/corrupt file revision stay on the server..) - throw new WebException(Strings.Jottacloud.FileUploadError, WebExceptionStatus.ProtocolError); + throw new WebException(Strings.Jottacloud.FileUploadError, WebExceptionStatus.ProtocolError); } + bool uploadCompletedSuccessfully = false; + var xFile = doc["file"]; + if (xFile != null) + { + var xRevState = xFile.SelectSingleNode("latestRevision"); + if (xRevState == null) + { + xRevState = xFile.SelectSingleNode("currentRevision/state"); + if (xRevState != null) + uploadCompletedSuccessfully = xRevState.InnerText == "COMPLETED"; // Success: There is no "latestRevision", only a "currentRevision" (and it specifies the file is complete, but I think it always will). + } + } + if (!uploadCompletedSuccessfully) // Report error (and we just let the incomplete/corrupt file revision stay on the server..) + throw new WebException(Strings.Jottacloud.FileUploadError, WebExceptionStatus.ProtocolError); } } - finally - { - try - { - if (tmpFile != null) - tmpFile.Dispose(); - } - catch { } - } } } } diff --git a/Duplicati/Library/Backend/Rclone/Rclone.cs b/Duplicati/Library/Backend/Rclone/Rclone.cs index 99f1873ff..bb78b96c5 100644 --- a/Duplicati/Library/Backend/Rclone/Rclone.cs +++ b/Duplicati/Library/Backend/Rclone/Rclone.cs @@ -158,9 +158,9 @@ namespace Duplicati.Library.Backend process.BeginOutputReadLine(); process.BeginErrorReadLine(); - using var timer = timeout == Timeout.InfiniteTimeSpan + var timer = timeout == Timeout.InfiniteTimeSpan ? new TaskCompletionSource().Task - : Task.Delay(timeout); + : Task.Delay(timeout, cancelToken); var killed = false; while (!process.HasExited) diff --git a/Duplicati/Library/Backend/S3/S3AwsClient.cs b/Duplicati/Library/Backend/S3/S3AwsClient.cs index d21ea417e..ae2c5957a 100644 --- a/Duplicati/Library/Backend/S3/S3AwsClient.cs +++ b/Duplicati/Library/Backend/S3/S3AwsClient.cs @@ -61,6 +61,10 @@ namespace Duplicati.Library.Backend /// The option to specify if chunk encoding should be used /// private readonly bool m_useChunkEncoding; + /// + /// The option to specify if payload signing should be disabled + /// + private readonly bool m_disablePayloadSigning; /// /// The DNS host of the S3 server @@ -93,7 +97,7 @@ namespace Duplicati.Library.Backend ]); public S3AwsClient(string awsID, string awsKey, string? locationConstraint, string servername, - string? storageClass, bool useSSL, bool disableChunkEncoding, TimeoutOptionsHelper.Timeouts timeouts, Dictionary options) + string? storageClass, bool useSSL, bool disableChunkEncoding, bool disablePayloadSigning, TimeoutOptionsHelper.Timeouts timeouts, Dictionary options) { var cfg = GetDefaultAmazonS3Config(); cfg.UseHttp = !useSSL; @@ -109,6 +113,7 @@ namespace Duplicati.Library.Backend m_storageClass = storageClass; m_dnsHost = string.IsNullOrWhiteSpace(cfg.ServiceURL) ? null : new System.Uri(cfg.ServiceURL).Host; m_useChunkEncoding = !disableChunkEncoding; + m_disablePayloadSigning = disablePayloadSigning; m_archiveClasses = ParseStorageClasses(options.GetValueOrDefault(S3_ARCHIVE_CLASSES_OPTION)); } @@ -217,13 +222,20 @@ namespace Duplicati.Library.Backend public virtual async Task AddFileStreamAsync(string bucketName, string keyName, Stream source, CancellationToken cancelToken) { + (source, var md5, var tmp) = await Utility.Utility.CalculateThrottledStreamHash(source, "MD5", cancelToken).ConfigureAwait(false); + using var _ = tmp; + + md5 = Convert.ToBase64String(Utility.Utility.HexStringAsByteArray(md5)); + using var ts = source.ObserveReadTimeout(m_timeouts.ReadWriteTimeout, false); var objectAddRequest = new PutObjectRequest { BucketName = bucketName, Key = keyName, InputStream = ts, - UseChunkEncoding = m_useChunkEncoding + UseChunkEncoding = m_useChunkEncoding, + MD5Digest = md5, + DisablePayloadSigning = m_disablePayloadSigning }; if (!string.IsNullOrWhiteSpace(m_storageClass)) objectAddRequest.StorageClass = new S3StorageClass(m_storageClass); diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index 017cce158..b6b21c166 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -41,6 +41,7 @@ namespace Duplicati.Library.Backend private const string SSL_OPTION = "use-ssl"; private const string S3_CLIENT_OPTION = "s3-client"; private const string S3_DISABLE_CHUNK_ENCODING_OPTION = "s3-disable-chunk-encoding"; + private const string S3_DISABLE_PAYLOAD_SIGNING_OPTION = "s3-disable-payload-signing"; private const string S3_LIST_API_VERSION_OPTION = "s3-list-api-version"; private const string S3_RECURSIVE_LIST = "s3-recursive-list"; @@ -242,14 +243,15 @@ namespace Duplicati.Library.Backend if (!options.ContainsKey("s3-ext-forcepathstyle") && !hostname.EndsWith(".amazonaws.com", StringComparison.OrdinalIgnoreCase)) options["s3-ext-forcepathstyle"] = "true"; - var disableChunkEncoding = Utility.Utility.ParseBoolOption(options, S3_DISABLE_CHUNK_ENCODING_OPTION); var s3ClientOptionValue = options.GetValueOrDefault(S3_CLIENT_OPTION); (var awsID, var awsKey) = auth.GetCredentials(); if (string.IsNullOrWhiteSpace(s3ClientOptionValue) || string.Equals(s3ClientOptionValue, "aws", StringComparison.OrdinalIgnoreCase)) { - m_s3Client = new S3AwsClient(awsID, awsKey, locationConstraint, hostname, storageClass, useSSL, disableChunkEncoding, timeout, options); + var disableChunkEncoding = Utility.Utility.ParseBoolOption(options, S3_DISABLE_CHUNK_ENCODING_OPTION); + var disablePayloadSigning = Utility.Utility.ParseBoolOption(options, S3_DISABLE_PAYLOAD_SIGNING_OPTION); + m_s3Client = new S3AwsClient(awsID, awsKey, locationConstraint, hostname, storageClass, useSSL, disableChunkEncoding, disablePayloadSigning, timeout, options); } else if (string.Equals(s3ClientOptionValue, "minio", StringComparison.OrdinalIgnoreCase)) { @@ -328,14 +330,15 @@ namespace Duplicati.Library.Backend var exts = S3AwsClient.GetAwsExtendedOptions(); return [ - new CommandLineArgument("aws-access-key-id", CommandLineArgument.ArgumentType.String, Strings.S3Backend.AMZUserIDDescriptionShort, Strings.S3Backend.AMZUserIDDescriptionLong, null, [AuthOptionsHelper.AuthUsernameOption], null), - new CommandLineArgument("aws-secret-access-key", CommandLineArgument.ArgumentType.Password, Strings.S3Backend.AMZKeyDescriptionShort, Strings.S3Backend.AMZKeyDescriptionLong,null, [AuthOptionsHelper.AuthPasswordOption], null ), + new CommandLineArgument(AUTH_USERNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.S3Backend.AMZUserIDDescriptionShort, Strings.S3Backend.AMZUserIDDescriptionLong, null, [AuthOptionsHelper.AuthUsernameOption], null), + new CommandLineArgument(AUTH_PASSWORD_OPTION, CommandLineArgument.ArgumentType.Password, Strings.S3Backend.AMZKeyDescriptionShort, Strings.S3Backend.AMZKeyDescriptionLong,null, [AuthOptionsHelper.AuthPasswordOption], null ), new CommandLineArgument(STORAGECLASS_OPTION, CommandLineArgument.ArgumentType.String, Strings.S3Backend.S3StorageclassDescriptionShort, Strings.S3Backend.S3StorageclassDescriptionLong, "", null, KNOWN_S3_STORAGE_CLASSES.Select(x => x.Value).WhereNotNullOrWhiteSpace().ToArray()), new CommandLineArgument(SERVER_NAME, CommandLineArgument.ArgumentType.String, Strings.S3Backend.S3ServerNameDescriptionShort, Strings.S3Backend.S3ServerNameDescriptionLong(hostnames.ToString()), DEFAULT_S3_HOST), new CommandLineArgument(LOCATION_OPTION, CommandLineArgument.ArgumentType.String, Strings.S3Backend.S3LocationDescriptionShort, Strings.S3Backend.S3LocationDescriptionLong(locations.ToString())), new CommandLineArgument(SSL_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionUseSSLShort, Strings.S3Backend.DescriptionUseSSLLong), new CommandLineArgument(S3_CLIENT_OPTION, CommandLineArgument.ArgumentType.Enumeration, Strings.S3Backend.S3ClientDescriptionShort, Strings.S3Backend.S3ClientDescriptionLong, "aws", null, new string[] { "aws", "minio" }), new CommandLineArgument(S3_DISABLE_CHUNK_ENCODING_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionDisableChunkEncodingShort, Strings.S3Backend.DescriptionDisableChunkEncodingLong, "false"), + new CommandLineArgument(S3_DISABLE_PAYLOAD_SIGNING_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionDisablePayloadSigningShort, Strings.S3Backend.DescriptionDisablePayloadSigningLong, "false"), new CommandLineArgument(S3AwsClient.S3_ARCHIVE_CLASSES_OPTION, CommandLineArgument.ArgumentType.Flags, Strings.S3Backend.S3ArchiveClassesDescriptionShort, Strings.S3Backend.S3ArchiveClassesDescriptionLong, string.Join(",", S3AwsClient.DEFAULT_ARCHIVE_CLASSES.Select(x => x.Value)), null, KNOWN_S3_STORAGE_CLASSES.Select(x => x.Value).WhereNotNullOrWhiteSpace().ToArray()), new CommandLineArgument(S3_LIST_API_VERSION_OPTION, CommandLineArgument.ArgumentType.Enumeration, Strings.S3Backend.DescriptionListApiVersionShort, Strings.S3Backend.DescriptionListApiVersionLong, "v1", null, ["v1", "v2"]), new CommandLineArgument(S3_RECURSIVE_LIST, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionRecursiveListShort, Strings.S3Backend.DescriptionRecursiveListLong, "false"), diff --git a/Duplicati/Library/Backend/S3/Strings.cs b/Duplicati/Library/Backend/S3/Strings.cs index 016f5c896..fc77f6ec6 100644 --- a/Duplicati/Library/Backend/S3/Strings.cs +++ b/Duplicati/Library/Backend/S3/Strings.cs @@ -47,6 +47,8 @@ namespace Duplicati.Library.Backend.Strings public static string DescriptionUseSSLShort { get { return LC.L(@"Instruct Duplicati to use an SSL (https) connection"); } } public static string DescriptionDisableChunkEncodingLong { get { return LC.L(@"This disables chunk encoding for the aws client, which is not supported by all S3 providers."); } } public static string DescriptionDisableChunkEncodingShort { get { return LC.L(@"Disable chunk encoding (aws client only)"); } } + public static string DescriptionDisablePayloadSigningLong { get { return LC.L(@"This disables payload signing for the aws client, which is not supported by all S3 providers."); } } + public static string DescriptionDisablePayloadSigningShort { get { return LC.L(@"Disable payload signing (aws client only)"); } } public static string S3StorageclassDescriptionLong { get { return LC.L(@"Use this option to specify a storage class. If this option is not used, the server will choose a default storage class."); } } public static string S3StorageclassDescriptionShort { get { return LC.L(@"Specify storage class"); } } public static string S3ArchiveClassesDescriptionShort { get { return LC.L(@"Specify archive storage class"); } } diff --git a/Duplicati/Library/Backends/BackendModules.cs b/Duplicati/Library/Backends/BackendModules.cs index dd66ce6c6..05ef8db4f 100644 --- a/Duplicati/Library/Backends/BackendModules.cs +++ b/Duplicati/Library/Backends/BackendModules.cs @@ -42,7 +42,6 @@ public static class BackendModules new Backend.AzureBlob.AzureBlobBackend(), new Backend.Backblaze.B2(), new Backend.Box.BoxBackend(), - new Backend.CloudFiles(), new Backend.Dropbox(), new Backend.FTP(), new Backend.AlternateFTPBackend(), diff --git a/Duplicati/Library/Logging/ILogDestination.cs b/Duplicati/Library/Logging/ILogDestination.cs index 5eca15ab3..c92302cd5 100644 --- a/Duplicati/Library/Logging/ILogDestination.cs +++ b/Duplicati/Library/Logging/ILogDestination.cs @@ -18,22 +18,18 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. - -using System; -using System.Collections.Generic; -using System.Text; - -namespace Duplicati.Library.Logging -{ - /// - /// Interface for a log destination - /// - public interface ILogDestination - { - /// - /// The function called when a message is logged - /// - /// The message to log - void WriteMessage(LogEntry entry); - } -} + +namespace Duplicati.Library.Logging +{ + /// + /// Interface for a log destination + /// + public interface ILogDestination + { + /// + /// The function called when a message is logged + /// + /// The message to log + void WriteMessage(LogEntry entry); + } +} diff --git a/Duplicati/Library/Logging/LogEntry.cs b/Duplicati/Library/Logging/LogEntry.cs index e12441052..2b49e2b86 100644 --- a/Duplicati/Library/Logging/LogEntry.cs +++ b/Duplicati/Library/Logging/LogEntry.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs index 0466b2fc5..6e291fc4e 100644 --- a/Duplicati/Library/Main/Operation/DeleteHandler.cs +++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs @@ -89,50 +89,56 @@ namespace Duplicati.Library.Main.Operation versionsToDelete = versionsToDelete.OrderBy(x => x.Version).Skip(1).ToList(); } - if (versionsToDelete.Count > 0) - Logging.Log.WriteInformationMessage(LOGTAG, "DeleteRemoteFileset", "Deleting {0} remote fileset(s) ...", versionsToDelete.Count); - - var lst = db.DropFilesetsFromTable(versionsToDelete.Select(x => x.Time).ToArray(), rtr.Transaction).ToArray(); - foreach (var f in lst) - db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, rtr.Transaction); - - if (!m_options.Dryrun) - rtr.Commit("CommitBeforeDelete"); - - foreach (var f in lst) + if (versionsToDelete.Count == 0) { - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - { - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - return; - } - - if (!m_options.Dryrun) - await backendManager.DeleteAsync(f.Key, f.Value, false, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - else - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFileset", "Would delete remote fileset: {0}", f.Key); - } - - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - - var count = lst.Length; - if (!m_options.Dryrun) - { - if (count == 0) - Logging.Log.WriteInformationMessage(LOGTAG, "DeleteResults", "No remote filesets were deleted"); - else - Logging.Log.WriteInformationMessage(LOGTAG, "DeleteResults", "Deleted {0} remote fileset(s)", count); + Logging.Log.WriteInformationMessage(LOGTAG, "NoFilesetsToDelete", "No remote filesets should be deleted"); } else { + Logging.Log.WriteInformationMessage(LOGTAG, "DeleteRemoteFileset", "Deleting {0} remote fileset(s) ...", versionsToDelete.Count); - if (count == 0) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteResults", "No remote filesets would be deleted"); + var lst = db.DropFilesetsFromTable(versionsToDelete.Select(x => x.Time).ToArray(), rtr.Transaction).ToArray(); + foreach (var f in lst) + db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, rtr.Transaction); + + if (!m_options.Dryrun) + rtr.Commit("CommitBeforeDelete"); + + foreach (var f in lst) + { + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + { + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + return; + } + + if (!m_options.Dryrun) + await backendManager.DeleteAsync(f.Key, f.Value, false, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + else + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFileset", "Would delete remote fileset: {0}", f.Key); + } + + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + + var count = lst.Length; + if (!m_options.Dryrun) + { + if (count == 0) + Logging.Log.WriteInformationMessage(LOGTAG, "DeleteResults", "No remote filesets were deleted"); + else + Logging.Log.WriteInformationMessage(LOGTAG, "DeleteResults", "Deleted {0} remote fileset(s)", count); + } else - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteResults", "{0} remote fileset(s) would be deleted", count); + { - if (count > 0 && m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteHelp", "Remove --dry-run to actually delete files"); + if (count == 0) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteResults", "No remote filesets would be deleted"); + else + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteResults", "{0} remote fileset(s) would be deleted", count); + + if (count > 0 && m_options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteHelp", "Remove --dry-run to actually delete files"); + } } if (!m_options.NoAutoCompact && (forceCompact || versionsToDelete.Count > 0)) diff --git a/Duplicati/Library/RestAPI/Abstractions/IApplicationSettings.cs b/Duplicati/Library/RestAPI/Abstractions/IApplicationSettings.cs new file mode 100644 index 000000000..67282546b --- /dev/null +++ b/Duplicati/Library/RestAPI/Abstractions/IApplicationSettings.cs @@ -0,0 +1,64 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +#nullable enable + +using System; +using System.Threading; +using Duplicati.Library.Interface; + +namespace Duplicati.WebserverCore.Abstractions; + +/// +/// Interface for application-wide settings for the server. +/// +public interface IApplicationSettings +{ + /// + /// Action to start or stop the usage reporter + /// + Action? StartOrStopUsageReporter { get; set; } + + /// + /// Gets the folder where Duplicati data is stored + /// + string DataFolder { get; } + + /// + /// Used to check the origin of the web server (e.g. Tray icon or a stand alone Server) + /// + string Origin { get; set; } + + /// + /// The application exit event + /// + ManualResetEvent ApplicationExitEvent { get; } + + /// + /// The shared secret provider from the server invocation + /// + ISecretProvider? SecretProvider { get; set; } + + /// + /// Flag to indicate if the settings encryption key was provided externally + /// + bool SettingsEncryptionKeyProvidedExternally { get; set; } +} + diff --git a/Duplicati/Library/RestAPI/Abstractions/ILogWriteHandler.cs b/Duplicati/Library/RestAPI/Abstractions/ILogWriteHandler.cs new file mode 100644 index 000000000..d041ad7b9 --- /dev/null +++ b/Duplicati/Library/RestAPI/Abstractions/ILogWriteHandler.cs @@ -0,0 +1,143 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using Duplicati.Library.Interface; +using Duplicati.Library.Logging; + +namespace Duplicati.WebserverCore.Abstractions; + +/// +/// Handles logging from the server, +/// and provides an entry point for the runner +/// to redirect log output to a file +/// +public interface ILogWriteHandler : ILogDestination, IDisposable +{ + /// + /// Represents a single log event + /// + public struct LiveLogEntry + { + /// + /// The context key used for conveying the backup ID + /// + public const string LOG_EXTRA_BACKUPID = "BackupID"; + /// + /// The context key used for conveying the task ID + /// + public const string LOG_EXTRA_TASKID = "TaskID"; + + /// + /// A unique ID that sequentially increments + /// + private static long _id; + + /// + /// The time the message was logged + /// + public readonly DateTime When; + + /// + /// The ID assigned to the message + /// + public readonly long ID; + + /// + /// The logged message + /// + public readonly string Message; + + /// + /// The log tag + /// + public readonly string Tag; + + /// + /// The message ID + /// + public readonly string MessageID; + + /// + /// The message ID + /// + public readonly string? ExceptionID; + + /// + /// The message type + /// + public readonly LogMessageType Type; + + /// + /// Exception data attached to the message + /// + public readonly Exception? Exception; + + /// + /// The backup ID, if any + /// + public readonly string? BackupID; + + /// + /// The task ID, if any + /// + public readonly string TaskID; + + /// + /// Initializes a new instance of the struct. + /// + /// The log entry to store + public LiveLogEntry(LogEntry entry) + { + this.ID = System.Threading.Interlocked.Increment(ref _id); + this.When = entry.When; + this.Message = entry.FormattedMessage; + this.Type = entry.Level; + this.Exception = entry.Exception; + this.Tag = entry.FilterTag; + this.MessageID = entry.Id; + this.BackupID = entry[LOG_EXTRA_BACKUPID]; + this.TaskID = entry[LOG_EXTRA_TASKID]; + + if (entry.Exception == null) + this.ExceptionID = null; + else if (entry.Exception is UserInformationException exception) + this.ExceptionID = exception.HelpID; + else + this.ExceptionID = entry.Exception.GetType().FullName; + + } + } + + + void RenewTimeout(LogMessageType type); + + void SetServerFile(string path, LogMessageType level); + + void AppendLogDestination(ILogDestination destination, LogMessageType level); + + LiveLogEntry[] AfterTime(DateTime offset, LogMessageType level); + + LiveLogEntry[] AfterID(long id, LogMessageType level, int pagesize); +} + diff --git a/Duplicati/Library/RestAPI/Abstractions/IWorkerThreadsManager.cs b/Duplicati/Library/RestAPI/Abstractions/IProgressStateProviderService.cs similarity index 73% rename from Duplicati/Library/RestAPI/Abstractions/IWorkerThreadsManager.cs rename to Duplicati/Library/RestAPI/Abstractions/IProgressStateProviderService.cs index aceea26b9..335bb5247 100644 --- a/Duplicati/Library/RestAPI/Abstractions/IWorkerThreadsManager.cs +++ b/Duplicati/Library/RestAPI/Abstractions/IProgressStateProviderService.cs @@ -18,20 +18,20 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. + #nullable enable + using System; -using Duplicati.Library.Utility; -using Duplicati.Server; -namespace Duplicati.Library.RestAPI.Abstractions; +namespace Duplicati.WebserverCore.Abstractions; -public interface IWorkerThreadsManager +/// +/// Produces system information. +/// +public interface IProgressStateProviderService { - void Spawn(Action item); - - Tuple? CurrentTask { get; } - WorkerThread? WorkerThread { get; } - void UpdateThrottleSpeeds(string? uploadSpeed, string? downloadSpeed); - - long AddTask(Runner.IRunnerData data, bool skipQueue = false); -} \ No newline at end of file + /// + /// Handler for progress state generation. + /// + Func? GenerateProgressState { get; set; } +} diff --git a/Duplicati/Library/RestAPI/Abstractions/IQueueRunnerService.cs b/Duplicati/Library/RestAPI/Abstractions/IQueueRunnerService.cs new file mode 100644 index 000000000..1ee4933de --- /dev/null +++ b/Duplicati/Library/RestAPI/Abstractions/IQueueRunnerService.cs @@ -0,0 +1,107 @@ + +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +#nullable enable + +using System; +using System.Collections.Generic; +using Duplicati.Library.Interface; +using Duplicati.Server.Serialization.Interface; + +namespace Duplicati.WebserverCore.Abstractions; + +/// +/// A cached task result +/// +/// The task ID +/// The backup ID +/// The time the task started +/// The time the task finished +/// The exception that was thrown +public sealed record CachedTaskResult(long TaskID, string? BackupId, DateTime? TaskStarted, DateTime? TaskFinished, Exception? Exception); + +/// +/// Class to encapsulate a thread that runs a list of queued operations +/// +/// The type to operate on +public interface IQueueRunnerService +{ + /// + /// Returns a copy of the current tasks in the queue + /// + /// A list of queued tasks + List GetCurrentTasks(); + /// + /// Gets a flag indicating if the queue is currently executing a task + /// + /// True if the queue is executing a task, false otherwise + bool GetIsActive(); + /// + /// Returns the currently executing task in the queue + /// + /// The currently executing task, or null if no task is executing + IQueuedTask? GetCurrentTask(); + + /// + /// Gets the cached task results for a given task ID + /// + /// The task ID + /// The cached task result + CachedTaskResult? GetCachedTaskResults(long taskID); + + /// + /// Adds a task to the queue + /// + /// The task to add + long AddTask(IQueuedTask task); + /// + /// Adds a task to the queue, optionally skipping the queue + /// + /// The task to add + /// Whether to skip the queue + long AddTask(IQueuedTask task, bool skipQueue); + /// + /// Removes a task from the queue + /// + /// Whether to wait for the task to finish + void Terminate(bool wait); + /// + /// Resumes processing items in the queue + /// + void Resume(); + /// + /// Pauses processing items in the queue + /// + void Pause(); + + /// + /// Gets the IDs of the tasks in the worker queue + /// + /// A list of tuples containing the task ID and backup ID + IList> GetQueueWithIds(); + + /// + /// Runs a task immediately, bypassing the queue. + /// Note that the task will run concurrently with the queue tasks and may cause database lock issues. + /// + /// The task to run + IBasicResults? RunImmediately(IQueuedTask task); +} \ No newline at end of file diff --git a/Duplicati/Library/RestAPI/Abstractions/IScheduler.cs b/Duplicati/Library/RestAPI/Abstractions/ISchedulerService.cs similarity index 72% rename from Duplicati/Library/RestAPI/Abstractions/IScheduler.cs rename to Duplicati/Library/RestAPI/Abstractions/ISchedulerService.cs index 1140bf84a..398543cdb 100644 --- a/Duplicati/Library/RestAPI/Abstractions/IScheduler.cs +++ b/Duplicati/Library/RestAPI/Abstractions/ISchedulerService.cs @@ -20,25 +20,12 @@ // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; -using Duplicati.Library.Utility; -using Duplicati.Server; using Duplicati.Server.Serialization.Interface; namespace Duplicati.WebserverCore.Abstractions; -public interface IScheduler +public interface ISchedulerService { - /// - /// Initializes scheduler - /// - /// The worker thread - void Init(WorkerThread worker); - - /// - /// Gets the current ids in the scheduler queue - /// - IList> GetSchedulerQueueIds(); - /// /// Gets the current proposed schedule /// @@ -50,21 +37,11 @@ public interface IScheduler /// True if the call should block until the thread has exited, false otherwise void Terminate(bool wait); - /// - /// Subscribes to the event that is triggered when the schedule changes - /// - void SubScribeToNewSchedule(Action handler); - /// /// A snapshot copy of the current schedule list /// List> Schedule { get; } - /// - /// A snapshot copy of the current worker queue, that is items that are scheduled, but waiting for execution - /// - List WorkerQueue { get; } - /// /// Forces the scheduler to re-evaluate the order. /// Call this method if something changes diff --git a/Duplicati/Library/RestAPI/Database/Connection.cs b/Duplicati/Library/RestAPI/Database/Connection.cs index 90bb7bbda..421031e14 100644 --- a/Duplicati/Library/RestAPI/Database/Connection.cs +++ b/Duplicati/Library/RestAPI/Database/Connection.cs @@ -32,6 +32,7 @@ using Duplicati.Library.AutoUpdater; using System.Data; using Duplicati.Library.Main.Database; using System.Globalization; +using Microsoft.Extensions.DependencyInjection; #nullable enable @@ -39,6 +40,11 @@ namespace Duplicati.Server.Database { public class Connection : IDisposable { + /// + /// The placeholder for passwords in the UI + /// + public const string PASSWORD_PLACEHOLDER = "**********"; + private readonly IDbConnection m_connection; private readonly IDbCommand m_errorcmd; public readonly object m_lock = new object(); @@ -47,6 +53,11 @@ namespace Duplicati.Server.Database private readonly Dictionary m_temporaryBackups = new Dictionary(); private readonly bool m_encryptSensitiveFields; private readonly EncryptedFieldHelper.KeyInstance? m_key; + private IServiceProvider? m_serviceProvider; + private INotificationUpdateService? m_notificationUpdateService; + private EventPollNotify? m_eventPollNotifyer; + private readonly string m_dataFolder; + private static readonly HashSet _encryptedFields = BackendLoader.Backends.SelectMany(x => x.SupportedCommands ?? []) .Concat(EncryptionLoader.Modules.SelectMany(x => x.SupportedCommands ?? [])) @@ -66,14 +77,31 @@ namespace Duplicati.Server.Database ]) .ToHashSet(StringComparer.OrdinalIgnoreCase); - public Connection(IDbConnection connection, bool disableFieldEncryption, EncryptedFieldHelper.KeyInstance? key) + public Connection(IDbConnection connection, bool disableFieldEncryption, EncryptedFieldHelper.KeyInstance? key, string dataFolder, Action startOrStopUsageReporter) { + m_dataFolder = dataFolder; m_encryptSensitiveFields = !disableFieldEncryption; m_key = key; m_connection = connection; m_errorcmd = m_connection.CreateCommand(@"INSERT INTO ""ErrorLog"" (""BackupID"", ""Message"", ""Exception"", ""Timestamp"") VALUES (@BackupId,@Message,@Exception,@Timestamp)"); - this.ApplicationSettings = new ServerSettings(this); + this.ApplicationSettings = new ServerSettings(this, startOrStopUsageReporter); + } + + /// + /// The service provider is used to resolve dependencies + /// + internal IServiceProvider? ServiceProvider => m_serviceProvider; + + /// + /// Set the service provider to be used for resolving dependencies + /// + /// The service provider + public void SetServiceProvider(IServiceProvider sp) + { + m_serviceProvider = sp; + m_notificationUpdateService = sp?.GetRequiredService(); + m_eventPollNotifyer = sp?.GetRequiredService(); } public bool IsEncryptingFields => m_encryptSensitiveFields; @@ -125,7 +153,7 @@ namespace Duplicati.Server.Database this.ApplicationSettings.PreloadSettingsHash = settingsHash; } - public void LogError(string backupid, string message, Exception ex) + public void LogError(string? backupid, string message, Exception ex) { lock (m_lock) { @@ -310,7 +338,7 @@ namespace Duplicati.Server.Database cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Option"" (""BackupID"", ""Filter"", ""Name"", ""Value"") VALUES (@BackupId, @Filter, @Name, @Value)"), (cmd, f) => { - if (FIXMEGlobal.PASSWORD_PLACEHOLDER.Equals(f.Value)) + if (PASSWORD_PLACEHOLDER.Equals(f.Value)) throw new Exception("Attempted to save a property with the placeholder password"); cmd.SetParameterValue("@BackupId", id) @@ -608,8 +636,8 @@ namespace Duplicati.Server.Database } } - FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); + m_notificationUpdateService?.IncrementLastDataUpdateId(); + m_eventPollNotifyer?.SignalNewEvent(); } private void AddOrUpdateBackup(IBackup item, bool updateSchedule, ISchedule? schedule) @@ -619,7 +647,7 @@ namespace Duplicati.Server.Database bool update = item.ID != null; if (!update && item.DBPath == null) { - var folder = FIXMEGlobal.DataFolder; + var folder = m_dataFolder; if (!System.IO.Directory.Exists(folder)) System.IO.Directory.CreateDirectory(folder); @@ -652,7 +680,7 @@ namespace Duplicati.Server.Database }, (cmd, n) => { - if (n.TargetURL.IndexOf(FIXMEGlobal.PASSWORD_PLACEHOLDER, StringComparison.Ordinal) >= 0) + if (n.TargetURL.IndexOf(PASSWORD_PLACEHOLDER, StringComparison.Ordinal) >= 0) throw new Exception("Attempted to save a backup with the password placeholder"); if (update && long.Parse(n.ID) <= 0) throw new Exception("Invalid update, cannot update application settings through update method"); @@ -722,8 +750,8 @@ namespace Duplicati.Server.Database } tr.Commit(); - FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); + m_notificationUpdateService?.IncrementLastDataUpdateId(); + m_eventPollNotifyer?.SignalNewEvent(); } } } @@ -735,8 +763,8 @@ namespace Duplicati.Server.Database { AddOrUpdateSchedule(item, tr); tr.Commit(); - FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); + m_notificationUpdateService?.IncrementLastDataUpdateId(); + m_eventPollNotifyer?.SignalNewEvent(); } } @@ -800,8 +828,8 @@ namespace Duplicati.Server.Database } } - FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); + m_notificationUpdateService?.IncrementLastDataUpdateId(); + m_eventPollNotifyer?.SignalNewEvent(); } public void DeleteBackup(IBackup backup) @@ -820,8 +848,8 @@ namespace Duplicati.Server.Database lock (m_lock) DeleteFromDb("Schedule", ID); - FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); + m_notificationUpdateService?.IncrementLastDataUpdateId(); + m_eventPollNotifyer?.SignalNewEvent(); } public void DeleteSchedule(ISchedule schedule) @@ -906,21 +934,27 @@ namespace Duplicati.Server.Database return false; DeleteFromDb(typeof(Notification).Name, id); - FIXMEGlobal.DataConnection.ApplicationSettings.UnackedError = notifications.Any(x => x.ID != id && x.Type == Duplicati.Server.Serialization.NotificationType.Error); - FIXMEGlobal.DataConnection.ApplicationSettings.UnackedWarning = notifications.Any(x => x.ID != id && x.Type == Duplicati.Server.Serialization.NotificationType.Warning); + this.ApplicationSettings.UnackedError = notifications.Any(x => x.ID != id && x.Type == Duplicati.Server.Serialization.NotificationType.Error); + this.ApplicationSettings.UnackedWarning = notifications.Any(x => x.ID != id && x.Type == Duplicati.Server.Serialization.NotificationType.Warning); } - // Guard against dismissing notifications before the provider is initialized - if (FIXMEGlobal.Provider != null) - { - FIXMEGlobal.NotificationUpdateService.IncrementLastNotificationUpdateId(); - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); - } + m_notificationUpdateService?.IncrementLastNotificationUpdateId(); + m_eventPollNotifyer?.SignalNewEvent(); return true; } - public void RegisterNotification(Serialization.NotificationType type, string title, string message, Exception ex, string backupid, string action, string logid, string messageid, string logtag, Func conflicthandler) + public void RegisterNotification( + Serialization.NotificationType type, + string title, + string message, + Exception? ex, + string? backupid, + string action, + string? logid, + string? messageid, + string? logtag, + Func conflicthandler) { lock (m_lock) { @@ -949,13 +983,13 @@ namespace Duplicati.Server.Database OverwriteAndUpdateDb(null, null, [notification], false); if (type == Serialization.NotificationType.Error) - FIXMEGlobal.DataConnection.ApplicationSettings.UnackedError = true; + ApplicationSettings.UnackedError = true; else if (type == Serialization.NotificationType.Warning) - FIXMEGlobal.DataConnection.ApplicationSettings.UnackedWarning = true; + ApplicationSettings.UnackedWarning = true; } - FIXMEGlobal.NotificationUpdateService.IncrementLastNotificationUpdateId(); - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); + m_notificationUpdateService?.IncrementLastNotificationUpdateId(); + m_eventPollNotifyer?.SignalNewEvent(); } //Workaround to clean up the database after invalid settings update diff --git a/Duplicati/Library/RestAPI/Database/ServerSettings.cs b/Duplicati/Library/RestAPI/Database/ServerSettings.cs index dea19e8ec..94f28764c 100644 --- a/Duplicati/Library/RestAPI/Database/ServerSettings.cs +++ b/Duplicati/Library/RestAPI/Database/ServerSettings.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -29,6 +29,8 @@ using System.Text; using System.Security.Cryptography.X509Certificates; using Duplicati.Library.Utility; using Duplicati.Library.AutoUpdater; +using Microsoft.Extensions.DependencyInjection; +using Duplicati.WebserverCore.Abstractions; #nullable enable @@ -78,10 +80,12 @@ namespace Duplicati.Server.Database private readonly Dictionary settings; private readonly Connection databaseConnection; - private UpdateInfo? m_latestUpdate; + private UpdateInfo? latestUpdate; + private readonly Action? startOrStopUsageReporter; - internal ServerSettings(Connection con) + internal ServerSettings(Connection con, Action startOrStopUsageReporter) { + this.startOrStopUsageReporter = startOrStopUsageReporter; settings = new Dictionary(); databaseConnection = con; ReloadSettings(); @@ -113,7 +117,7 @@ namespace Duplicati.Server.Database lock (databaseConnection.m_lock) { - m_latestUpdate = null; + latestUpdate = null; if (clearExisting) settings.Clear(); @@ -142,17 +146,17 @@ namespace Duplicati.Server.Database Value = n.Value }, Database.Connection.SERVER_SETTINGS_ID); - if (FIXMEGlobal.IsServerStarted) + var provider = databaseConnection.ServiceProvider; + if (provider != null) { - FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); + provider?.GetRequiredService()?.IncrementLastDataUpdateId(); + provider?.GetRequiredService()?.SignalNewEvent(); // If throttle options were changed, update now - FIXMEGlobal.WorkerThreadsManager.UpdateThrottleSpeeds(UploadSpeedLimit, DownloadSpeedLimit); + provider?.GetRequiredService()?.GetCurrentTask()?.UpdateThrottleSpeeds(UploadSpeedLimit, DownloadSpeedLimit); } // In case the usage reporter is enabled or disabled, refresh now - if (FIXMEGlobal.StartOrStopUsageReporter != null) - FIXMEGlobal.StartOrStopUsageReporter(); + startOrStopUsageReporter?.Invoke(); } public string? StartupDelayDuration @@ -602,7 +606,7 @@ namespace Duplicati.Server.Database lock (databaseConnection.m_lock) settings[CONST.UPDATE_CHECK_INTERVAL] = value; SaveSettings(); - FIXMEGlobal.UpdatePoller.Reschedule(); + databaseConnection?.ServiceProvider?.GetRequiredService()?.Reschedule(); } } @@ -631,11 +635,11 @@ namespace Duplicati.Server.Database try { - if (m_latestUpdate != null) - return m_latestUpdate; + if (latestUpdate != null) + return latestUpdate; using (var tr = new System.IO.StringReader(updateNew)) - return m_latestUpdate = Server.Serialization.Serializer.Deserialize(tr); + return latestUpdate = Server.Serialization.Serializer.Deserialize(tr); } catch { @@ -655,7 +659,7 @@ namespace Duplicati.Server.Database result = sb.ToString(); } - m_latestUpdate = value; + latestUpdate = value; lock (databaseConnection.m_lock) settings[CONST.UPDATE_CHECK_NEW_VERSION] = result; diff --git a/Duplicati/Library/RestAPI/FIXMEGlobal.cs b/Duplicati/Library/RestAPI/FIXMEGlobal.cs deleted file mode 100644 index 8b6253cda..000000000 --- a/Duplicati/Library/RestAPI/FIXMEGlobal.cs +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -using Duplicati.Server; -using System; -using System.Collections.Generic; -using Duplicati.Library.RestAPI.Abstractions; -using Duplicati.Library.Utility; -using Duplicati.WebserverCore.Abstractions; -using Microsoft.Extensions.DependencyInjection; -using Duplicati.Library.Interface; - -namespace Duplicati.Library.RestAPI -{ - /** - * In the absense of dependancy injection, there is a significant amount of variables exposed through Program as globals. - * This causes a problem decoupling classes and leads to circular dependancies. - */ - public static class FIXMEGlobal - { - /// - /// The placeholder for passwords in the UI - /// - public const string PASSWORD_PLACEHOLDER = "**********"; - - public static IServiceProvider Provider { get; set; } - - /// - /// This is the only access to the database - /// - public static Server.Database.Connection DataConnection; - - /// - /// A delegate method for creating a copy of the current progress state - /// - public static Func GenerateProgressState; - - /// - /// The status event signaler, used to control long polling of status updates - /// - public static EventPollNotify StatusEventNotifyer => Provider.GetRequiredService(); - - /// - /// For keeping and incrementing last last events Ids of db save and last notification - /// - public static INotificationUpdateService NotificationUpdateService => Provider.GetRequiredService(); - /// - /// Checks if the server has started and is listening for events - /// - public static bool IsServerStarted => Provider != null; - - /// - /// This is the working thread - /// - public static WorkerThread WorkThread => - Provider.GetRequiredService().WorkerThread; - - public static IWorkerThreadsManager WorkerThreadsManager => - Provider.GetRequiredService(); - - public static Action StartOrStopUsageReporter; - - /// - /// Gets the folder where Duplicati data is stored - /// - public static string DataFolder; - - /// - /// This is the scheduling thread - /// - public static IScheduler Scheduler => Provider.GetRequiredService(); - - /// - /// The log redirect handler - /// - public static readonly LogWriteHandler LogHandler = new LogWriteHandler(); - - /// - /// The update poll thread. - /// - public static UpdatePollThread UpdatePoller => Provider.GetRequiredService(); - - - /// - /// Used to check the origin of the web server (e.g. Tray icon or a stand alone Server) - /// - public static string Origin = "Server"; - - - /// - /// The application exit event - /// - public static System.Threading.ManualResetEvent ApplicationExitEvent; - - - /// - /// List of completed task results - /// - public static readonly List> TaskResultCache = new List>(); - - - /// - /// This is the lock to be used before manipulating the shared resources - /// - public static readonly object MainLock = new object(); - - /// - /// The shared secret provider from the server invocation - /// - public static ISecretProvider SecretProvider { get; set; } - /// - /// Flag to indicate if the settings encryption key was provided externally - /// - public static bool SettingsEncryptionKeyProvidedExternally { get; set; } - - } -} diff --git a/Duplicati/Library/RestAPI/LogWriteHandler.cs b/Duplicati/Library/RestAPI/LogWriteHandler.cs index 72aa92f60..6b0613d67 100644 --- a/Duplicati/Library/RestAPI/LogWriteHandler.cs +++ b/Duplicati/Library/RestAPI/LogWriteHandler.cs @@ -1,30 +1,30 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Linq; -using Duplicati.Library.Logging; using System.Collections.Generic; -using Duplicati.Library.Interface; using Duplicati.Library.Main; +using Duplicati.WebserverCore.Abstractions; +using Duplicati.Library.Logging; namespace Duplicati.Server { @@ -33,7 +33,7 @@ namespace Duplicati.Server /// and provides an entry point for the runner /// to redirect log output to a file /// - public class LogWriteHandler : ILogDestination, IDisposable + public class LogWriteHandler : ILogWriteHandler { /// /// The number of messages to keep when inactive @@ -44,101 +44,6 @@ namespace Duplicati.Server /// private const int ACTIVE_SIZE = 5000; - /// - /// The context key used for conveying the backup ID - /// - public const string LOG_EXTRA_BACKUPID = "BackupID"; - /// - /// The context key used for conveying the task ID - /// - public const string LOG_EXTRA_TASKID = "TaskID"; - - /// - /// Represents a single log event - /// - public struct LogEntry - { - /// - /// A unique ID that sequentially increments - /// - private static long _id; - - /// - /// The time the message was logged - /// - public readonly DateTime When; - - /// - /// The ID assigned to the message - /// - public readonly long ID; - - /// - /// The logged message - /// - public readonly string Message; - - /// - /// The log tag - /// - public readonly string Tag; - - /// - /// The message ID - /// - public readonly string MessageID; - - /// - /// The message ID - /// - public readonly string ExceptionID; - - /// - /// The message type - /// - public readonly LogMessageType Type; - - /// - /// Exception data attached to the message - /// - public readonly Exception Exception; - - /// - /// The backup ID, if any - /// - public readonly string BackupID; - - /// - /// The task ID, if any - /// - public readonly string TaskID; - - /// - /// Initializes a new instance of the struct. - /// - /// The log entry to store - public LogEntry(Duplicati.Library.Logging.LogEntry entry) - { - this.ID = System.Threading.Interlocked.Increment(ref _id); - this.When = entry.When; - this.Message = entry.FormattedMessage; - this.Type = entry.Level; - this.Exception = entry.Exception; - this.Tag = entry.FilterTag; - this.MessageID = entry.Id; - this.BackupID = entry[LOG_EXTRA_BACKUPID]; - this.TaskID = entry[LOG_EXTRA_TASKID]; - - if (entry.Exception == null) - this.ExceptionID = null; - else if (entry.Exception is UserInformationException exception) - this.ExceptionID = exception.HelpID; - else - this.ExceptionID = entry.Exception.GetType().FullName; - - } - } - /// /// Basic implementation of a ring-buffer /// @@ -208,7 +113,7 @@ namespace Duplicati.Server private readonly DateTime[] m_timeouts; private readonly object m_lock = new object(); private volatile bool m_anytimeouts = false; - private RingBuffer m_buffer; + private RingBuffer m_buffer; private readonly ControllerMultiLogTarget m_target = new ControllerMultiLogTarget(null, LogMessageType.Warning, null); @@ -218,7 +123,7 @@ namespace Duplicati.Server { var fields = Enum.GetValues(typeof(LogMessageType)); m_timeouts = new DateTime[fields.Length]; - m_buffer = new RingBuffer(INACTIVE_SIZE); + m_buffer = new RingBuffer(INACTIVE_SIZE); } public void RenewTimeout(LogMessageType type) @@ -228,7 +133,7 @@ namespace Duplicati.Server m_timeouts[(int)type] = DateTime.Now.AddSeconds(30); m_anytimeouts = true; if (m_buffer == null || m_buffer.Size == INACTIVE_SIZE) - m_buffer = new RingBuffer(ACTIVE_SIZE, m_buffer); + m_buffer = new RingBuffer(ACTIVE_SIZE, m_buffer); } } @@ -248,7 +153,7 @@ namespace Duplicati.Server UpdateLogLevel(); } - public LogEntry[] AfterTime(DateTime offset, LogMessageType level) + public ILogWriteHandler.LiveLogEntry[] AfterTime(DateTime offset, LogMessageType level) { RenewTimeout(level); UpdateLogLevel(); @@ -257,13 +162,13 @@ namespace Duplicati.Server lock (m_lock) { if (m_buffer == null) - return new LogEntry[0]; + return new ILogWriteHandler.LiveLogEntry[0]; return m_buffer.FlatArray((x) => x.When > offset && x.Type >= level); } } - public LogEntry[] AfterID(long id, LogMessageType level, int pagesize) + public ILogWriteHandler.LiveLogEntry[] AfterID(long id, LogMessageType level, int pagesize) { RenewTimeout(level); UpdateLogLevel(); @@ -271,7 +176,7 @@ namespace Duplicati.Server lock (m_lock) { if (m_buffer == null) - return new LogEntry[0]; + return []; var buffer = m_buffer.FlatArray((x) => x.ID > id && x.Type >= level); // Return the newest entries @@ -305,7 +210,7 @@ namespace Duplicati.Server #region ILog implementation - public void WriteMessage(Duplicati.Library.Logging.LogEntry entry) + public void WriteMessage(LogEntry entry) { if (entry.Level < m_logLevel) return; @@ -329,13 +234,13 @@ namespace Duplicati.Server UpdateLogLevel(); m_anytimeouts = false; if (m_buffer == null || m_buffer.Size != INACTIVE_SIZE) - m_buffer = new RingBuffer(INACTIVE_SIZE, m_buffer); + m_buffer = new RingBuffer(INACTIVE_SIZE, m_buffer); } } if (m_buffer != null) - m_buffer.Enqueue(new LogEntry(entry)); + m_buffer.Enqueue(new ILogWriteHandler.LiveLogEntry(entry)); } } diff --git a/Duplicati/Library/RestAPI/Runner.cs b/Duplicati/Library/RestAPI/Runner.cs index c55fe657d..bd9e0df2e 100644 --- a/Duplicati/Library/RestAPI/Runner.cs +++ b/Duplicati/Library/RestAPI/Runner.cs @@ -19,6 +19,8 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +#nullable enable + using System; using System.Linq; using System.Collections.Generic; @@ -26,46 +28,57 @@ using Duplicati.Library.Interface; using Duplicati.Server.Serialization; using Duplicati.Library.RestAPI; using Duplicati.Library.Utility; +using System.Threading.Tasks; +using Duplicati.Server.Database; +using Duplicati.Server.Serialization.Interface; +using Duplicati.WebserverCore.Abstractions; namespace Duplicati.Server { public static class Runner { - public interface IRunnerData : Duplicati.Server.Serialization.Interface.IQueuedTask + public interface IRunnerData : Serialization.Interface.IQueuedTask { - Duplicati.Server.Serialization.Interface.IBackup Backup { get; } - IDictionary ExtraOptions { get; } - string[] FilterStrings { get; } - string[] ExtraArguments { get; } + Serialization.Interface.IBackup? Backup { get; } + IDictionary? ExtraOptions { get; } + string[]? FilterStrings { get; } + string[]? ExtraArguments { get; } int PageSize { get; } int PageOffset { get; } - void Stop(); - void Abort(); - void Pause(bool alsoTransfers); - void Resume(); - void UpdateThrottleSpeed(string uploadSpeed, string downloadSpeed); - void SetController(Duplicati.Library.Main.Controller controller); + void SetController(Library.Main.Controller? controller); } private class RunnerData : IRunnerData { private static long RunnerTaskID = 1; - public Duplicati.Server.Serialization.DuplicatiOperation Operation { get; internal set; } - public Duplicati.Server.Serialization.Interface.IBackup Backup { get; internal set; } - public IDictionary ExtraOptions { get; internal set; } - public string[] FilterStrings { get; internal set; } + public Func? OnStarting { get; set; } + public Func? OnFinished { get; set; } - public string BackupID { get { return Backup.ID; } } + /// + /// Callback to be executed when the task is finished. + /// Only supported for the delete operation for now. + /// + internal Action? AfterTaskFinished { get; set; } + + public DuplicatiOperation Operation { get; internal set; } + public Serialization.Interface.IBackup? Backup { get; internal set; } + public IDictionary? ExtraOptions { get; internal set; } + public string[]? FilterStrings { get; internal set; } + + public string? BackupID { get { return Backup?.ID; } } public long TaskID { get { return m_taskID; } } - public string[] ExtraArguments { get; internal set; } + public string[]? ExtraArguments { get; internal set; } public int PageSize { get; internal set; } = 0; public int PageOffset { get; internal set; } = 0; - internal Duplicati.Library.Main.Controller Controller { get; set; } + public DateTime? TaskStarted { get; set; } + public DateTime? TaskFinished { get; set; } - public void SetController(Duplicati.Library.Main.Controller controller) + internal Library.Main.Controller? Controller { get; set; } + + public void SetController(Library.Main.Controller? controller) { Controller = controller; } @@ -93,7 +106,7 @@ namespace Duplicati.Server public long OriginalUploadSpeed { get; set; } public long OriginalDownloadSpeed { get; set; } - public void UpdateThrottleSpeed(string uploadSpeed, string downloadSpeed) + public void UpdateThrottleSpeeds(string? uploadSpeed, string? downloadSpeed) { var controller = this.Controller; if (controller == null) @@ -108,14 +121,14 @@ namespace Duplicati.Server try { if (!string.IsNullOrWhiteSpace(uploadSpeed)) - server_upload_throttle = Duplicati.Library.Utility.Sizeparser.ParseSize(uploadSpeed, "kb"); + server_upload_throttle = Sizeparser.ParseSize(uploadSpeed, "kb"); } catch { } try { if (!string.IsNullOrWhiteSpace(downloadSpeed)) - server_download_throttle = Duplicati.Library.Utility.Sizeparser.ParseSize(downloadSpeed, "kb"); + server_download_throttle = Sizeparser.ParseSize(downloadSpeed, "kb"); } catch { } @@ -159,13 +172,13 @@ namespace Duplicati.Server return new CustomRunnerTask(runner); } - public static IRunnerData CreateTask(Duplicati.Server.Serialization.DuplicatiOperation operation, Duplicati.Server.Serialization.Interface.IBackup backup, IDictionary extraOptions = null, string[] filterStrings = null, string[] extraArguments = null, int pageSize = 0, int pageOffset = 0) + public static IRunnerData CreateTask(DuplicatiOperation operation, IBackup backup, IDictionary? extraOptions = null, string[]? filterStrings = null, string[]? extraArguments = null, int pageSize = 0, int pageOffset = 0) { return new RunnerData() { Operation = operation, Backup = backup, - ExtraOptions = extraOptions, + ExtraOptions = extraOptions ?? new Dictionary(), FilterStrings = filterStrings, ExtraArguments = extraArguments, PageSize = pageSize, @@ -173,15 +186,26 @@ namespace Duplicati.Server }; } - public static IRunnerData CreateListTask(Duplicati.Server.Serialization.Interface.IBackup backup, string[] filters, bool onlyPrefix, bool allVersions, bool folderContents, DateTime time) + public static IRunnerData CreateDeleteTask(IBackup backup, IDictionary extraOptions, Action? afterTaskFinished = null) { - var dict = new Dictionary(); + return new RunnerData() + { + Operation = DuplicatiOperation.Delete, + Backup = backup, + ExtraOptions = extraOptions, + AfterTaskFinished = afterTaskFinished + }; + } + + public static IRunnerData CreateListTask(IBackup backup, string[]? filters, bool onlyPrefix, bool allVersions, bool folderContents, DateTime time) + { + var dict = new Dictionary(); if (onlyPrefix) dict["list-prefix-only"] = "true"; if (allVersions) dict["all-versions"] = "true"; if (time.Ticks > 0) - dict["time"] = Duplicati.Library.Utility.Utility.SerializeDateTime(time.ToUniversalTime()); + dict["time"] = Utility.SerializeDateTime(time.ToUniversalTime()); if (folderContents) dict["list-folder-contents"] = "true"; @@ -192,19 +216,19 @@ namespace Duplicati.Server filters); } - public static IRunnerData CreateListFilesetsTask(Duplicati.Server.Serialization.Interface.IBackup backup, Dictionary extraOptions = null) + public static IRunnerData CreateListFilesetsTask(IBackup backup, Dictionary? extraOptions = null) { return CreateTask( DuplicatiOperation.ListFilesets, backup, - extraOptions ?? new Dictionary()); + extraOptions ?? new Dictionary()); } - public static IRunnerData CreateListFolderContents(Duplicati.Server.Serialization.Interface.IBackup backup, string[] folders, DateTime time, int pageSize, int pageOffset) + public static IRunnerData CreateListFolderContents(IBackup backup, string[]? folders, DateTime time, int pageSize, int pageOffset) { - var dict = new Dictionary(); + var dict = new Dictionary(); if (time.Ticks > 0) - dict["time"] = Duplicati.Library.Utility.Utility.SerializeDateTime(time.ToUniversalTime()); + dict["time"] = Utility.SerializeDateTime(time.ToUniversalTime()); return CreateTask( DuplicatiOperation.ListFolderContents, @@ -215,9 +239,9 @@ namespace Duplicati.Server pageOffset: pageOffset); } - public static IRunnerData ListFileVersionsTask(Duplicati.Server.Serialization.Interface.IBackup backup, string[] filepaths, int pageSize, int pageOffset) + public static IRunnerData ListFileVersionsTask(IBackup backup, string[]? filepaths, int pageSize, int pageOffset) { - var dict = new Dictionary(); + var dict = new Dictionary(); return CreateTask( DuplicatiOperation.ListFileVersions, backup, @@ -227,11 +251,11 @@ namespace Duplicati.Server pageOffset: pageOffset); } - public static IRunnerData CreateSearchEntriesTask(Duplicati.Server.Serialization.Interface.IBackup backup, string[] filters, string[] folders, DateTime time, int pageSize, int pageOffset) + public static IRunnerData CreateSearchEntriesTask(IBackup backup, string[]? filters, string[]? folders, DateTime time, int pageSize, int pageOffset) { - var dict = new Dictionary(); + var dict = new Dictionary(); if (time.Ticks > 0) - dict["time"] = Duplicati.Library.Utility.Utility.SerializeDateTime(time.ToUniversalTime()); + dict["time"] = Utility.SerializeDateTime(time.ToUniversalTime()); return CreateTask( DuplicatiOperation.SearchEntries, @@ -244,17 +268,17 @@ namespace Duplicati.Server } - public static IRunnerData CreateRestoreTask(Duplicati.Server.Serialization.Interface.IBackup backup, string[] filters, - DateTime time, string restoreTarget, bool overwrite, bool restore_permissions, - bool skip_metadata, string passphrase) + public static IRunnerData CreateRestoreTask(IBackup backup, string[]? filters, + DateTime time, string? restoreTarget, bool overwrite, bool restore_permissions, + bool skip_metadata, string? passphrase) { - var dict = new Dictionary + var dict = new Dictionary { - ["time"] = Library.Utility.Utility.SerializeDateTime(time.ToUniversalTime()), - ["overwrite"] = overwrite ? Boolean.TrueString : Boolean.FalseString, - ["restore-permissions"] = restore_permissions ? Boolean.TrueString : Boolean.FalseString, - ["skip-metadata"] = skip_metadata ? Boolean.TrueString : Boolean.FalseString, - ["allow-passphrase-change"] = Boolean.TrueString + ["time"] = Utility.SerializeDateTime(time.ToUniversalTime()), + ["overwrite"] = overwrite ? bool.TrueString : bool.FalseString, + ["restore-permissions"] = restore_permissions ? bool.TrueString : bool.FalseString, + ["skip-metadata"] = skip_metadata ? bool.TrueString : bool.FalseString, + ["allow-passphrase-change"] = bool.TrueString }; if (!string.IsNullOrWhiteSpace(restoreTarget)) dict["restore-path"] = SpecialFolders.ExpandEnvironmentVariables(restoreTarget); @@ -267,26 +291,26 @@ namespace Duplicati.Server dict, filters); } - private class MessageSink : Duplicati.Library.Main.IMessageSink + private class MessageSink : Library.Main.IMessageSink { - private class ProgressState : Server.Serialization.Interface.IProgressEventData + private class ProgressState : IProgressEventData { - private readonly string m_backupID; + private readonly string? m_backupID; private readonly long m_taskID; - internal Duplicati.Library.Main.BackendActionType m_backendAction; - internal string m_backendPath; + internal Library.Main.BackendActionType m_backendAction; + internal string? m_backendPath; internal long m_backendFileSize; internal long m_backendFileProgress; internal long m_backendSpeed; internal bool m_backendIsBlocking; - internal string m_currentFilename; + internal string? m_currentFilename; internal long m_currentFilesize; internal long m_currentFileoffset; internal bool m_currentFilecomplete; - internal Duplicati.Library.Main.OperationPhase m_phase; + internal Library.Main.OperationPhase m_phase; internal float m_overallProgress; internal long m_processedFileCount; internal long m_processedFileSize; @@ -294,7 +318,7 @@ namespace Duplicati.Server internal long m_totalFileSize; internal bool m_stillCounting; - public ProgressState(long taskId, string backupId) + public ProgressState(long taskId, string? backupId) { m_backupID = backupId; m_taskID = taskId; @@ -306,15 +330,15 @@ namespace Duplicati.Server } #region IProgressEventData implementation - public string BackupID { get { return m_backupID; } } + public string? BackupID { get { return m_backupID; } } public long TaskID { get { return m_taskID; } } public string BackendAction { get { return m_backendAction.ToString(); } } - public string BackendPath { get { return m_backendPath; } } + public string? BackendPath { get { return m_backendPath; } } public long BackendFileSize { get { return m_backendFileSize; } } public long BackendFileProgress { get { return m_backendFileProgress; } } public long BackendSpeed { get { return m_backendSpeed; } } public bool BackendIsBlocking { get { return m_backendIsBlocking; } } - public string CurrentFilename { get { return m_currentFilename; } } + public string? CurrentFilename { get { return m_currentFilename; } } public long CurrentFilesize { get { return m_currentFilesize; } } public long CurrentFileoffset { get { return m_currentFileoffset; } } public bool CurrentFilecomplete { get { return m_currentFilecomplete; } } @@ -329,16 +353,16 @@ namespace Duplicati.Server } private readonly ProgressState m_state; - private Duplicati.Library.Main.IBackendProgress m_backendProgress; - private Duplicati.Library.Main.IOperationProgress m_operationProgress; + private Library.Main.IBackendProgress? m_backendProgress; + private Library.Main.IOperationProgress? m_operationProgress; private readonly object m_lock = new object(); - public MessageSink(long taskId, string backupId) + public MessageSink(long taskId, string? backupId) { m_state = new ProgressState(taskId, backupId); } - public Server.Serialization.Interface.IProgressEventData Copy() + public Serialization.Interface.IProgressEventData Copy() { lock (m_lock) { @@ -387,16 +411,18 @@ namespace Duplicati.Server #endregion } - public static string GetCommandLine(IRunnerData data) + public static string GetCommandLine(Connection databaseConnection, IRunnerData data) { var backup = data.Backup; + if (backup == null) + throw new ArgumentNullException(nameof(backup)); - var options = ApplyOptions(backup, GetCommonOptions()); + var options = ApplyOptions(databaseConnection, backup, GetCommonOptions(databaseConnection)); if (data.ExtraOptions != null) foreach (var k in data.ExtraOptions) options[k.Key] = k.Value; - var cf = FIXMEGlobal.DataConnection.Filters; + var cf = databaseConnection.Filters; var bf = backup.Filters; var sources = @@ -411,36 +437,38 @@ namespace Duplicati.Server ); var cmd = new System.Text.StringBuilder(); - cmd.Append(Library.Utility.Utility.WrapAsCommandLine(new string[] { exe, "backup", backup.TargetURL }, false)); + cmd.Append(Utility.WrapAsCommandLine([exe, "backup", backup.TargetURL], false)); cmd.Append(" "); - cmd.Append(Library.Utility.Utility.WrapAsCommandLine(sources, true)); + cmd.Append(Utility.WrapAsCommandLine(sources, true)); // TODO: We should check each option to see if it is a path, and allow expansion on that foreach (var opt in options) - cmd.AppendFormat(" --{0}={1}", opt.Key, Library.Utility.Utility.WrapCommandLineElement(opt.Value, false)); + cmd.AppendFormat(" --{0}={1}", opt.Key, Utility.WrapCommandLineElement(opt.Value, false)); if (cf != null) foreach (var f in cf) - cmd.AppendFormat(" --{0}={1}", f.Include ? "include" : "exclude", Library.Utility.Utility.WrapCommandLineElement(f.Expression, true)); + cmd.AppendFormat(" --{0}={1}", f.Include ? "include" : "exclude", Utility.WrapCommandLineElement(f.Expression, true)); if (bf != null) foreach (var f in bf) - cmd.AppendFormat(" --{0}={1}", f.Include ? "include" : "exclude", Library.Utility.Utility.WrapCommandLineElement(f.Expression, true)); + cmd.AppendFormat(" --{0}={1}", f.Include ? "include" : "exclude", Utility.WrapCommandLineElement(f.Expression, true)); return cmd.ToString(); } - public static string[] GetCommandLineParts(IRunnerData data) + public static string[] GetCommandLineParts(Connection databaseConnection, IRunnerData data) { var backup = data.Backup; + if (backup == null) + throw new ArgumentNullException(nameof(backup)); - var options = ApplyOptions(backup, GetCommonOptions()); + var options = ApplyOptions(databaseConnection, backup, GetCommonOptions(databaseConnection)); if (data.ExtraOptions != null) foreach (var k in data.ExtraOptions) options[k.Key] = k.Value; - var cf = FIXMEGlobal.DataConnection.Filters; + var cf = databaseConnection.Filters; var bf = backup.Filters; var sources = @@ -469,44 +497,56 @@ namespace Duplicati.Server return parts.ToArray(); } - public static Duplicati.Library.Interface.IBasicResults Run(IRunnerData data, bool fromQueue) + public static IBasicResults? Run(Connection databaseConnection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IProgressStateProviderService progressStateProviderService, IApplicationSettings applicationSettings, IQueuedTask data, bool fromQueue) { + if (data is IRunnerData runnerData) + return RunInternal(databaseConnection, eventPollNotify, notificationUpdateService, progressStateProviderService, applicationSettings, runnerData, fromQueue); + + throw new ArgumentException("Invalid task type", nameof(data)); + } + + private static IBasicResults? RunInternal(Connection databaseConnection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IProgressStateProviderService progressStateProviderService, IApplicationSettings applicationSettings, IRunnerData data, bool fromQueue) + { + data.TaskStarted = DateTime.Now; if (data is CustomRunnerTask task) { try { var sink = new MessageSink(task.TaskID, null); - FIXMEGlobal.GenerateProgressState = sink.Copy; - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); + progressStateProviderService.GenerateProgressState = sink.Copy; + eventPollNotify.SignalNewEvent(); task.Run(sink); } catch (Exception ex) { - FIXMEGlobal.DataConnection.LogError(string.Empty, "Failed while executing custom task", ex); + databaseConnection.LogError(string.Empty, "Failed while executing custom task", ex); + } + finally + { + data.TaskFinished = DateTime.Now; } return null; } var backup = data.Backup; - if (backup.Metadata == null) - { - backup.Metadata = new Dictionary(); - } + if (backup == null) + throw new ArgumentNullException(nameof(backup)); - Duplicati.Library.Utility.TempFolder tempfolder = null; + backup.Metadata ??= new Dictionary(); + TempFolder? tempfolder = null; try { var sink = new MessageSink(data.TaskID, backup.ID); if (fromQueue) { - FIXMEGlobal.GenerateProgressState = () => sink.Copy(); - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); + progressStateProviderService.GenerateProgressState = () => sink.Copy(); + eventPollNotify.SignalNewEvent(); } - var options = ApplyOptions(backup, GetCommonOptions()); + var options = ApplyOptions(databaseConnection, backup, GetCommonOptions(databaseConnection)); if (data.ExtraOptions != null) foreach (var k in data.ExtraOptions) options[k.Key] = k.Value; @@ -514,14 +554,14 @@ namespace Duplicati.Server // Pack in the system or task config for easy restore if (data.Operation == DuplicatiOperation.Backup && options.ContainsKey("store-task-config")) { - tempfolder = StoreTaskConfigAndGetTempFolder(data, options); + tempfolder = StoreTaskConfigAndGetTempFolder(databaseConnection, data, options); } // Attach a log scope that tags all messages to relay the TaskID and BackupID using (Library.Logging.Log.StartScope(log => { - log[LogWriteHandler.LOG_EXTRA_TASKID] = data.TaskID.ToString(); - log[LogWriteHandler.LOG_EXTRA_BACKUPID] = data.BackupID; + log[ILogWriteHandler.LiveLogEntry.LOG_EXTRA_TASKID] = data.TaskID.ToString(); + log[ILogWriteHandler.LiveLogEntry.LOG_EXTRA_BACKUPID] = data.BackupID; })) using (tempfolder) @@ -542,22 +582,23 @@ namespace Duplicati.Server catch { } ((RunnerData)data).Controller = controller; - data.UpdateThrottleSpeed(FIXMEGlobal.DataConnection.ApplicationSettings.UploadSpeedLimit, FIXMEGlobal.DataConnection.ApplicationSettings.DownloadSpeedLimit); + var appSettings = databaseConnection.ApplicationSettings; + data.UpdateThrottleSpeeds(appSettings.UploadSpeedLimit, appSettings.DownloadSpeedLimit); // Pass on the provider, will be replaced if configured in the backup - controller.SetSecretProvider(FIXMEGlobal.SecretProvider); + controller.SetSecretProvider(applicationSettings.SecretProvider); if (backup.Metadata.ContainsKey("LastCompactFinished")) - controller.LastCompact = Library.Utility.Utility.DeserializeDateTime(backup.Metadata["LastCompactFinished"]); + controller.LastCompact = Utility.DeserializeDateTime(backup.Metadata["LastCompactFinished"]); if (backup.Metadata.ContainsKey("LastVacuumFinished")) - controller.LastVacuum = Library.Utility.Utility.DeserializeDateTime(backup.Metadata["LastVacuumFinished"]); + controller.LastVacuum = Utility.DeserializeDateTime(backup.Metadata["LastVacuumFinished"]); switch (data.Operation) { case DuplicatiOperation.Backup: { - var filter = ApplyFilter(backup, GetCommonFilter()); + var filter = ApplyFilter(backup, GetCommonFilter(databaseConnection)); var sources = (from n in backup.Sources let p = SpecialFolders.ExpandEnvironmentVariables(n) @@ -565,62 +606,62 @@ namespace Duplicati.Server select p).ToArray(); var r = controller.Backup(sources, filter); - UpdateMetadata(backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r); return r; } case DuplicatiOperation.List: { var r = controller.List(data.FilterStrings, null); - UpdateMetadata(backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r); return r; } case DuplicatiOperation.Repair: { var r = controller.Repair(data.FilterStrings == null ? null : new Library.Utility.FilterExpression(data.FilterStrings)); - UpdateMetadata(backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r); return r; } case DuplicatiOperation.RepairUpdate: { var r = controller.UpdateDatabaseWithVersions(); - UpdateMetadata(backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r); return r; } case DuplicatiOperation.Remove: { var r = controller.Delete(); - UpdateMetadata(backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r); return r; } case DuplicatiOperation.Restore: { var r = controller.Restore(data.FilterStrings); - UpdateMetadata(backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r); return r; } case DuplicatiOperation.Verify: { var r = controller.Test(); - UpdateMetadata(backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r); return r; } case DuplicatiOperation.Compact: { var r = controller.Compact(); - UpdateMetadata(backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r); return r; } case DuplicatiOperation.CreateReport: { - using (var tf = new Duplicati.Library.Utility.TempFile()) + using (var tf = new Library.Utility.TempFile()) { var r = controller.CreateLogDatabase(tf); - var tempid = FIXMEGlobal.DataConnection.RegisterTempFile("create-bug-report", r.TargetPath, DateTime.Now.AddDays(3)); + var tempid = databaseConnection.RegisterTempFile("create-bug-report", r.TargetPath, DateTime.Now.AddDays(3)); - if (string.Equals(tf, r.TargetPath, Library.Utility.Utility.ClientFilenameStringComparison)) + if (string.Equals(tf, r.TargetPath, Utility.ClientFilenameStringComparison)) tf.Protected = true; - FIXMEGlobal.DataConnection.RegisterNotification( + databaseConnection.RegisterNotification( NotificationType.Information, "Bugreport ready", "Bugreport is ready for download", @@ -640,38 +681,40 @@ namespace Duplicati.Server case DuplicatiOperation.ListRemote: { var r = controller.ListRemote(); - UpdateMetadata(backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r); return r; } case DuplicatiOperation.Delete: { - if (Library.Utility.Utility.ParseBoolOption(data.ExtraOptions.AsReadOnly(), "delete-remote-files")) - controller.DeleteAllRemoteFiles(); - - if (Library.Utility.Utility.ParseBoolOption(data.ExtraOptions.AsReadOnly(), "delete-local-db")) + if (data.ExtraOptions != null) { - string dbpath; - options.TryGetValue("dbpath", out dbpath); + if (Utility.ParseBoolOption(data.ExtraOptions.AsReadOnly(), "delete-remote-files")) + controller.DeleteAllRemoteFiles(); - if (!string.IsNullOrWhiteSpace(dbpath) && System.IO.File.Exists(dbpath)) - System.IO.File.Delete(dbpath); + if (Utility.ParseBoolOption(data.ExtraOptions.AsReadOnly(), "delete-local-db")) + { + options.TryGetValue("dbpath", out var dbpath); + + if (!string.IsNullOrWhiteSpace(dbpath) && System.IO.File.Exists(dbpath)) + System.IO.File.Delete(dbpath); + } } - FIXMEGlobal.DataConnection.DeleteBackup(backup); - FIXMEGlobal.Scheduler.Reschedule(); + databaseConnection.DeleteBackup(backup); + (data as RunnerData)?.AfterTaskFinished?.Invoke(data); return null; } case DuplicatiOperation.Vacuum: { var r = controller.Vacuum(); - UpdateMetadata(backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r); return r; } case DuplicatiOperation.ListFilesets: { var r = controller.ListFilesets(); - UpdateMetadata(backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r); return r; } case DuplicatiOperation.ListFolderContents: @@ -696,8 +739,9 @@ namespace Duplicati.Server } catch (Exception ex) { - FIXMEGlobal.DataConnection.LogError(data.Backup.ID, string.Format("Failed while executing {0} \"{1}\" (id: {2})", data.Operation, data.Backup.Name, data.Backup.ID), ex); - UpdateMetadataError(data.Backup, ex); + databaseConnection.LogError(data.Backup?.ID, string.Format("Failed while executing {0} \"{1}\" (id: {2})", data.Operation, data.Backup?.Name, data.Backup?.ID), ex); + if (data.Backup != null) + UpdateMetadataError(databaseConnection, notificationUpdateService, data.Backup, ex); Library.UsageReporter.Reporter.Report(ex); if (!fromQueue) @@ -707,29 +751,33 @@ namespace Duplicati.Server } finally { - ((RunnerData)data).Controller = null; + data.SetController(null); + data.TaskFinished = DateTime.Now; } } - private static Duplicati.Library.Utility.TempFolder StoreTaskConfigAndGetTempFolder(IRunnerData data, Dictionary options) + private static TempFolder? StoreTaskConfigAndGetTempFolder(Connection databaseConnection, IRunnerData data, Dictionary options) { + if (data.Backup == null) + throw new ArgumentNullException(nameof(data.Backup)); + var all_tasks = string.Equals(options["store-task-config"], "all", StringComparison.OrdinalIgnoreCase) || string.Equals(options["store-task-config"], "*", StringComparison.OrdinalIgnoreCase); - var this_task = Duplicati.Library.Utility.Utility.ParseBool(options["store-task-config"], false); + var this_task = Utility.ParseBool(options["store-task-config"], false); options.Remove("store-task-config"); - Duplicati.Library.Utility.TempFolder tempfolder = null; + TempFolder? tempfolder = null; if (all_tasks || this_task) { - tempfolder = new Duplicati.Library.Utility.TempFolder(); + tempfolder = new TempFolder(); var temppath = System.IO.Path.Combine(tempfolder, "task-setup.json"); - using (var tempfile = Duplicati.Library.Utility.TempFile.WrapExistingFile(temppath)) + using (var tempfile = Library.Utility.TempFile.WrapExistingFile(temppath)) { - object taskdata = null; + object? taskdata = null; if (all_tasks) - taskdata = FIXMEGlobal.DataConnection.Backups.Where(x => !x.IsTemporary).Select(x => FIXMEGlobal.DataConnection.PrepareBackupForExport(FIXMEGlobal.DataConnection.GetBackup(x.ID))); + taskdata = databaseConnection.Backups.Where(x => !x.IsTemporary).Select(x => databaseConnection.PrepareBackupForExport(databaseConnection.GetBackup(x.ID)!)); else - taskdata = new[] { FIXMEGlobal.DataConnection.PrepareBackupForExport(data.Backup) }; + taskdata = new[] { databaseConnection.PrepareBackupForExport(data.Backup) }; using (var fs = System.IO.File.OpenWrite(tempfile)) using (var sw = new System.IO.StreamWriter(fs, System.Text.Encoding.UTF8)) @@ -737,7 +785,7 @@ namespace Duplicati.Server tempfile.Protected = true; - options.TryGetValue("control-files", out string controlfiles); + options.TryGetValue("control-files", out var controlfiles); if (string.IsNullOrWhiteSpace(controlfiles)) controlfiles = tempfile; @@ -750,20 +798,20 @@ namespace Duplicati.Server return tempfolder; } - private static void UpdateMetadataError(Duplicati.Server.Serialization.Interface.IBackup backup, Exception ex) + private static void UpdateMetadataError(Connection databaseConnection, INotificationUpdateService notificationUpdateService, IBackup backup, Exception ex) { - backup.Metadata["LastErrorDate"] = Library.Utility.Utility.SerializeDateTime(DateTime.UtcNow); + backup.Metadata["LastErrorDate"] = Utility.SerializeDateTime(DateTime.UtcNow); backup.Metadata["LastErrorMessage"] = ex.Message; if (!backup.IsTemporary) - FIXMEGlobal.DataConnection.SetMetadata(backup.Metadata, long.Parse(backup.ID), null); + databaseConnection.SetMetadata(backup.Metadata, long.Parse(backup.ID), null); - string messageid = null; + string? messageid = null; if (ex is UserInformationException exception) messageid = exception.HelpID; - FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); - FIXMEGlobal.DataConnection.RegisterNotification( + notificationUpdateService.IncrementLastDataUpdateId(); + databaseConnection.RegisterNotification( NotificationType.Error, backup.IsTemporary ? "Error" : string.Format("Error while running {0}", backup.Name), @@ -781,31 +829,31 @@ namespace Duplicati.Server ); } - private static void UpdateMetadataLastCompact(Duplicati.Server.Serialization.Interface.IBackup backup, Duplicati.Library.Interface.ICompactResults r) + private static void UpdateMetadataLastCompact(IBackup backup, ICompactResults r) { if (r != null) { backup.Metadata["LastCompactDuration"] = r.Duration.ToString(); - backup.Metadata["LastCompactStarted"] = Library.Utility.Utility.SerializeDateTime(r.BeginTime.ToUniversalTime()); - backup.Metadata["LastCompactFinished"] = Library.Utility.Utility.SerializeDateTime(r.EndTime.ToUniversalTime()); + backup.Metadata["LastCompactStarted"] = Utility.SerializeDateTime(r.BeginTime.ToUniversalTime()); + backup.Metadata["LastCompactFinished"] = Utility.SerializeDateTime(r.EndTime.ToUniversalTime()); } } - private static void UpdateMetadataLastVacuum(Duplicati.Server.Serialization.Interface.IBackup backup, Duplicati.Library.Interface.IVacuumResults r) + private static void UpdateMetadataLastVacuum(IBackup backup, IVacuumResults r) { if (r != null) { backup.Metadata["LastVacuumDuration"] = r.Duration.ToString(); - backup.Metadata["LastVacuumStarted"] = Library.Utility.Utility.SerializeDateTime(r.BeginTime.ToUniversalTime()); - backup.Metadata["LastVacuumFinished"] = Library.Utility.Utility.SerializeDateTime(r.EndTime.ToUniversalTime()); + backup.Metadata["LastVacuumStarted"] = Utility.SerializeDateTime(r.BeginTime.ToUniversalTime()); + backup.Metadata["LastVacuumFinished"] = Utility.SerializeDateTime(r.EndTime.ToUniversalTime()); } } - private static void UpdateMetadata(Duplicati.Server.Serialization.Interface.IBackup backup, Duplicati.Library.Interface.IParsedBackendStatistics r) + private static void UpdateMetadataStatistics(IBackup backup, IParsedBackendStatistics r) { if (r != null) { - backup.Metadata["LastBackupDate"] = Library.Utility.Utility.SerializeDateTime(r.LastBackupDate.ToUniversalTime()); + backup.Metadata["LastBackupDate"] = Utility.SerializeDateTime(r.LastBackupDate.ToUniversalTime()); backup.Metadata["BackupListCount"] = r.BackupListCount.ToString(); backup.Metadata["TotalQuotaSpace"] = r.TotalQuotaSpace.ToString(); backup.Metadata["FreeQuotaSpace"] = r.FreeQuotaSpace.ToString(); @@ -814,28 +862,28 @@ namespace Duplicati.Server backup.Metadata["TargetFilesSize"] = r.KnownFileSize.ToString(); backup.Metadata["TargetFilesCount"] = r.KnownFileCount.ToString(); backup.Metadata["TargetFilesetsCount"] = r.KnownFilesets.ToString(); - backup.Metadata["TargetSizeString"] = Duplicati.Library.Utility.Utility.FormatSizeString(r.KnownFileSize); + backup.Metadata["TargetSizeString"] = Utility.FormatSizeString(r.KnownFileSize); } } - private static void UpdateMetadata(Duplicati.Server.Serialization.Interface.IBackup backup, Duplicati.Library.Interface.IBasicResults result) + private static void UpdateMetadataBase(Connection databaseConnection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IBackup backup, IBasicResults result) { if (result is IRestoreResults r1) { backup.Metadata["LastRestoreDuration"] = r1.Duration.ToString(); - backup.Metadata["LastRestoreStarted"] = Library.Utility.Utility.SerializeDateTime(result.BeginTime.ToUniversalTime()); - backup.Metadata["LastRestoreFinished"] = Library.Utility.Utility.SerializeDateTime(result.EndTime.ToUniversalTime()); + backup.Metadata["LastRestoreStarted"] = Utility.SerializeDateTime(result.BeginTime.ToUniversalTime()); + backup.Metadata["LastRestoreFinished"] = Utility.SerializeDateTime(result.EndTime.ToUniversalTime()); } if (result is IParsedBackendStatistics r2 && !result.Interrupted) { - UpdateMetadata(backup, r2); + UpdateMetadataStatistics(backup, r2); } if (result is IBackendStatsticsReporter r3 && !result.Interrupted) { if (r3.BackendStatistics is IParsedBackendStatistics statistics) - UpdateMetadata(backup, statistics); + UpdateMetadataStatistics(backup, statistics); } if (result is ICompactResults r4 && !result.Interrupted) @@ -858,8 +906,8 @@ namespace Duplicati.Server backup.Metadata["SourceFilesSize"] = r.SizeOfExaminedFiles.ToString(); backup.Metadata["SourceFilesCount"] = r.ExaminedFiles.ToString(); backup.Metadata["SourceSizeString"] = Duplicati.Library.Utility.Utility.FormatSizeString(r.SizeOfExaminedFiles); - backup.Metadata["LastBackupStarted"] = Library.Utility.Utility.SerializeDateTime(r.BeginTime.ToUniversalTime()); - backup.Metadata["LastBackupFinished"] = Library.Utility.Utility.SerializeDateTime(r.EndTime.ToUniversalTime()); + backup.Metadata["LastBackupStarted"] = Utility.SerializeDateTime(r.BeginTime.ToUniversalTime()); + backup.Metadata["LastBackupFinished"] = Utility.SerializeDateTime(r.EndTime.ToUniversalTime()); backup.Metadata["LastBackupDuration"] = r.Duration.ToString(); if (r.CompactResults != null) @@ -889,7 +937,7 @@ namespace Duplicati.Server titleType = "Warning"; } - FIXMEGlobal.DataConnection.RegisterNotification( + databaseConnection.RegisterNotification( r.FilesWithError == 0 && !r.Errors.Any() ? NotificationType.Warning : NotificationType.Error, backup.IsTemporary ? "Warning" : $"{titleType} while running {backup.Name}", message, @@ -935,7 +983,7 @@ namespace Duplicati.Server else if (result.ParsedResult == ParsedResultType.Error && result.Errors.Count() == 1) message = $"Error: {result.Errors.Single()}"; - FIXMEGlobal.DataConnection.RegisterNotification( + databaseConnection.RegisterNotification( type, title, message, @@ -950,10 +998,10 @@ namespace Duplicati.Server } if (!backup.IsTemporary) - FIXMEGlobal.DataConnection.SetMetadata(backup.Metadata, long.Parse(backup.ID), null); + databaseConnection.SetMetadata(backup.Metadata, long.Parse(backup.ID), null); - FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); + notificationUpdateService.IncrementLastDataUpdateId(); + eventPollNotify.SignalNewEvent(); } private static bool TestIfOptionApplies() @@ -962,23 +1010,20 @@ namespace Duplicati.Server return true; } - private static void DisableModule(string module, Dictionary options) + private static void DisableModule(string module, Dictionary options) { - string disabledModules; - string enabledModules; - - if (options.TryGetValue("enable-module", out enabledModules)) + if (options.TryGetValue("enable-module", out var enabledModules)) { var emods = (enabledModules ?? "").Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries); options["enable-module"] = string.Join(",", emods.Where(x => module.Equals(x, StringComparison.OrdinalIgnoreCase))); } - options.TryGetValue("disable-module", out disabledModules); + options.TryGetValue("disable-module", out var disabledModules); var mods = (disabledModules ?? "").Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries); options["disable-module"] = string.Join(",", mods.Union(new string[] { module }).Distinct(StringComparer.OrdinalIgnoreCase)); } - internal static Dictionary ApplyOptions(Duplicati.Server.Serialization.Interface.IBackup backup, Dictionary options) + internal static Dictionary ApplyOptions(Connection databaseConnection, Serialization.Interface.IBackup backup, Dictionary options) { options["backup-name"] = backup.Name; options["dbpath"] = backup.DBPath; @@ -998,7 +1043,7 @@ namespace Duplicati.Server DisableModule("console-password-input", options); // Patch in additional report urls - var additionalReportUrl = FIXMEGlobal.DataConnection.ApplicationSettings.AdditionalReportUrl; + var additionalReportUrl = databaseConnection.ApplicationSettings.AdditionalReportUrl; if (!string.IsNullOrWhiteSpace(additionalReportUrl)) { options["send-http-json-urls"] = string.Join(";", @@ -1012,7 +1057,7 @@ namespace Duplicati.Server return options; } - private static Library.Utility.IFilter ApplyFilter(Serialization.Interface.IBackup backup, Library.Utility.IFilter filter) + private static Library.Utility.IFilter? ApplyFilter(IBackup backup, Library.Utility.IFilter? filter) { var f2 = backup.Filters; if (f2 != null && f2.Length > 0) @@ -1024,26 +1069,26 @@ namespace Duplicati.Server ? SpecialFolders.ExpandEnvironmentVariablesRegexp(n.Expression) : SpecialFolders.ExpandEnvironmentVariables(n.Expression) orderby n.Order - select (Library.Utility.IFilter)(new Library.Utility.FilterExpression(exp, n.Include))) - .Aggregate((a, b) => Library.Utility.FilterExpression.Combine(a, b)); + select (Library.Utility.IFilter)new FilterExpression(exp, n.Include)) + .Aggregate((a, b) => FilterExpression.Combine(a, b)); - filter = Library.Utility.FilterExpression.Combine(filter, nf); + filter = FilterExpression.Combine(filter, nf); } return filter; } - public static Dictionary GetCommonOptions() + public static Dictionary GetCommonOptions(Connection databaseConnection) { return - (from n in FIXMEGlobal.DataConnection.Settings + (from n in databaseConnection.Settings where TestIfOptionApplies() - select n).ToDictionary(k => k.Name.StartsWith("--", StringComparison.Ordinal) ? k.Name.Substring(2) : k.Name, k => k.Value); + select n).ToDictionary(k => k.Name.StartsWith("--", StringComparison.Ordinal) ? k.Name.Substring(2) : k.Name, k => (string?)k.Value); } - private static Duplicati.Library.Utility.IFilter GetCommonFilter() + private static Library.Utility.IFilter? GetCommonFilter(Connection databaseConnection) { - var filters = FIXMEGlobal.DataConnection.Filters; + var filters = databaseConnection.Filters; if (filters == null || filters.Length == 0) return null; @@ -1051,8 +1096,8 @@ namespace Duplicati.Server (from n in filters orderby n.Order let exp = Environment.ExpandEnvironmentVariables(n.Expression) - select (Duplicati.Library.Utility.IFilter)(new Duplicati.Library.Utility.FilterExpression(exp, n.Include))) - .Aggregate((a, b) => Duplicati.Library.Utility.FilterExpression.Combine(a, b)); + select (Library.Utility.IFilter)new FilterExpression(exp, n.Include)) + .Aggregate((a, b) => FilterExpression.Combine(a, b)); } } } diff --git a/Duplicati/Library/RestAPI/Scheduler.cs b/Duplicati/Library/RestAPI/Scheduler.cs index fc744e41c..bc1f175cc 100644 --- a/Duplicati/Library/RestAPI/Scheduler.cs +++ b/Duplicati/Library/RestAPI/Scheduler.cs @@ -19,6 +19,8 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +#nullable enable + using Duplicati.Server.Serialization.Interface; using System; @@ -27,7 +29,9 @@ using System.Text; using System.Linq; using System.Threading; using Duplicati.Library.Utility; -using Duplicati.Library.RestAPI; +using System.Threading.Tasks; +using Duplicati.WebserverCore.Abstractions; +using Duplicati.Server.Database; // TODO: Rewrite this class. // It should just signal what new backups to run, and not mix with the worker thread. @@ -39,7 +43,7 @@ namespace Duplicati.Server /// public class Scheduler { - private static readonly string LOGTAG = Duplicati.Library.Logging.Log.LogTagFromType(); + private static readonly string LOGTAG = Library.Logging.Log.LogTagFromType(); /// /// The thread that runs the scheduler @@ -51,11 +55,6 @@ namespace Duplicati.Server /// private volatile bool m_terminate; - /// - /// The worker thread that is invoked to do work - /// - private WorkerThread m_worker; - /// /// The wait event /// @@ -67,9 +66,13 @@ namespace Duplicati.Server private readonly object m_lock = new object(); /// - /// An event that is raised when the schedule changes + /// The queue runner service /// - public event EventHandler NewSchedule; + private readonly IQueueRunnerService m_queueRunnerService; + /// + /// The data connection + /// + private readonly Connection m_dataConnection; /// /// The currently scheduled items @@ -79,45 +82,31 @@ namespace Duplicati.Server /// /// List of update tasks, used to set the timestamp on the schedule once completed /// - private Dictionary> m_updateTasks; + private Dictionary> m_updateTasks; /// /// Constructs a new scheduler /// - public Scheduler() + /// The data connection + /// The queue runner service + public Scheduler(Connection connection, IQueueRunnerService queueRunnerService) { - } - - /// - /// Initializes scheduler - /// - /// The worker thread - public void Init(WorkerThread worker) - { - m_worker = worker; + m_dataConnection = connection; + m_queueRunnerService = queueRunnerService; m_thread = new Thread(new ThreadStart(Runner)); - m_worker.CompletedWork += OnCompleted; - m_worker.StartingWork += OnStartingWork; - m_schedule = new KeyValuePair[0]; + m_schedule = []; m_terminate = false; m_event = new AutoResetEvent(false); - m_updateTasks = new Dictionary>(); + m_updateTasks = new Dictionary>(); m_thread.IsBackground = true; m_thread.Name = "TaskScheduler"; m_thread.Start(); } - public IList> GetSchedulerQueueIds() - { - return (from n in WorkerQueue - where n.Backup != null - select new Tuple(n.TaskID, n.Backup.ID)).ToList(); - } - public IList> GetProposedSchedule() { return ( - from n in FIXMEGlobal.Scheduler.Schedule + from n in this.Schedule let backupid = (from t in n.Value.Tags where t != null && t.StartsWith("ID=", StringComparison.Ordinal) select t.Substring("ID=".Length)).FirstOrDefault() @@ -147,14 +136,6 @@ namespace Duplicati.Server } } - /// - /// A snapshot copy of the current worker queue, that is items that are scheduled, but waiting for execution - /// - public List WorkerQueue - { - get { return m_worker?.CurrentTasks?.Where(t => t != null)?.ToList() ?? []; } - } - /// /// Terminates the thread. Any items still in queue will be removed /// @@ -233,9 +214,9 @@ namespace Duplicati.Server return res; } - private void OnCompleted(WorkerThread worker, Runner.IRunnerData task) + private Task OnCompleted(Runner.IRunnerData task) { - Tuple t = null; + Tuple? t = null; lock (m_lock) { if (task != null && m_updateTasks.TryGetValue(task, out t)) @@ -246,26 +227,28 @@ namespace Duplicati.Server { t.Item1.Time = t.Item2; t.Item1.LastRun = t.Item3; - FIXMEGlobal.DataConnection.AddOrUpdateSchedule(t.Item1); + m_dataConnection.AddOrUpdateSchedule(t.Item1); } + + return Task.CompletedTask; } - private void OnStartingWork(WorkerThread worker, Runner.IRunnerData task) + private Task OnStartingWork(Runner.IRunnerData task) { if (task is null) - { - return; - } + return Task.CompletedTask; lock (m_lock) { - if (m_updateTasks.TryGetValue(task, out Tuple scheduleInfo)) + if (m_updateTasks.TryGetValue(task, out var scheduleInfo)) { // Item2 is the scheduled start time (Time in the Schedule table). // Item3 is the actual start time (LastRun in the Schedule table). m_updateTasks[task] = Tuple.Create(scheduleInfo.Item1, scheduleInfo.Item2, DateTime.UtcNow); } } + + return Task.CompletedTask; } /// @@ -280,8 +263,8 @@ namespace Duplicati.Server // to avoid frequent db lookups //Determine schedule list - var timeZoneInfo = FIXMEGlobal.DataConnection.ApplicationSettings.Timezone; - var lst = FIXMEGlobal.DataConnection.Schedules; + var timeZoneInfo = m_dataConnection.ApplicationSettings.Timezone; + var lst = m_dataConnection.Schedules; foreach (var sc in lst) { if (!string.IsNullOrEmpty(sc.Repeat)) @@ -315,7 +298,7 @@ namespace Duplicati.Server } catch (Exception ex) { - FIXMEGlobal.DataConnection.LogError(sc.ID.ToString(), "Scheduler failed to find next date", + m_dataConnection.LogError(sc.ID.ToString(), "Scheduler failed to find next date", ex); } @@ -324,37 +307,37 @@ namespace Duplicati.Server { var jobsToRun = new List(); //TODO: Cache this to avoid frequent lookups - foreach (var id in FIXMEGlobal.DataConnection.GetBackupIDsForTags(sc.Tags).Distinct() + foreach (var id in m_dataConnection.GetBackupIDsForTags(sc.Tags).Distinct() .Select(x => x.ToString())) { //See if it is already queued - var tmplst = from n in m_worker.CurrentTasks - where n.Operation == Duplicati.Server.Serialization.DuplicatiOperation.Backup - select n.Backup; - var tastTemp = m_worker.CurrentTask; + var tmplst = from n in m_queueRunnerService.GetCurrentTasks() + where n.Operation == Serialization.DuplicatiOperation.Backup + select n.BackupID; + var tastTemp = m_queueRunnerService.GetCurrentTask(); if (tastTemp != null && tastTemp.Operation == - Duplicati.Server.Serialization.DuplicatiOperation.Backup) - tmplst = tmplst.Union(new[] { tastTemp.Backup }); + Serialization.DuplicatiOperation.Backup) + tmplst = tmplst.Union(new[] { tastTemp.BackupID }); //If it is not already in queue, put it there - if (!tmplst.Any(x => x.ID == id)) + if (!tmplst.Any(x => x == id)) { - var entry = FIXMEGlobal.DataConnection.GetBackup(id); + var entry = m_dataConnection.GetBackup(id); if (entry != null) { - Dictionary options = Duplicati.Server.Runner.GetCommonOptions(); - Duplicati.Server.Runner.ApplyOptions(entry, options); - if ((new Duplicati.Library.Main.Options(options)).DisableOnBattery && - (Duplicati.Library.Utility.Power.PowerSupply.GetSource() == - Duplicati.Library.Utility.Power.PowerSupply.Source.Battery)) + var options = Server.Runner.GetCommonOptions(m_dataConnection); + Server.Runner.ApplyOptions(m_dataConnection, entry, options); + if (new Library.Main.Options(options).DisableOnBattery && + (Library.Utility.Power.PowerSupply.GetSource() == + Library.Utility.Power.PowerSupply.Source.Battery)) { - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, + Library.Logging.Log.WriteInformationMessage(LOGTAG, "BackupDisabledOnBattery", "Scheduled backup disabled while on battery power."); } else { - Dictionary taskOptions = null; + Dictionary? taskOptions = null; try { var nextRun = GetNextValidTime(start, @@ -362,15 +345,17 @@ namespace Duplicati.Server Math.Max(DateTime.UtcNow.AddSeconds(1).Ticks, start.AddSeconds(1).Ticks), DateTimeKind.Utc), sc.Repeat, sc.AllowedDays, timeZoneInfo); - taskOptions = new Dictionary() + taskOptions = new Dictionary() { { "next-scheduled-run", Utility.SerializeDateTime(nextRun.ToUniversalTime()) } }; } catch { } - jobsToRun.Add(Server.Runner.CreateTask( - Serialization.DuplicatiOperation.Backup, entry, taskOptions)); + var job = Server.Runner.CreateTask(Serialization.DuplicatiOperation.Backup, entry, taskOptions); + job.OnStarting = () => OnStartingWork(job); + job.OnFinished = (_) => OnCompleted(job); + jobsToRun.Add(job); } } } @@ -386,12 +371,12 @@ namespace Duplicati.Server } catch (Exception ex) { - FIXMEGlobal.DataConnection.LogError(sc.ID.ToString(), + m_dataConnection.LogError(sc.ID.ToString(), "Scheduler failed to find next date", ex); continue; } - Server.Runner.IRunnerData lastJob = jobsToRun.LastOrDefault(); + var lastJob = jobsToRun.LastOrDefault(); if (lastJob != null) { lock (m_lock) @@ -403,7 +388,7 @@ namespace Duplicati.Server } foreach (var job in jobsToRun) - m_worker.AddTask(job); + m_queueRunnerService.AddTask(job); if (start < DateTime.UtcNow) { @@ -428,11 +413,6 @@ namespace Duplicati.Server foreach (var c in (from n in scheduled where !existing.ContainsKey(n.Key) select n.Key).ToArray()) scheduled.Remove(c); - //Raise event if needed - // TODO: This triggers a new data event and a reconnect with long-poll - if (NewSchedule != null) - NewSchedule(this, null); - int waittime = 0; //Figure out a sensible amount of time to sleep the thread diff --git a/Duplicati/Library/RestAPI/Serializable/ServerSettings.cs b/Duplicati/Library/RestAPI/Serializable/ServerSettings.cs index 180e7f0fd..6174fab70 100644 --- a/Duplicati/Library/RestAPI/Serializable/ServerSettings.cs +++ b/Duplicati/Library/RestAPI/Serializable/ServerSettings.cs @@ -18,9 +18,8 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; + using System.Linq; -using Duplicati.Library.RestAPI; using Duplicati.Server.Serialization.Interface; namespace Duplicati.Server.Serializable @@ -38,7 +37,7 @@ namespace Duplicati.Server.Serializable /// /// Constructor for backend interface /// - public DynamicModule(Duplicati.Library.Interface.IBackend backend) + public DynamicModule(Library.Interface.IBackend backend) { this.Key = backend.ProtocolKey; this.Description = backend.Description; @@ -49,7 +48,7 @@ namespace Duplicati.Server.Serializable /// /// Constructor for compression module interface /// - public DynamicModule(Duplicati.Library.Interface.ICompression module) + public DynamicModule(Library.Interface.ICompression module) { this.Key = module.FilenameExtension; this.Description = module.Description; @@ -60,7 +59,7 @@ namespace Duplicati.Server.Serializable /// /// Constructor for encryption module interface /// - public DynamicModule(Duplicati.Library.Interface.IEncryption module) + public DynamicModule(Library.Interface.IEncryption module) { this.Key = module.FilenameExtension; this.Description = module.Description; @@ -71,7 +70,7 @@ namespace Duplicati.Server.Serializable /// /// Constructor for generic module interface /// - public DynamicModule(Duplicati.Library.Interface.IGenericModule module) + public DynamicModule(Library.Interface.IGenericModule module) { this.Key = module.Key; this.Description = module.Description; @@ -82,7 +81,7 @@ namespace Duplicati.Server.Serializable /// /// Constructor for webmodule interface /// - public DynamicModule(Duplicati.Library.Interface.IWebModule module) + public DynamicModule(Library.Interface.IWebModule module) { this.Key = module.Key; this.Description = module.Description; @@ -93,7 +92,7 @@ namespace Duplicati.Server.Serializable /// /// Constructor for sercretprovider interface /// - public DynamicModule(Duplicati.Library.Interface.ISecretProvider module) + public DynamicModule(Library.Interface.ISecretProvider module) { this.Key = module.Key; this.Description = module.Description; @@ -116,13 +115,13 @@ namespace Duplicati.Server.Serializable /// /// The options supported by the module /// - public Duplicati.Library.Interface.ICommandLineArgument[] Options { get; private set; } + public Library.Interface.ICommandLineArgument[] Options { get; private set; } } /// /// Gets all supported options /// - public static Duplicati.Library.Interface.ICommandLineArgument[] Options + public static Library.Interface.ICommandLineArgument[] Options { get { @@ -242,21 +241,5 @@ namespace Duplicati.Server.Serializable .ToArray(); } } - - /// - /// The filters that are applied to all backups - /// - public static IFilter[] Filters - { - get { return FIXMEGlobal.DataConnection.Filters; } - } - - /// - /// The settings applied to all backups by default - /// - public static ISetting[] Settings - { - get { return FIXMEGlobal.DataConnection.Settings; } - } } } diff --git a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj index 1fb23330e..ee6d149bb 100644 --- a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj +++ b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj @@ -8,7 +8,7 @@ - + diff --git a/Duplicati/Library/Utility/Options/AuthIdOptionsHelper.cs b/Duplicati/Library/Utility/Options/AuthIdOptionsHelper.cs index 2689d19d0..33a2a7653 100644 --- a/Duplicati/Library/Utility/Options/AuthIdOptionsHelper.cs +++ b/Duplicati/Library/Utility/Options/AuthIdOptionsHelper.cs @@ -44,7 +44,7 @@ public static class AuthIdOptionsHelper /// The authentication options public static CommandLineArgument[] GetOptions(string tokenurl, string? prefix = null) => [ - new CommandLineArgument($"{prefix}{AuthIdOption}", CommandLineArgument.ArgumentType.String, Strings.AuthIdSettingsHelper.AuthidShort, Strings.AuthIdSettingsHelper.AuthidLong(tokenurl)) + new CommandLineArgument($"{prefix}{AuthIdOption}", CommandLineArgument.ArgumentType.Password, Strings.AuthIdSettingsHelper.AuthidShort, Strings.AuthIdSettingsHelper.AuthidLong(tokenurl)) ]; /// diff --git a/Duplicati/Library/Utility/OverrideableStream.cs b/Duplicati/Library/Utility/OverrideableStream.cs index 35e8a5273..b331c0b2f 100644 --- a/Duplicati/Library/Utility/OverrideableStream.cs +++ b/Duplicati/Library/Utility/OverrideableStream.cs @@ -37,6 +37,11 @@ namespace Duplicati.Library.Utility /// protected Stream m_basestream; + /// + /// The base stream that is wrapped + /// + public Stream BaseStream => m_basestream; + /// /// Creates a new instance /// diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index c83243532..d29f548ab 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -23,6 +23,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.IO; using System.Linq; @@ -1397,7 +1398,8 @@ namespace Duplicati.Library.Utility /// The wrapped commandline element. /// The argument to wrap. /// A flag indicating if environment variables are allowed to be expanded - public static string WrapCommandLineElement(string arg, bool allowEnvExpansion) + [return: NotNullIfNotNull("arg")] + public static string? WrapCommandLineElement(string? arg, bool allowEnvExpansion) { if (string.IsNullOrWhiteSpace(arg)) return arg; @@ -1847,5 +1849,90 @@ namespace Duplicati.Library.Utility : Timeout.Infinite }; } + + /// + /// The types of streams that are considered basic (i.e. not wrapped) + /// + private static readonly IReadOnlySet _basicStreamTypes = new HashSet + { + typeof(FileStream), + typeof(MemoryStream), + typeof(NetworkStream), + typeof(BufferedStream), + typeof(System.IO.Compression.DeflateStream), + typeof(System.IO.Compression.GZipStream), + typeof(System.IO.Compression.ZLibStream) + }; + + /// + /// Unwraps a stream from layers of wrapping streams + /// + /// The stream to unwrap + /// The unwrapped stream + public static Stream UnwrapThrottledStream(this Stream stream) + { + if (stream == null) + throw new ArgumentNullException(nameof(stream)); + + var previousStream = stream; + + do + { + previousStream = stream; + + while (stream is WrappingStream wrappingStream) + stream = wrappingStream.BaseStream; + while (stream is OverrideableStream overrideableStream) + stream = overrideableStream.BaseStream; + + } while (stream != previousStream); + +#if DEBUG + if (!_basicStreamTypes.Contains(stream.GetType())) + throw new InvalidOperationException($"The unwrapped stream is not a basic stream, but a {stream.GetType()}"); +#endif + + return stream; + } + + /// + /// Calculates the hash of a throttled stream and returns the stream to read + /// + /// The source stream + /// The hash algorithm to use + /// The cancellation token to observe + /// A tuple with the stream, the hash and a temporary file if used + public static async Task<(Stream content, string hash, TempFile? tmpfile)> CalculateThrottledStreamHash(Stream stream, string hashalgorithm, CancellationToken cancelToken) + { + TempFile? tmp = null; + string contentHash; + var measure = stream.UnwrapThrottledStream(); + if (measure.CanSeek) + { + var p = measure.Position; + + // Compute the hash + using (var hashalg = HashFactory.CreateHasher(hashalgorithm)) + contentHash = ByteArrayAsHexString(hashalg.ComputeHash(measure)); + + measure.Position = p; + } + else + { + // No seeking possible, use a temp file + tmp = new TempFile(); + await using (var sr = File.OpenWrite(tmp)) + using (var hasher = HashFactory.CreateHasher(hashalgorithm)) + await using (var hc = new HashCalculatingStream(measure, hasher)) + { + await CopyStreamAsync(hc, sr, cancelToken).ConfigureAwait(false); + contentHash = hc.GetFinalHashString(); + } + + stream = File.OpenRead(tmp); + } + + return (stream, contentHash, tmp); + } } } diff --git a/Duplicati/Library/Utility/WorkerThread.cs b/Duplicati/Library/Utility/WorkerThread.cs deleted file mode 100644 index 285aac182..000000000 --- a/Duplicati/Library/Utility/WorkerThread.cs +++ /dev/null @@ -1,401 +0,0 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -using System; -using System.Collections.Generic; -using System.Text; -using System.Threading; - - -// TODO: Delete this class. -// It is essentially a queue that is processed by a worker thread, and can be implemented using a BlockingCollection or similar. - -namespace Duplicati.Library.Utility -{ - /// - /// Class to encapsulate a thread that runs a list of queued operations - /// - /// The type to operate on - public class WorkerThread where Tx : class - { - /// - /// Locking object for shared data - /// - private readonly object m_lock = new object(); - /// - /// The wait event - /// - private readonly AutoResetEvent m_event; - /// - /// The internal list of tasks to perform - /// - private Queue m_tasks; - /// - /// A flag used to terminate the thread - /// - private volatile bool m_terminate; - /// - /// The coordinating thread - /// - private Thread m_thread; - - /// - /// A value indicating if the coordinating thread is running - /// - private volatile bool m_active; - - /// - /// The current task being processed - /// - private Tx m_currentTask; - /// - /// A callback that performs the actual work on the item - /// - private readonly Action m_delegate; - - /// - /// An event that is raised when the runner state changes - /// - public event Action, RunState> WorkerStateChanged; - - /// - /// Event that occurs when a new operation is being processed - /// - public event Action, Tx> StartingWork; - /// - /// Event that occurs when an operation has completed - /// - public event Action, Tx> CompletedWork; - /// - /// Event that occurs when an error is detected - /// - public event Action, Tx, Exception> OnError; - /// - /// An event that occurs when a new task is added to the queue or an existing one is removed - /// - public event Action> WorkQueueChanged; - - /// - /// The internal state - /// - private volatile RunState m_state; - - /// - /// The states the scheduler can take - /// - public enum RunState - { - /// - /// The program is running as normal - /// - Run, - /// - /// The program is suspended by the user - /// - Paused - } - - /// - /// Constructs a new WorkerThread - /// - /// The callback that performs the work - public WorkerThread(Action item, bool paused) - { - m_delegate = item; - m_event = new AutoResetEvent(paused); - m_terminate = false; - m_tasks = new Queue(); - m_state = paused ? WorkerThread.RunState.Paused : WorkerThread.RunState.Run; - - m_thread = new Thread(new ThreadStart(Runner)); - m_thread.IsBackground = true; - m_thread.Name = "WorkerThread<" + typeof(Tx).Name + ">"; - m_thread.Start(); - } - - /// - /// Gets a copy of the current queue - /// - public List CurrentTasks - { - get - { - lock (m_lock) - return new List(m_tasks); - } - - } - - /// - /// Gets a value indicating if the worker is running - /// - public bool Active - { - get { return m_active; } - } - - /// - /// Adds a task to the queue - /// - /// The task to add - public void AddTask(Tx task) - { - lock (m_lock) - { - m_tasks.Enqueue(task); - m_event.Set(); - } - - if (WorkQueueChanged != null) - WorkQueueChanged(this); - } - - /// - /// An overloaded AddTask method that allows a task to skip to the front of a queue - /// It does this by creating a new queue, adding the new task first, and then adding - /// all the old tasks to the new queue. It's cleaner to use a linked list, - /// but the performance difference is negligible on such a small queue. - /// - /// Task. - /// If set to true skip queue. - public void AddTask(Tx task, bool skipQueue) - { - if (!skipQueue) - { - // Fall back to default AddTask method - AddTask(task); - return; - } - - lock (m_lock) - { - Queue newQueue = new Queue(); - newQueue.Enqueue(task); - while (m_tasks.Count > 0) - { - Tx n = m_tasks.Dequeue(); - newQueue.Enqueue(n); - } - m_tasks = newQueue; - m_event.Set(); - } - - if (WorkQueueChanged != null) - WorkQueueChanged(this); - } - - - /// - /// Removes a task from the queue, does not remove the task if it is currently running - /// - /// The task to remove - public void RemoveTask(Tx task) - { - lock (m_lock) - { - Queue tmp = new Queue(); - while (m_tasks.Count > 0) - { - Tx n = m_tasks.Dequeue(); - if (n != task) - tmp.Enqueue(n); - } - - m_tasks = tmp; - } - - if (WorkQueueChanged != null) - WorkQueueChanged(this); - } - - /// - /// This will clear the pending queue - /// True if the current running thread should be aborted - /// - public void ClearQueue(bool abortThread) - { - lock (m_lock) - m_tasks.Clear(); - - if (abortThread) - { - try - { - m_thread.Interrupt(); - m_thread.Join(500); - } - catch - { - } - - m_thread = new Thread(new ThreadStart(Runner)); - m_thread.Start(); - } - } - - /// - /// Gets a reference to the currently executing task. - /// BEWARE: This is not protected by a mutex, DO NOT MODIFY IT!!!! - /// - public Tx CurrentTask - { - get - { - return m_currentTask; - } - } - - /// - /// Terminates the thread. Any items still in queue will be removed - /// - /// True if the call should block until the thread has exited, false otherwise - public void Terminate(bool wait) - { - m_terminate = true; - m_event.Set(); - - if (wait) - m_thread.Join(); - } - - /// - /// This is the thread entry point - /// - private void Runner() - { - while (!m_terminate) - { - m_currentTask = null; - - lock (m_lock) - if (m_state == WorkerThread.RunState.Run && m_tasks.Count > 0) - m_currentTask = m_tasks.Dequeue(); - - if (m_currentTask == null && !m_terminate) - { - if (m_state == WorkerThread.RunState.Run) - m_event.WaitOne(); //Sleep until signaled - else - { - if (WorkerStateChanged != null) - WorkerStateChanged(this, m_state); - - //Sleep for brief periods, until signaled - while (!m_terminate && m_state != WorkerThread.RunState.Run) - m_event.WaitOne(1000 * 60 * 5, false); - - //If we were not terminated, we are now ready to run - if (!m_terminate) - { - m_state = WorkerThread.RunState.Run; - if (WorkerStateChanged != null) - WorkerStateChanged(this, m_state); - } - } - } - - if (m_terminate) - return; - - if (m_currentTask == null && m_state == WorkerThread.RunState.Run) - lock (m_lock) - if (m_tasks.Count > 0) - m_currentTask = m_tasks.Dequeue(); - - if (m_currentTask == null) - continue; - - if (StartingWork != null) - StartingWork(this, m_currentTask); - - try - { - m_active = true; - m_delegate(m_currentTask); - } - catch (Exception ex) - { - //TODO: Here where Thread.ResetAbort() was called we shall integrate the CancelationToken pattern. - if (OnError != null) - try { OnError(this, m_currentTask, ex); } - catch - { - // ignored - } - } - finally - { - //TODO: Here where Thread.ResetAbort() was called we shall integrate the CancelationToken pattern. - m_active = false; - } - - var task = m_currentTask; - m_currentTask = null; - - if (CompletedWork != null) - try { CompletedWork(this, task); } - catch (Exception ex) - { - try { OnError(this, task, ex); } - catch - { - // ignored - } - } - } - } - - /// - /// Gets the current run state - /// - public RunState State { get { return m_state; } } - - /// - /// Instructs Duplicati to run scheduled backups - /// - public void Resume() - { - m_state = RunState.Run; - m_event.Set(); - } - - /// - /// Instructs Duplicati to pause scheduled backups - /// - public void Pause() - { - m_state = RunState.Paused; - m_event.Set(); - } - - /// - /// Waits the specified number of milliseconds for the thread to terminate - /// - /// The number of milliseconds to wait - /// True if the thread is terminated, false if a timeout occured - public bool Join(int millisecondTimeout) - { - if (m_thread != null) - return m_thread.Join(millisecondTimeout); - return true; - } - } -} \ No newline at end of file diff --git a/Duplicati/Server/Duplicati.Server.Serialization/Interface/IQueuedTask.cs b/Duplicati/Server/Duplicati.Server.Serialization/Interface/IQueuedTask.cs index 385287292..ccaa14a35 100644 --- a/Duplicati/Server/Duplicati.Server.Serialization/Interface/IQueuedTask.cs +++ b/Duplicati/Server/Duplicati.Server.Serialization/Interface/IQueuedTask.cs @@ -17,16 +17,71 @@ // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -namespace Duplicati.Server.Serialization.Interface -{ - public interface IQueuedTask - { - long TaskID { get; } - string BackupID { get; } - Duplicati.Server.Serialization.DuplicatiOperation Operation { get; } - - } -} - +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.Threading.Tasks; + +namespace Duplicati.Server.Serialization.Interface; + +/// +/// Represents a queued task. +/// +public interface IQueuedTask +{ + /// + /// The task ID. + /// + long TaskID { get; } + /// + /// The backup ID, if applicable. + /// + string? BackupID { get; } + /// + /// The operation type of the task. + /// + DuplicatiOperation Operation { get; } + /// + /// Callback to be executed when the task is starting. + /// + Func? OnStarting { get; set; } + /// + /// Callback to be executed when the task is finished. + /// If the task completes successfully, the exception parameter will be null. + /// + Func? OnFinished { get; set; } + + /// + /// Updates the throttle speeds for the task. + /// + /// The upload speed to set. + /// The download speed to set. + void UpdateThrottleSpeeds(string? uploadSpeed, string? downloadSpeed); + /// + /// The time when the task was starting to execute. + /// + DateTime? TaskStarted { get; set; } + /// + /// The time when the task was finished executing. + /// + DateTime? TaskFinished { get; set; } + /// + /// Stops the task. + /// + void Stop(); + /// + /// Aborts the task. + /// + void Abort(); + /// + /// Pauses the task. + /// + /// If true, also pauses transfers. + void Pause(bool alsoTransfers); + /// + /// Resumes the task. + /// + void Resume(); +} diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index fc1f7809a..d9505180b 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -32,11 +32,11 @@ using Duplicati.Library.Interface; using Duplicati.Library.Logging; using Duplicati.Library.Main; using Duplicati.Library.Main.Database; -using Duplicati.Library.RestAPI; using Duplicati.Library.Utility; using Duplicati.Server.Database; using Duplicati.WebserverCore; using Duplicati.WebserverCore.Abstractions; +using Duplicati.WebserverCore.Services; using Microsoft.Extensions.DependencyInjection; namespace Duplicati.Server @@ -84,61 +84,21 @@ namespace Duplicati.Server /// private static readonly string ENV_NAME_PREFIX = AutoUpdateSettings.AppName.ToUpperInvariant(); - /// - /// Gets the folder where Duplicati data is stored - /// - public static string DataFolder { get => FIXMEGlobal.DataFolder; private set => FIXMEGlobal.DataFolder = value; } - /// /// The single instance /// public static SingleInstance ApplicationInstance = null; - /// - /// This is the only access to the database - /// - public static Database.Connection DataConnection { get => FIXMEGlobal.DataConnection; set => FIXMEGlobal.DataConnection = value; } - - /// - /// This is the lock to be used before manipulating the shared resources - /// - public static object MainLock { get => FIXMEGlobal.MainLock; } - - /// - /// This is the scheduling thread - /// - public static IScheduler Scheduler { get => FIXMEGlobal.Scheduler; } - - /// - /// List of completed task results - /// - public static List> TaskResultCache { get => FIXMEGlobal.TaskResultCache; } - - /// - /// The maximum number of completed task results to keep in memory - /// - private static readonly int MAX_TASK_RESULT_CACHE_SIZE = 100; - /// /// The thread running the ping-pong handler /// - private static System.Threading.Thread PingPongThread; - - /// - /// The path to the file that contains the current database - /// - private static string DatabasePath; + private static Thread PingPongThread; /// /// The controller interface for pause/resume and throttle options /// public static LiveControls LiveControl { get => DuplicatiWebserver.Provider.GetRequiredService(); } - /// - /// The application exit event - /// - public static System.Threading.ManualResetEvent ApplicationExitEvent { get => FIXMEGlobal.ApplicationExitEvent; set => FIXMEGlobal.ApplicationExitEvent = value; } - /// /// Duplicati webserver instance /// @@ -152,68 +112,38 @@ namespace Duplicati.Server DuplicatiWebserver.Stop().GetAwaiter().GetResult(); } - /// - /// The update poll thread. - /// - public static UpdatePollThread UpdatePoller => FIXMEGlobal.UpdatePoller; - /// /// An event that is set once the server is ready to respond to requests /// - public static readonly System.Threading.ManualResetEvent ServerStartedEvent = new System.Threading.ManualResetEvent(false); + public static readonly ManualResetEvent ServerStartedEvent = new ManualResetEvent(false); /// - /// The status event signaler, used to control long polling of status updates + /// Timer for purging temp files and log data /// - public static EventPollNotify StatusEventNotifyer => FIXMEGlobal.Provider.GetRequiredService(); - - /// - /// A delegate method for creating a copy of the current progress state - /// - public static Func GenerateProgressState { get => FIXMEGlobal.GenerateProgressState; set => FIXMEGlobal.GenerateProgressState = value; } - - /// - /// The log redirect handler - /// - public static LogWriteHandler LogHandler { get => FIXMEGlobal.LogHandler; } - private static System.Threading.Timer PurgeTempFilesTimer = null; - public static int ServerPort - { - get - { - return DuplicatiWebserver.Port; - } - } - - public static bool IsFirstRun - { - get { return DataConnection.ApplicationSettings.IsFirstRun; } - set { DataConnection.ApplicationSettings.IsFirstRun = value; } - } - - public static bool ServerPortChanged - { - get { return DataConnection.ApplicationSettings.ServerPortChanged; } - set { DataConnection.ApplicationSettings.ServerPortChanged = value; } - } - - static Program() - { - FIXMEGlobal.StartOrStopUsageReporter = Program.StartOrStopUsageReporter; - } - /// /// The main entry point for the application. /// + /// The command line arguments [STAThread] public static int Main(string[] _args) + => Main(null, _args); + + /// + /// The main entry point for the application. + /// + /// The application settings + /// The command line arguments + [STAThread] + public static int Main(IApplicationSettings applicationSettings, string[] _args) { PreloadSettingsLoader.ConfigurePreloadSettings(ref _args, PackageHelper.NamedExecutable.Server, out var preloadDbSettings); + applicationSettings ??= new ApplicationSettings(); + //If this executable is invoked directly, write to console, otherwise throw exceptions - var writeToConsoleOnException = FIXMEGlobal.Origin == "Server"; + var writeToConsoleOnException = applicationSettings.Origin == "Server"; // Prepared for the future, where we might want to have a silent console mode var silentConsole = false; @@ -238,7 +168,7 @@ namespace Duplicati.Server SystemContextSettings.StartSession(); ApplyEnvironmentVariables(commandlineOptions); - ApplySecretProvider(commandlineOptions, CancellationToken.None).Await(); + ApplySecretProvider(applicationSettings, commandlineOptions, CancellationToken.None).Await(); var parameterFileOption = PARAMETERS_FILE_OPTION_EXTRAS.Prepend(PARAMETERS_FILE_OPTION) .FirstOrDefault(x => commandlineOptions.ContainsKey(x)); @@ -251,62 +181,70 @@ namespace Duplicati.Server return 100; } - ConfigureLogging(commandlineOptions); + var logHandler = new LogWriteHandler(); + using var logScope = ConfigureLogging(logHandler, commandlineOptions); // Validate after logging is configured CommandLineArgumentValidator.ValidateArguments(SupportedCommands, commandlineOptions, KnownDuplicateOptions, ValidationIgnoredOptions); var crashed = false; var terminated = false; + IQueueRunnerService queueRunner = null; + UpdatePollThread updatePollThread = null; + EventPollNotify eventPollNotify = null; + ISchedulerService scheduler = null; try { - DataConnection = GetDatabaseConnection(commandlineOptions, silentConsole); + var connection = GetDatabaseConnection(applicationSettings, commandlineOptions, silentConsole); - if (!DataConnection.ApplicationSettings.FixedInvalidBackupId) - DataConnection.FixInvalidBackupId(); + if (!connection.ApplicationSettings.FixedInvalidBackupId) + connection.FixInvalidBackupId(); - DataConnection.ApplicationSettings.UpgradePasswordToKBDF(); - CreateApplicationInstance(writeToConsoleOnException); + connection.ApplicationSettings.UpgradePasswordToKBDF(); + CreateApplicationInstance(applicationSettings.DataFolder, writeToConsoleOnException); - StartOrStopUsageReporter(); + applicationSettings.StartOrStopUsageReporter = () => StartOrStopUsageReporter(connection); + applicationSettings.StartOrStopUsageReporter?.Invoke(); - AdjustApplicationSettings(commandlineOptions); - - ApplicationExitEvent = new System.Threading.ManualResetEvent(false); + AdjustApplicationSettings(connection, commandlineOptions); UpdaterManager.OnError += obj => { - DataConnection.LogError(null, "Error in updater", obj); + connection.LogError(null, "Error in updater", obj); }; - DuplicatiWebserver = StartWebServer(commandlineOptions, DataConnection).Await(); + DuplicatiWebserver = StartWebServer(commandlineOptions, connection, logHandler, applicationSettings).Await(); - UpdatePoller.Init(Library.Utility.Utility.ParseBoolOption(commandlineOptions, DISABLE_UPDATE_CHECK_OPTION)); + connection.SetServiceProvider(DuplicatiWebserver.Provider); + queueRunner = DuplicatiWebserver.Provider.GetRequiredService(); + updatePollThread = DuplicatiWebserver.Provider.GetRequiredService(); + eventPollNotify = DuplicatiWebserver.Provider.GetRequiredService(); + scheduler = DuplicatiWebserver.Provider.GetRequiredService(); - SetPurgeTempFilesTimer(commandlineOptions); + updatePollThread.Init(Library.Utility.Utility.ParseBoolOption(commandlineOptions, DISABLE_UPDATE_CHECK_OPTION)); - LiveControl.StateChanged = LiveControl_StateChanged; + SetPurgeTempFilesTimer(connection, commandlineOptions); - SetWorkerThread(); + LiveControl.StateChanged = (e) => { LiveControl_StateChanged(queueRunner, connection, eventPollNotify, e); }; if (Library.Utility.Utility.ParseBoolOption(commandlineOptions, PING_PONG_KEEPALIVE_OPTION)) { - PingPongThread = new System.Threading.Thread(PingPongMethod) { IsBackground = true }; + PingPongThread = new Thread(() => PingPongMethod(applicationSettings)) { IsBackground = true }; PingPongThread.Start(); } - DataConnection.ReWriteAllFieldsIfEncryptionChanged(); - DataConnection.SetPreloadSettingsIfChanged(preloadDbSettings); - EmitWarningsForConfigurationIssues(commandlineOptions); + connection.ReWriteAllFieldsIfEncryptionChanged(); + connection.SetPreloadSettingsIfChanged(preloadDbSettings); + EmitWarningsForConfigurationIssues(connection, applicationSettings, commandlineOptions); Log.WriteInformationMessage(LOGTAG, "ServerStarted", Strings.Program.ServerStarted(DuplicatiWebserver.Interface, DuplicatiWebserver.Port)); logMessageToConsole(Strings.Program.ServerStarted(DuplicatiWebserver.Interface, DuplicatiWebserver.Port)); - if (FIXMEGlobal.Origin == "Server" && DataConnection.ApplicationSettings.AutogeneratedPassphrase) + if (applicationSettings.Origin == "Server" && connection.ApplicationSettings.AutogeneratedPassphrase) { var signinToken = DuplicatiWebserver.Provider.GetRequiredService().CreateSigninToken("server-cli"); - var hostname = (DataConnection.ApplicationSettings.AllowedHostnames ?? string.Empty).Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries).FirstOrDefault(x => x != "*") ?? "localhost"; - var scheme = DataConnection.ApplicationSettings.UseHTTPS ? "https" : "http"; + var hostname = (connection.ApplicationSettings.AllowedHostnames ?? string.Empty).Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries).FirstOrDefault(x => x != "*") ?? "localhost"; + var scheme = connection.ApplicationSettings.UseHTTPS ? "https" : "http"; var url = $"{scheme}://{hostname}:{DuplicatiWebserver.Port}/signin.html?token={signinToken}"; Log.WriteWarningMessage(LOGTAG, "ServerStartedSignin", null, Strings.Program.ServerStartedSignin(url)); @@ -322,7 +260,7 @@ namespace Duplicati.Server } terminated = true; - ApplicationExitEvent.Set(); + applicationSettings.ApplicationExitEvent.Set(); }); var stopCounter = 0; @@ -344,7 +282,7 @@ namespace Duplicati.Server }; ServerStartedEvent.Set(); - ApplicationExitEvent.WaitOne(); + applicationSettings.ApplicationExitEvent.WaitOne(); } catch (SingleInstance.MultipleInstanceException mex) { @@ -374,19 +312,19 @@ namespace Duplicati.Server Log.WriteInformationMessage(LOGTAG, "ServerStopping", Strings.Program.ServerStopping); var steps = new Action[] { - () => StatusEventNotifyer.SignalNewEvent(), - () => { if (ShutdownModernWebserver != null) ShutdownModernWebserver(); }, - () => UpdatePoller?.Terminate(), - () => Scheduler?.Terminate(true), - () => FIXMEGlobal.WorkThread?.Terminate(true), + () => eventPollNotify?.SignalNewEvent(), + () => ShutdownModernWebserver(), + () => updatePollThread?.Terminate(), + () => scheduler?.Terminate(true), + () => queueRunner?.Terminate(true), () => ApplicationInstance?.Dispose(), () => PurgeTempFilesTimer?.Dispose(), () => Library.UsageReporter.Reporter.ShutDown(), () => PingPongThread?.Interrupt(), () => { - Library.Logging.Log.WriteInformationMessage(LOGTAG, "ServerStopped", Strings.Program.ServerStopped); - LogHandler?.Dispose(); + Log.WriteInformationMessage(LOGTAG, "ServerStopped", Strings.Program.ServerStopped); + logHandler?.Dispose(); } }; @@ -412,7 +350,7 @@ namespace Duplicati.Server return 0; } - private static async Task StartWebServer(IReadOnlyDictionary options, Connection connection) + private static async Task StartWebServer(IReadOnlyDictionary options, Connection connection, ILogWriteHandler logWriteHandler, IApplicationSettings applicationSettings) { var server = await WebServerLoader.TryRunServer(options, connection, async parsedOptions => { @@ -421,7 +359,6 @@ namespace Duplicati.Server parsedOptions.Port, parsedOptions.Interface, parsedOptions.Certificate, - parsedOptions.Servername, parsedOptions.AllowedHostnames, parsedOptions.DisableStaticFiles, parsedOptions.SPAPaths, @@ -429,7 +366,7 @@ namespace Duplicati.Server parsedOptions.PreAuthTokens ); - var server = DuplicatiWebserver.CreateWebServer(mappedSettings, connection); + var server = DuplicatiWebserver.CreateWebServer(mappedSettings, connection, logWriteHandler, applicationSettings); // Start the server, but catch any configuration issues var task = server.Start(); @@ -440,57 +377,13 @@ namespace Duplicati.Server return server; }).ConfigureAwait(false); - FIXMEGlobal.Provider = server.Provider; - ServerPortChanged |= server.Port != DataConnection.ApplicationSettings.LastWebserverPort; - DataConnection.ApplicationSettings.LastWebserverPort = server.Port; + connection.ApplicationSettings.ServerPortChanged |= server.Port != connection.ApplicationSettings.LastWebserverPort; + connection.ApplicationSettings.LastWebserverPort = server.Port; return server; } - private static void SetWorkerThread() - { - FIXMEGlobal.WorkerThreadsManager.Spawn(x => { Runner.Run(x, true); }); - FIXMEGlobal.WorkThread.StartingWork += (worker, task) => { SignalNewEvent(null, null); }; - FIXMEGlobal.WorkThread.CompletedWork += (worker, task) => { SignalNewEvent(null, null); }; - FIXMEGlobal.WorkThread.WorkQueueChanged += (worker) => { SignalNewEvent(null, null); }; - FIXMEGlobal.Scheduler.SubScribeToNewSchedule(() => SignalNewEvent(null, null)); - FIXMEGlobal.WorkThread.OnError += (worker, task, exception) => - { - DataConnection.LogError(task?.BackupID, "Error in worker", exception); - }; - - var lastScheduleId = FIXMEGlobal.NotificationUpdateService.LastDataUpdateId; - StatusEventNotifyer.NewEvent += (sender, e) => - { - if (lastScheduleId == FIXMEGlobal.NotificationUpdateService.LastDataUpdateId) return; - lastScheduleId = FIXMEGlobal.NotificationUpdateService.LastDataUpdateId; - Scheduler.Reschedule(); - }; - - void RegisterTaskResult(long id, Exception ex) - { - lock (MainLock) - { - // If the new results says it crashed, we store that instead of success - if (TaskResultCache.Count > 0 && TaskResultCache.Last().Key == id) - { - if (ex != null && TaskResultCache.Last().Value == null) - TaskResultCache.RemoveAt(TaskResultCache.Count - 1); - else - return; - } - - TaskResultCache.Add(new KeyValuePair(id, ex)); - while (TaskResultCache.Count > MAX_TASK_RESULT_CACHE_SIZE) - TaskResultCache.RemoveAt(0); - } - } - - FIXMEGlobal.WorkThread.CompletedWork += (worker, task) => { RegisterTaskResult(task.TaskID, null); }; - FIXMEGlobal.WorkThread.OnError += (worker, task, exception) => { RegisterTaskResult(task.TaskID, exception); }; - } - - private static void SetPurgeTempFilesTimer(Dictionary commandlineOptions) + private static void SetPurgeTempFilesTimer(Connection connection, Dictionary commandlineOptions) { var lastPurge = new DateTime(0); @@ -505,7 +398,7 @@ namespace Duplicati.Server lastPurge = DateTime.Now; - foreach (var e in DataConnection.GetTempFiles().Where((f) => f.Expires < DateTime.Now)) + foreach (var e in connection.GetTempFiles().Where((f) => f.Expires < DateTime.Now)) { try { @@ -514,16 +407,16 @@ namespace Duplicati.Server } catch (Exception ex) { - DataConnection.LogError(null, $"Failed to delete temp file: {e.Path}", ex); + connection.LogError(null, $"Failed to delete temp file: {e.Path}", ex); } - DataConnection.DeleteTempFile(e.ID); + connection.DeleteTempFile(e.ID); } Library.Utility.TempFile.RemoveOldApplicationTempFiles((path, ex) => { - DataConnection.LogError(null, $"Failed to delete temp file: {path}", ex); + connection.LogError(null, $"Failed to delete temp file: {path}", ex); }); if (!commandlineOptions.TryGetValue("log-retention", out string pts)) @@ -531,11 +424,11 @@ namespace Duplicati.Server pts = DEFAULT_LOG_RETENTION; } - DataConnection.PurgeLogData(Library.Utility.Timeparser.ParseTimeInterval(pts, DateTime.Now, true)); + connection.PurgeLogData(Library.Utility.Timeparser.ParseTimeInterval(pts, DateTime.Now, true)); } catch (Exception ex) { - DataConnection.LogError(null, "Failed during temp file cleanup", ex); + connection.LogError(null, "Failed during temp file cleanup", ex); } }; @@ -545,37 +438,37 @@ namespace Duplicati.Server DEBUG_MODE ? TimeSpan.FromHours(1) : TimeSpan.FromDays(1)); } - private static void AdjustApplicationSettings(Dictionary commandlineOptions) + private static void AdjustApplicationSettings(Connection connection, Dictionary commandlineOptions) { // This clears the JWT config, and a new will be generated, invalidating all existing tokens if (Library.Utility.Utility.ParseBoolOption(commandlineOptions, WebServerLoader.OPTION_WEBSERVICE_RESET_JWT_CONFIG)) { - DataConnection.ApplicationSettings.JWTConfig = null; + connection.ApplicationSettings.JWTConfig = null; // Clean up stored tokens as they are now invalid - DataConnection.ExecuteWithCommand((con) => con.ExecuteNonQuery("DELETE FROM TokenFamily")); + connection.ExecuteWithCommand((con) => con.ExecuteNonQuery("DELETE FROM TokenFamily")); } if (Library.Utility.Utility.ParseBoolOption(commandlineOptions, WebServerLoader.OPTION_WEBSERVICE_ENABLE_FOREVER_TOKEN)) - DataConnection.ApplicationSettings.EnableForeverTokens(); + connection.ApplicationSettings.EnableForeverTokens(); if (commandlineOptions.ContainsKey(WebServerLoader.OPTION_WEBSERVICE_DISABLE_VISUAL_CAPTCHA)) - DataConnection.ApplicationSettings.DisableVisualCaptcha = Library.Utility.Utility.ParseBool(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_DISABLE_VISUAL_CAPTCHA], true); + connection.ApplicationSettings.DisableVisualCaptcha = Library.Utility.Utility.ParseBool(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_DISABLE_VISUAL_CAPTCHA], true); if (commandlineOptions.ContainsKey(WebServerLoader.OPTION_WEBSERVICE_DISABLE_SIGNIN_TOKENS)) - DataConnection.ApplicationSettings.DisableSigninTokens = Library.Utility.Utility.ParseBool(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_DISABLE_SIGNIN_TOKENS], true); + connection.ApplicationSettings.DisableSigninTokens = Library.Utility.Utility.ParseBool(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_DISABLE_SIGNIN_TOKENS], true); if (commandlineOptions.ContainsKey(WebServerLoader.OPTION_WEBSERVICE_PASSWORD)) - DataConnection.ApplicationSettings.SetWebserverPassword(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_PASSWORD]); + connection.ApplicationSettings.SetWebserverPassword(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_PASSWORD]); if (commandlineOptions.ContainsKey(WebServerLoader.OPTION_WEBSERVICE_ALLOWEDHOSTNAMES)) - DataConnection.ApplicationSettings.SetAllowedHostnames(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_ALLOWEDHOSTNAMES]); + connection.ApplicationSettings.SetAllowedHostnames(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_ALLOWEDHOSTNAMES]); else if (commandlineOptions.ContainsKey(WebServerLoader.OPTION_WEBSERVICE_ALLOWEDHOSTNAMES_ALT)) - DataConnection.ApplicationSettings.SetAllowedHostnames(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_ALLOWEDHOSTNAMES_ALT]); + connection.ApplicationSettings.SetAllowedHostnames(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_ALLOWEDHOSTNAMES_ALT]); if (commandlineOptions.ContainsKey(WebServerLoader.OPTION_WEBSERVICE_TIMEZONE) && !string.IsNullOrEmpty(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_TIMEZONE])) try { - DataConnection.ApplicationSettings.Timezone = TimeZoneHelper.FindTimeZone(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_TIMEZONE]); + connection.ApplicationSettings.Timezone = TimeZoneHelper.FindTimeZone(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_TIMEZONE]); } catch (Exception ex) { @@ -583,33 +476,33 @@ namespace Duplicati.Server } // The database has recorded a new version - if (DataConnection.ApplicationSettings.UpdatedVersion != null) + if (connection.ApplicationSettings.UpdatedVersion != null) { // Check if the running version is newer than the recorded version - if (UpdaterManager.TryParseVersion(DataConnection.ApplicationSettings.UpdatedVersion.Version) <= UpdaterManager.TryParseVersion(UpdaterManager.SelfVersion.Version)) + if (UpdaterManager.TryParseVersion(connection.ApplicationSettings.UpdatedVersion.Version) <= UpdaterManager.TryParseVersion(UpdaterManager.SelfVersion.Version)) { // Clean up lingering update notifications - var updateNotifications = DataConnection.GetNotifications().Where(x => x.Action == "update:new").ToList(); + var updateNotifications = connection.GetNotifications().Where(x => x.Action == "update:new").ToList(); foreach (var n in updateNotifications) - DataConnection.DismissNotification(n.ID); + connection.DismissNotification(n.ID); // Clear up the recorded version - DataConnection.ApplicationSettings.UpdatedVersion = null; + connection.ApplicationSettings.UpdatedVersion = null; } } } - private static void EmitWarningsForConfigurationIssues(Dictionary commandlineOptions) + private static void EmitWarningsForConfigurationIssues(Connection connection, IApplicationSettings applicationSettings, Dictionary commandlineOptions) { - if (DataConnection.ApplicationSettings.LastConfigIssueCheckVersion != UpdaterManager.SelfVersion.Version) + if (connection.ApplicationSettings.LastConfigIssueCheckVersion != UpdaterManager.SelfVersion.Version) { - var updateNotifications = DataConnection.GetNotifications().Where(x => x.Action.StartsWith("config:issue:")).ToList(); + var updateNotifications = connection.GetNotifications().Where(x => x.Action.StartsWith("config:issue:")).ToList(); foreach (var n in updateNotifications) - DataConnection.DismissNotification(n.ID); + connection.DismissNotification(n.ID); - if (!DataConnection.IsEncryptingFields && !Library.Utility.Utility.ParseBoolOption(commandlineOptions, DISABLE_DB_ENCRYPTION_OPTION)) + if (!connection.IsEncryptingFields && !Library.Utility.Utility.ParseBoolOption(commandlineOptions, DISABLE_DB_ENCRYPTION_OPTION)) { - DataConnection.RegisterNotification( + connection.RegisterNotification( Serialization.NotificationType.Warning, "Unencrypted database", "The database is not encrypted. This is a security risk and should be fixed as soon as possible.", @@ -626,9 +519,9 @@ namespace Duplicati.Server ); } - if (OperatingSystem.IsWindows() && DataFolder.StartsWith(Util.AppendDirSeparator(Environment.GetFolderPath(Environment.SpecialFolder.Windows)), StringComparison.OrdinalIgnoreCase)) + if (OperatingSystem.IsWindows() && applicationSettings.DataFolder.StartsWith(Util.AppendDirSeparator(Environment.GetFolderPath(Environment.SpecialFolder.Windows)), StringComparison.OrdinalIgnoreCase)) { - DataConnection.RegisterNotification( + connection.RegisterNotification( Serialization.NotificationType.Warning, "Incorrect storage folder", "The server configuraion is stored inside the Windows folder. Please move the configuration to a different location, or it may be deleted on Windows version upgrades.", @@ -645,16 +538,16 @@ namespace Duplicati.Server ); } - DataConnection.ApplicationSettings.LastConfigIssueCheckVersion = UpdaterManager.SelfVersion.Version; + connection.ApplicationSettings.LastConfigIssueCheckVersion = UpdaterManager.SelfVersion.Version; } } - private static void CreateApplicationInstance(bool writeToConsoleOnExceptionw) + private static void CreateApplicationInstance(string dataFolder, bool writeToConsoleOnExceptionw) { try { //This will also create DATAFOLDER if it does not exist - ApplicationInstance = new SingleInstance(DataFolder); + ApplicationInstance = new SingleInstance(dataFolder); } catch (Exception ex) { @@ -698,8 +591,8 @@ namespace Duplicati.Server commandlineOptions[SETTINGS_ENCRYPTION_KEY_OPTION] = Environment.GetEnvironmentVariable(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME); } - private static async Task ApplySecretProvider(Dictionary commandlineOptions, CancellationToken cancellationToken) - => FIXMEGlobal.SecretProvider = await SecretProviderHelper.ApplySecretProviderAsync([], [], commandlineOptions, TempFolder.SystemTempPath, FIXMEGlobal.SecretProvider, cancellationToken).ConfigureAwait(false); + private static async Task ApplySecretProvider(IApplicationSettings applicationSettings, Dictionary commandlineOptions, CancellationToken cancellationToken) + => applicationSettings.SecretProvider = await SecretProviderHelper.ApplySecretProviderAsync([], [], commandlineOptions, TempFolder.SystemTempPath, applicationSettings.SecretProvider, cancellationToken).ConfigureAwait(false); private class ConsoleLogDestination(LogMessageType level) : ILogDestination { @@ -710,8 +603,9 @@ namespace Duplicati.Server } } - private static void ConfigureLogging(Dictionary commandlineOptions) + private static IDisposable ConfigureLogging(ILogWriteHandler logWriteHandler, Dictionary commandlineOptions) { + IDisposable logScope; //Log various information in the logfile if (DEBUG_MODE && !commandlineOptions.ContainsKey(LOG_FILE_OPTION)) { @@ -722,7 +616,7 @@ namespace Duplicati.Server System.IO.File.Delete(commandlineOptions[LOG_FILE_OPTION]); } - Log.StartScope(LogHandler, null); + logScope = Log.StartScope(logWriteHandler, null); if (commandlineOptions.ContainsKey(LOG_FILE_OPTION)) { @@ -730,7 +624,7 @@ namespace Duplicati.Server if (commandlineOptions.ContainsKey(LOG_LEVEL_OPTION)) Enum.TryParse(commandlineOptions[LOG_LEVEL_OPTION], true, out loglevel); - LogHandler.SetServerFile(commandlineOptions[LOG_FILE_OPTION], loglevel); + logWriteHandler.SetServerFile(commandlineOptions[LOG_FILE_OPTION], loglevel); } if (Library.Utility.Utility.ParseBoolOption(commandlineOptions, LOG_CONSOLE_OPTION)) @@ -739,7 +633,7 @@ namespace Duplicati.Server if (commandlineOptions.ContainsKey(LOG_LEVEL_OPTION)) Enum.TryParse(commandlineOptions[LOG_LEVEL_OPTION], true, out loglevel); - LogHandler.AppendLogDestination(new ConsoleLogDestination(loglevel), loglevel); + logWriteHandler.AppendLogDestination(new ConsoleLogDestination(loglevel), loglevel); } if (commandlineOptions.TryGetValue(WINDOWS_EVENTLOG_OPTION, out var source) && !string.IsNullOrEmpty(source)) @@ -765,16 +659,17 @@ namespace Duplicati.Server if (WindowsEventLogSource.SourceExists(source)) { - var loglevel = Library.Logging.LogMessageType.Information; + var loglevel = LogMessageType.Information; if (commandlineOptions.ContainsKey(WINDOWS_EVENTLOG_LEVEL_OPTION)) Enum.TryParse(commandlineOptions[WINDOWS_EVENTLOG_LEVEL_OPTION], true, out loglevel); - LogHandler.AppendLogDestination(new WindowsEventLogSource(source), loglevel); + logWriteHandler.AppendLogDestination(new WindowsEventLogSource(source), loglevel); } } } - CrashlogHelper.OnUnobservedTaskException += (ex) => LogHandler.WriteMessage(new Library.Logging.LogEntry(ex.Message, null, Library.Logging.LogMessageType.Error, LOGTAG, "UnobservedTaskException", ex)); + CrashlogHelper.OnUnobservedTaskException += (ex) => logWriteHandler.WriteMessage(new LogEntry(ex.Message, null, Library.Logging.LogMessageType.Error, LOGTAG, "UnobservedTaskException", ex)); + return logScope; } private static int ShowHelp(bool writeToConsoleOnExceptionw) @@ -792,15 +687,13 @@ namespace Duplicati.Server throw new Exception("Server invoked with --help"); } - public static Connection GetDatabaseConnection(Dictionary commandlineOptions, bool silentConsole) + public static Connection GetDatabaseConnection(IApplicationSettings applicationSettings, Dictionary commandlineOptions, bool silentConsole) { - DataFolder = DataFolderManager.GetDataFolder(DataFolderManager.AccessMode.ReadWritePermissionSet); - // Emit a warning if the database is stored in the Windows folder - if (Util.IsPathUnderWindowsFolder(DataFolder)) + if (Util.IsPathUnderWindowsFolder(applicationSettings.DataFolder)) Log.WriteWarningMessage(LOGTAG, "DatabaseInWindowsFolder", null, "The database is stored in the Windows folder, this is not recommended as it will be deleted on Windows upgrades."); - CrashlogHelper.DefaultLogDir = DataFolder; + CrashlogHelper.DefaultLogDir = applicationSettings.DataFolder; var sqliteVersion = new Version(Duplicati.Library.SQLiteHelper.SQLiteLoader.SQLiteVersion); if (sqliteVersion < new Version(3, 6, 3)) @@ -814,15 +707,15 @@ namespace Duplicati.Server try { - DatabasePath = System.IO.Path.Combine(DataFolder, DataFolderManager.SERVER_DATABASE_FILENAME); + var databasePath = System.IO.Path.Combine(applicationSettings.DataFolder, DataFolderManager.SERVER_DATABASE_FILENAME); - if (!System.IO.Directory.Exists(System.IO.Path.GetDirectoryName(DatabasePath))) - System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(DatabasePath)); + if (!System.IO.Directory.Exists(System.IO.Path.GetDirectoryName(databasePath))) + System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(databasePath)); // Attempt to open the database, removing any encryption present - Library.SQLiteHelper.SQLiteLoader.OpenDatabase(con, DatabasePath, Library.SQLiteHelper.SQLiteRC4Decrypter.GetEncryptionPassword(commandlineOptions)); + Library.SQLiteHelper.SQLiteLoader.OpenDatabase(con, databasePath, Library.SQLiteHelper.SQLiteRC4Decrypter.GetEncryptionPassword(commandlineOptions)); - Library.SQLiteHelper.DatabaseUpgrader.UpgradeDatabase(con, DatabasePath, typeof(Library.RestAPI.Database.DatabaseSchemaMarker)); + Library.SQLiteHelper.DatabaseUpgrader.UpgradeDatabase(con, databasePath, typeof(Library.RestAPI.Database.DatabaseSchemaMarker)); } catch (Exception ex) { @@ -839,7 +732,7 @@ namespace Duplicati.Server var usingBlacklistedKey = encKey?.IsBlacklisted ?? false; var hasValidEncryptionKey = encKey != null; - FIXMEGlobal.SettingsEncryptionKeyProvidedExternally = hasValidEncryptionKey; + applicationSettings.SettingsEncryptionKeyProvidedExternally = hasValidEncryptionKey; if (requireDbEncryptionKey && !(hasValidEncryptionKey || disableDbEncryption)) throw new UserInformationException(Strings.Program.DatabaseEncryptionKeyRequired(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION), "RequireDbEncryptionKey"); @@ -856,7 +749,7 @@ namespace Duplicati.Server if (hasEncryptedFields) { - Library.Logging.Log.WriteWarningMessage(LOGTAG, "EncryptionKeyMissing", null, Strings.Program.EncryptionKeyMissing(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME)); + Log.WriteWarningMessage(LOGTAG, "EncryptionKeyMissing", null, Strings.Program.EncryptionKeyMissing(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME)); if (!silentConsole) Console.WriteLine(Strings.Program.EncryptionKeyMissing(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME)); } @@ -871,61 +764,55 @@ namespace Duplicati.Server if (!hasValidEncryptionKey && !disableDbEncryption) { disableDbEncryption = true; - Log.WriteWarningMessage(LOGTAG, "MissingEncryptionKey", null, Strings.Program.NoEncryptionKeySpecified(Library.Encryption.EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION)); + Log.WriteWarningMessage(LOGTAG, "MissingEncryptionKey", null, Strings.Program.NoEncryptionKeySpecified(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION)); if (!silentConsole) - Console.WriteLine(Strings.Program.NoEncryptionKeySpecified(Library.Encryption.EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION)); + Console.WriteLine(Strings.Program.NoEncryptionKeySpecified(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION)); } if (usingBlacklistedKey && !disableDbEncryption) { disableDbEncryption = true; - Log.WriteErrorMessage(LOGTAG, "BlacklistedEncryptionKey", null, Strings.Program.BlacklistedEncryptionKey(Library.Encryption.EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION)); + Log.WriteErrorMessage(LOGTAG, "BlacklistedEncryptionKey", null, Strings.Program.BlacklistedEncryptionKey(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION)); if (!silentConsole) - Console.WriteLine(Strings.Program.BlacklistedEncryptionKey(Library.Encryption.EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION)); + Console.WriteLine(Strings.Program.BlacklistedEncryptionKey(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION)); } - return new Connection(con, disableDbEncryption, encKey); + return new Connection(con, disableDbEncryption, encKey, applicationSettings.DataFolder, applicationSettings.StartOrStopUsageReporter); } - public static void StartOrStopUsageReporter() + private static void StartOrStopUsageReporter(Connection connection) { var disableUsageReporter = - string.Equals(DataConnection.ApplicationSettings.UsageReporterLevel, "none", StringComparison.OrdinalIgnoreCase) + string.Equals(connection.ApplicationSettings.UsageReporterLevel, "none", StringComparison.OrdinalIgnoreCase) || - string.Equals(DataConnection.ApplicationSettings.UsageReporterLevel, "disabled", StringComparison.OrdinalIgnoreCase); + string.Equals(connection.ApplicationSettings.UsageReporterLevel, "disabled", StringComparison.OrdinalIgnoreCase); - if (!Enum.TryParse(DataConnection.ApplicationSettings.UsageReporterLevel, true, out var reportLevel)) + if (!Enum.TryParse(connection.ApplicationSettings.UsageReporterLevel, true, out var reportLevel)) Library.UsageReporter.Reporter.SetReportLevel(null, disableUsageReporter); else Library.UsageReporter.Reporter.SetReportLevel(reportLevel, disableUsageReporter); } - private static void SignalNewEvent(object sender, EventArgs e) - { - StatusEventNotifyer.SignalNewEvent(); - } - /// /// This event handler updates the trayicon menu with the current state of the runner. /// /// - private static void LiveControl_StateChanged(LiveControls.LiveControlEvent e) + private static void LiveControl_StateChanged(IQueueRunnerService queueRunnerService, Connection connection, EventPollNotify eventPollNotify, LiveControls.LiveControlEvent e) { - var worker = FIXMEGlobal.WorkThread; - var appSettings = FIXMEGlobal.DataConnection.ApplicationSettings; + var appSettings = connection.ApplicationSettings; switch (e.State) { case LiveControls.LiveControlState.Paused: { - worker.Pause(); - worker.CurrentTask?.Pause(e.TransfersPaused); + queueRunnerService.Pause(); + queueRunnerService.GetCurrentTask()?.Pause(e.TransfersPaused); appSettings.PausedUntil = e.WaitTimeExpiration; break; } case LiveControls.LiveControlState.Running: { - worker.Resume(); - worker.CurrentTask?.Resume(); + queueRunnerService.Resume(); + queueRunnerService.GetCurrentTask()?.Resume(); appSettings.PausedUntil = null; break; } @@ -934,14 +821,15 @@ namespace Duplicati.Server break; } - StatusEventNotifyer.SignalNewEvent(); + eventPollNotify.SignalNewEvent(); } /// /// Simple method for tracking if the server has crashed /// - private static void PingPongMethod() + /// The application settings + private static void PingPongMethod(IApplicationSettings applicationSettings) { var rd = new System.IO.StreamReader(Console.OpenStandardInput()); var wr = new System.IO.StreamWriter(Console.OpenStandardOutput()); @@ -952,7 +840,7 @@ namespace Duplicati.Server { // TODO: All calls to ApplicationExitEvent and TrayIcon->Quit // should check if we are running something - ApplicationExitEvent.Set(); + applicationSettings.ApplicationExitEvent.Set(); } else { diff --git a/Duplicati/Server/WebServerLoader.cs b/Duplicati/Server/WebServerLoader.cs index cfb537d19..c043059ae 100644 --- a/Duplicati/Server/WebServerLoader.cs +++ b/Duplicati/Server/WebServerLoader.cs @@ -158,7 +158,6 @@ public static class WebServerLoader /// The listining port /// The listening interface /// SSL certificate, if any - /// The servername to report /// The allowed hostnames /// If static files should be disabled /// The paths to serve as SPAs @@ -168,7 +167,6 @@ public static class WebServerLoader int Port, System.Net.IPAddress Interface, X509Certificate2Collection? Certificate, - string Servername, IEnumerable AllowedHostnames, bool DisableStaticFiles, IEnumerable SPAPaths, @@ -264,7 +262,6 @@ public static class WebServerLoader -1, listenInterface, connection.ApplicationSettings.UseHTTPS ? connection.ApplicationSettings.ServerSSLCertificate : null, - string.Format("{0} v{1}", Library.AutoUpdater.AutoUpdateSettings.AppName, Library.AutoUpdater.UpdaterManager.SelfVersion.Version), (connection.ApplicationSettings.AllowedHostnames ?? string.Empty).Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries), Utility.ParseBoolOption(options, OPTION_WEBSERVICE_API_ONLY), spaPathsString.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries), diff --git a/Duplicati/UnitTest/ImportExportTests.cs b/Duplicati/UnitTest/ImportExportTests.cs index 50f6e49f4..1d61312f3 100644 --- a/Duplicati/UnitTest/ImportExportTests.cs +++ b/Duplicati/UnitTest/ImportExportTests.cs @@ -29,7 +29,7 @@ using Duplicati.Server.Database; using Duplicati.Server.Serializable; using Duplicati.Server.Serialization; using Duplicati.Server.Serialization.Interface; -using Microsoft.Extensions.DependencyInjection; +using Duplicati.WebserverCore.Services; using NUnit.Framework; using Backup = Duplicati.Server.Database.Backup; @@ -107,10 +107,8 @@ namespace Duplicati.UnitTest } byte[] jsonByteArray; - using (Program.DataConnection = Program.GetDatabaseConnection(advancedOptions, true)) - { - jsonByteArray = BackupImportExportHandler.ExportToJSON(Program.DataConnection, backup, null); - } + using (var con = Program.GetDatabaseConnection(new ApplicationSettings(), advancedOptions, true)) + jsonByteArray = BackupImportExportHandler.ExportToJSON(con, backup, null); // The username should not have the '%40' converted to '@' since the import code // cannot handle it (see issue #3619). @@ -130,13 +128,6 @@ namespace Duplicati.UnitTest var metadata = new Dictionary { { "SourceFilesCount", "1" } }; var advancedOptions = new Dictionary { { "server-datafolder", this.serverDatafolder } }; - // Mock the setup enough to get the import/export working - var serviceCollection = new ServiceCollection(); - serviceCollection.AddSingleton(); - serviceCollection.AddSingleton(new EventPollNotify()); - FIXMEGlobal.Provider = new DefaultServiceProviderFactory().CreateServiceProvider(serviceCollection); - FIXMEGlobal.DataFolder = this.serverDatafolder; - var dbpath = Path.Combine(this.serverDatafolder, Library.AutoUpdater.DataFolderManager.SERVER_DATABASE_FILENAME); if (File.Exists(dbpath)) File.Delete(dbpath); @@ -145,7 +136,7 @@ namespace Duplicati.UnitTest SQLiteLoader.OpenDatabase(con, dbpath, null); DatabaseUpgrader.UpgradeDatabase(con, dbpath, typeof(Library.RestAPI.Database.DatabaseSchemaMarker)); - using (var connection = new Connection(con, true, null)) + using (var connection = new Connection(con, true, null, this.serverDatafolder, null)) { // Unencrypted file, don't import metadata. string unencryptedWithoutMetadata = Path.Combine(this.serverDatafolder, Path.GetRandomFileName()); diff --git a/Duplicati/WebserverCore/Dto/GetTaskStateDto.cs b/Duplicati/WebserverCore/Dto/GetTaskStateDto.cs index 3a29ecb06..016aa49f8 100644 --- a/Duplicati/WebserverCore/Dto/GetTaskStateDto.cs +++ b/Duplicati/WebserverCore/Dto/GetTaskStateDto.cs @@ -25,6 +25,8 @@ namespace Duplicati.WebserverCore.Dto; /// /// The status /// The ID +/// When the task started +/// When the task finished /// The error message /// The exception -public record GetTaskStateDto(string Status, long ID, string? ErrorMessage = null, string? Exception = null); +public record GetTaskStateDto(string Status, long ID, DateTime? TaskStarted, DateTime? TaskFinished, string? ErrorMessage = null, string? Exception = null); diff --git a/Duplicati/WebserverCore/Dto/LogEntry.cs b/Duplicati/WebserverCore/Dto/LogEntry.cs index 0affbfeb9..1e19fab8b 100644 --- a/Duplicati/WebserverCore/Dto/LogEntry.cs +++ b/Duplicati/WebserverCore/Dto/LogEntry.cs @@ -19,7 +19,7 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Logging; -using Duplicati.Server; +using Duplicati.WebserverCore.Abstractions; namespace Duplicati.WebserverCore.Dto; @@ -56,7 +56,7 @@ public sealed record LogEntry /// /// The message ID /// - public required string ExceptionID { get; init; } + public required string? ExceptionID { get; init; } /// /// The message type @@ -71,7 +71,7 @@ public sealed record LogEntry /// /// The backup ID, if any /// - public required string BackupID { get; init; } + public required string? BackupID { get; init; } /// /// The task ID, if any @@ -83,7 +83,7 @@ public sealed record LogEntry /// /// The internal record /// The DTO record - public static LogEntry FromInternalEntry(LogWriteHandler.LogEntry entry) + public static LogEntry FromInternalEntry(ILogWriteHandler.LiveLogEntry entry) { return new LogEntry { diff --git a/Duplicati/WebserverCore/Dto/ServerStatusDto.cs b/Duplicati/WebserverCore/Dto/ServerStatusDto.cs index 8426d0675..97b09011a 100644 --- a/Duplicati/WebserverCore/Dto/ServerStatusDto.cs +++ b/Duplicati/WebserverCore/Dto/ServerStatusDto.cs @@ -30,7 +30,7 @@ public sealed record ServerStatusDto /// /// Gets or sets the active task. /// - public required Tuple? ActiveTask { get; init; } + public required Tuple? ActiveTask { get; init; } /// /// Gets or sets the state of the program. @@ -40,7 +40,7 @@ public sealed record ServerStatusDto /// /// Gets the IDs of the tasks in the scheduler queue. /// - public required IList> SchedulerQueueIds { get; init; } = []; + public required IList> SchedulerQueueIds { get; init; } = []; /// /// Gets or sets the proposed schedule. diff --git a/Duplicati/WebserverCore/DuplicatiWebserver.cs b/Duplicati/WebserverCore/DuplicatiWebserver.cs index 1482319f3..ccaf2bcf6 100644 --- a/Duplicati/WebserverCore/DuplicatiWebserver.cs +++ b/Duplicati/WebserverCore/DuplicatiWebserver.cs @@ -108,7 +108,6 @@ public class DuplicatiWebserver /// The listining port /// The listening interface /// The certificate, if using SSL - /// The servername to report /// The allowed hostnames /// If static files should be disabled /// The paths to serve as SPAs @@ -119,7 +118,6 @@ public class DuplicatiWebserver int Port, System.Net.IPAddress Interface, X509Certificate2Collection? Certificate, - string Servername, IEnumerable AllowedHostnames, bool DisableStaticFiles, IEnumerable SPAPaths, @@ -132,8 +130,10 @@ public class DuplicatiWebserver /// /// The settings for the server /// The connection to the database + /// The log write handler + /// The application settings /// The new webserver instance - public static DuplicatiWebserver CreateWebServer(InitSettings settings, Connection connection) + public static DuplicatiWebserver CreateWebServer(InitSettings settings, Connection connection, ILogWriteHandler logWriteHandler, IApplicationSettings applicationSettings) { var builder = WebApplication.CreateBuilder(new WebApplicationOptions() { @@ -279,7 +279,7 @@ public class DuplicatiWebserver builder.Services.AddHealthChecks() .AddCheck("Basic", () => HealthCheckResult.Healthy("Service is running")); - builder.Services.AddDuplicati(connection); + builder.Services.AddDuplicati(connection, logWriteHandler, applicationSettings); // Prevent logs from spamming the console, but allow enabling for debugging if (Environment.GetEnvironmentVariable("DUPLICATI_WEBSERVER_LOGGING") != "1") @@ -331,6 +331,13 @@ public class DuplicatiWebserver if (!settings.DisableStaticFiles) app.UseDefaultStaticFiles(settings.WebRoot, settings.SPAPaths); + app.Use(async (context, next) => + { + // Set up the log scope to capture any log messages for the livelog / server log + using (var scope = Library.Logging.Log.StartScope(logWriteHandler)) + await next(); + }); + app.UseExceptionHandler(app => { app.Run(async context => @@ -387,9 +394,9 @@ public class DuplicatiWebserver var _ = Task.Run(() => app.Services.GetRequiredService().GetSystemInfo(null)); // Get a string description of the listen interface used - var listenInterface = settings.Interface == System.Net.IPAddress.Any + var listenInterface = settings.Interface == IPAddress.Any ? "*" - : settings.Interface == System.Net.IPAddress.Loopback + : settings.Interface == IPAddress.Loopback ? "localhost" : settings.Interface.ToString(); diff --git a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs index 038a1a429..413226cfe 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs @@ -21,7 +21,7 @@ using System.Text.Json; using System.Text.Json.Serialization; using Duplicati.Library.Interface; -using Duplicati.Library.RestAPI.Abstractions; +using Duplicati.Library.RestAPI; using Duplicati.Server; using Duplicati.Server.Database; using Duplicati.Server.Serialization; @@ -42,8 +42,8 @@ public class BackupGet : IEndpointV1 => ExecuteGet(connection, GetBackup(connection, id))) .RequireAuthorization(); - group.MapGet("/backup/{id}/files", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery] string? filter, [FromQuery] string? time, [FromQuery(Name = "all-versions")] bool? allVersions, [FromQuery(Name = "prefix-only")] bool? prefixOnly, [FromQuery(Name = "folder-contents")] bool? folderContents) - => ExecuteGetFiles(GetBackup(connection, id), filter, time, allVersions ?? false, prefixOnly ?? false, folderContents ?? false, new Dictionary())) + group.MapGet("/backup/{id}/files", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id, [FromQuery] string? filter, [FromQuery] string? time, [FromQuery(Name = "all-versions")] bool? allVersions, [FromQuery(Name = "prefix-only")] bool? prefixOnly, [FromQuery(Name = "folder-contents")] bool? folderContents) + => ExecuteGetFiles(queueRunnerService, GetBackup(connection, id), filter, time, allVersions ?? false, prefixOnly ?? false, folderContents ?? false, new Dictionary())) .RequireAuthorization(); group.MapGet("/backup/{id}/log", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery] long? offset, [FromQuery] long? pagesize) @@ -54,16 +54,16 @@ public class BackupGet : IEndpointV1 => ExecuteGetRemotelog(connection, GetBackup(connection, id), offset, pagesize ?? 100)) .RequireAuthorization(); - group.MapGet("/backup/{id}/filesets", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery(Name = "include-metadata")] bool? includeMetadata, [FromQuery(Name = "from-remote-only")] bool? fromRemoteOnly) - => ExecuteGetFilesets(GetBackup(connection, id), includeMetadata ?? false, fromRemoteOnly ?? false)) + group.MapGet("/backup/{id}/filesets", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id, [FromQuery(Name = "include-metadata")] bool? includeMetadata, [FromQuery(Name = "from-remote-only")] bool? fromRemoteOnly) + => ExecuteGetFilesets(queueRunnerService, GetBackup(connection, id), includeMetadata ?? false, fromRemoteOnly ?? false)) .RequireAuthorization(); group.MapGet("/backup/{id}/export-argsonly", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery(Name = "export-passwords")] bool? exportPasswords, [FromQuery] string? passphrase) - => ExecuteGetExportArgsOnly(GetBackup(connection, id), exportPasswords ?? false)) + => ExecuteGetExportArgsOnly(connection, GetBackup(connection, id), exportPasswords ?? false)) .RequireAuthorization(); group.MapGet("/backup/{id}/export-cmdline", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery(Name = "export-passwords")] bool? exportPasswords, [FromQuery] string? passphrase) - => ExecuteGetExportCmdline(GetBackup(connection, id), exportPasswords ?? false)) + => ExecuteGetExportCmdline(connection, GetBackup(connection, id), exportPasswords ?? false)) .RequireAuthorization(); group.MapGet("/backup/{id}/export", ([FromServices] Connection connection, [FromServices] IHttpContextAccessor httpContextAccessor, [FromServices] IJWTTokenProvider jWTTokenProvider, [FromRoute] string id, [FromQuery(Name = "export-passwords")] bool? exportPasswords, [FromQuery] string? passphrase, [FromQuery] string token, CancellationToken ct) => @@ -86,8 +86,8 @@ public class BackupGet : IEndpointV1 => ExecuteGetIsdbUsedElsewhere(GetBackup(connection, id))) .RequireAuthorization(); - group.MapGet("/backup/{id}/isactive", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id) - => ExecuteGetIsActive(workerThreadsManager, GetBackup(connection, id))) + group.MapGet("/backup/{id}/isactive", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id) + => ExecuteGetIsActive(queueRunnerService, GetBackup(connection, id))) .RequireAuthorization(); } @@ -143,7 +143,7 @@ public class BackupGet : IEndpointV1 ); } - private static Dictionary SearchFiles(IBackup backup, string? filter, string? timestring, bool allVersions, bool prefixOnly, bool folderContents, Dictionary extraValues) + private static Dictionary SearchFiles(IQueueRunnerService queueRunnerService, IBackup backup, string? filter, string? timestring, bool allVersions, bool prefixOnly, bool folderContents, Dictionary extraValues) { if (string.IsNullOrWhiteSpace(timestring) && !allVersions) throw new BadRequestException("Invalid or missing time"); @@ -152,7 +152,7 @@ public class BackupGet : IEndpointV1 if (!allVersions) time = Library.Utility.Timeparser.ParseTimeInterval(timestring, DateTime.Now); - var r = Runner.Run(Runner.CreateListTask(backup, [filter], prefixOnly, allVersions, folderContents, time), false) as Duplicati.Library.Interface.IListResults; + var r = queueRunnerService.RunImmediately(Runner.CreateListTask(backup, filter == null ? null : [filter], prefixOnly, allVersions, folderContents, time)) as IListResults; if (r == null) throw new ServerErrorException("No result from list operation"); @@ -171,8 +171,8 @@ public class BackupGet : IEndpointV1 return result; } - private static Dictionary ExecuteGetFiles(IBackup bk, string? filter, string? timestring, bool allVersions, bool prefixOnly, bool folderContents, Dictionary extraValues) - => SearchFiles(bk, filter, timestring, allVersions, prefixOnly, folderContents, extraValues); + private static Dictionary ExecuteGetFiles(IQueueRunnerService queueRunnerService, IBackup bk, string? filter, string? timestring, bool allVersions, bool prefixOnly, bool folderContents, Dictionary extraValues) + => SearchFiles(queueRunnerService, bk, filter, timestring, allVersions, prefixOnly, folderContents, extraValues); private static List> ExecuteGetLog(Connection connection, IBackup bk, long? offset, long pagesize) { @@ -203,9 +203,9 @@ public class BackupGet : IEndpointV1 } } - private static IEnumerable ExecuteGetFilesets(IBackup bk, bool includeMetadata, bool fromRemoteOnly) + private static IEnumerable ExecuteGetFilesets(IQueueRunnerService queueRunnerService, IBackup bk, bool includeMetadata, bool fromRemoteOnly) { - var extra = new Dictionary + var extra = new Dictionary { ["list-sets-only"] = "true" }; @@ -219,7 +219,7 @@ public class BackupGet : IEndpointV1 try { - var r = Runner.Run(Runner.CreateTask(DuplicatiOperation.List, bk, extra), false) as IListResults; + var r = queueRunnerService.RunImmediately(Runner.CreateTask(DuplicatiOperation.List, bk, extra)) as IListResults; if (r == null) throw new ServerErrorException("No result from list operation"); @@ -240,20 +240,20 @@ public class BackupGet : IEndpointV1 backup.SanitizeTargetUrl(); } - private static Dto.ExportCommandlineDto ExecuteGetExportCmdline(IBackup backup, bool exportPasswords) + private static Dto.ExportCommandlineDto ExecuteGetExportCmdline(Connection connection, IBackup backup, bool exportPasswords) { if (!exportPasswords) RemovePasswords(backup); - return new Dto.ExportCommandlineDto(Runner.GetCommandLine(Runner.CreateTask(DuplicatiOperation.Backup, backup))); + return new Dto.ExportCommandlineDto(Runner.GetCommandLine(connection, Runner.CreateTask(DuplicatiOperation.Backup, backup))); } - private static Dto.ExportArgsOnlyDto ExecuteGetExportArgsOnly(IBackup backup, bool exportPasswords) + private static Dto.ExportArgsOnlyDto ExecuteGetExportArgsOnly(Connection connection, IBackup backup, bool exportPasswords) { if (!exportPasswords) RemovePasswords(backup); - var parts = Runner.GetCommandLineParts(Runner.CreateTask(DuplicatiOperation.Backup, backup)); + var parts = Runner.GetCommandLineParts(connection, Runner.CreateTask(DuplicatiOperation.Backup, backup)); return new Dto.ExportArgsOnlyDto( parts.First(), parts.Skip(1).Where(x => !x.StartsWith("--", StringComparison.Ordinal)), @@ -304,17 +304,14 @@ public class BackupGet : IEndpointV1 private static Dto.IsDbUsedElsewhereDto ExecuteGetIsdbUsedElsewhere(IBackup bk) => new Dto.IsDbUsedElsewhereDto(Library.Main.CLIDatabaseLocator.IsDatabasePathInUse(bk.DBPath)); - private static Dto.IsBackupActiveDto ExecuteGetIsActive(IWorkerThreadsManager workerThreadsManager, IBackup bk) + private static Dto.IsBackupActiveDto ExecuteGetIsActive(IQueueRunnerService queueRunnerService, IBackup bk) { - if (workerThreadsManager.WorkerThread == null) - throw new InvalidOperationException("Worker thread not available"); - - var t = workerThreadsManager.WorkerThread.CurrentTask; - var bt = t?.Backup; - if (bt != null && bk.ID == bt.ID) + var t = queueRunnerService.GetCurrentTask(); + var bt = t?.BackupID; + if (bt != null && bk.ID == bt) return new Dto.IsBackupActiveDto("OK", true); - if (workerThreadsManager.WorkerThread.CurrentTasks.Any(x => x?.Backup == null || x.Backup.ID == bk.ID)) + if (queueRunnerService.GetCurrentTasks().Any(x => x?.BackupID == null || x.BackupID == bk.ID)) return new Dto.IsBackupActiveDto("OK", true); return new Dto.IsBackupActiveDto("OK", false); diff --git a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupPost.cs b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupPost.cs index 8b58ff846..3b40ff06d 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupPost.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupPost.cs @@ -18,7 +18,6 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using Duplicati.Library.RestAPI.Abstractions; using Duplicati.Server; using Duplicati.Server.Database; using Duplicati.Server.Serialization; @@ -45,44 +44,44 @@ public class BackupPost : IEndpointV1 => UpdateDatabasePath(connection, GetBackup(connection, id), input.path, false)) .RequireAuthorization(); - group.MapPost("/backup/{id}/restore", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id, [FromBody] Dto.RestoreInputDto input) - => ExecuteRestore(GetBackup(connection, id), workerThreadsManager, input)) + group.MapPost("/backup/{id}/restore", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id, [FromBody] Dto.RestoreInputDto input) + => ExecuteRestore(GetBackup(connection, id), queueRunnerService, input)) .RequireAuthorization(); - group.MapPost("/backup/{id}/createreport", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id) - => ExecuteCreateReport(GetBackup(connection, id), workerThreadsManager)) + group.MapPost("/backup/{id}/createreport", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id) + => ExecuteCreateReport(GetBackup(connection, id), queueRunnerService)) .RequireAuthorization(); - group.MapPost("/backup/{id}/repair", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id, Dto.RepairInputDto? input) - => ExecuteRepair(GetBackup(connection, id), workerThreadsManager, input)) + group.MapPost("/backup/{id}/repair", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id, Dto.RepairInputDto? input) + => ExecuteRepair(GetBackup(connection, id), queueRunnerService, input)) .RequireAuthorization(); - group.MapPost("/backup/{id}/repairupdate", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id, Dto.RepairInputDto? input) - => ExecuteRepairUpdate(GetBackup(connection, id), workerThreadsManager, input)) + group.MapPost("/backup/{id}/repairupdate", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id, Dto.RepairInputDto? input) + => ExecuteRepairUpdate(GetBackup(connection, id), queueRunnerService, input)) .RequireAuthorization(); - group.MapPost("/backup/{id}/vacuum", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id) - => ExecuteVacuum(GetBackup(connection, id), workerThreadsManager)) + group.MapPost("/backup/{id}/vacuum", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id) + => ExecuteVacuum(GetBackup(connection, id), queueRunnerService)) .RequireAuthorization(); - group.MapPost("/backup/{id}/verify", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id) - => ExecuteVerify(GetBackup(connection, id), workerThreadsManager)) + group.MapPost("/backup/{id}/verify", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id) + => ExecuteVerify(GetBackup(connection, id), queueRunnerService)) .RequireAuthorization(); - group.MapPost("/backup/{id}/compact", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id) - => ExecuteCompact(GetBackup(connection, id), workerThreadsManager)) + group.MapPost("/backup/{id}/compact", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id) + => ExecuteCompact(GetBackup(connection, id), queueRunnerService)) .RequireAuthorization(); - group.MapPost("/backup/{id}/start", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id) - => ExecuteRunBackup(GetBackup(connection, id), workerThreadsManager)) + group.MapPost("/backup/{id}/start", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id) + => ExecuteRunBackup(GetBackup(connection, id), queueRunnerService)) .RequireAuthorization(); - group.MapPost("/backup/{id}/run", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id) - => ExecuteRunBackup(GetBackup(connection, id), workerThreadsManager)) + group.MapPost("/backup/{id}/run", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id) + => ExecuteRunBackup(GetBackup(connection, id), queueRunnerService)) .RequireAuthorization(); - group.MapPost("/backup/{id}/report-remote-size", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id) - => ExecuteReportRemoteSize(GetBackup(connection, id), workerThreadsManager)) + group.MapPost("/backup/{id}/report-remote-size", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id) + => ExecuteReportRemoteSize(GetBackup(connection, id), queueRunnerService)) .RequireAuthorization(); group.MapPost("/backup/{id}/copytotemp", ([FromServices] Connection connection, [FromRoute] string id) @@ -114,8 +113,8 @@ public class BackupPost : IEndpointV1 - private static Dto.TaskStartedDto ExecuteRestore(IBackup backup, IWorkerThreadsManager workerThreadsManager, Dto.RestoreInputDto input) - => new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateRestoreTask( + private static Dto.TaskStartedDto ExecuteRestore(IBackup backup, IQueueRunnerService queueRunnerService, Dto.RestoreInputDto input) + => new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateRestoreTask( backup, input.paths ?? [], Library.Utility.Timeparser.ParseTimeInterval(input.time, DateTime.Now), @@ -125,31 +124,31 @@ public class BackupPost : IEndpointV1 input.skip_metadata ?? false, string.IsNullOrWhiteSpace(input.passphrase) ? null : input.passphrase))); - private static Dto.TaskStartedDto ExecuteCreateReport(IBackup backup, IWorkerThreadsManager workerThreadsManager) - => new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateTask(DuplicatiOperation.CreateReport, backup))); + private static Dto.TaskStartedDto ExecuteCreateReport(IBackup backup, IQueueRunnerService queueRunnerService) + => new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateTask(DuplicatiOperation.CreateReport, backup))); - private static Dto.TaskStartedDto ExecuteReportRemoteSize(IBackup backup, IWorkerThreadsManager workerThreadsManager) - => new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateTask(DuplicatiOperation.ListRemote, backup))); + private static Dto.TaskStartedDto ExecuteReportRemoteSize(IBackup backup, IQueueRunnerService queueRunnerService) + => new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateTask(DuplicatiOperation.ListRemote, backup))); - private static Dto.TaskStartedDto ExecuteRepair(IBackup backup, IWorkerThreadsManager workerThreadsManager, Dto.RepairInputDto? input) - => DoRepair(backup, false, workerThreadsManager, input); + private static Dto.TaskStartedDto ExecuteRepair(IBackup backup, IQueueRunnerService queueRunnerService, Dto.RepairInputDto? input) + => DoRepair(backup, false, queueRunnerService, input); - private static Dto.TaskStartedDto ExecuteRepairUpdate(IBackup backup, IWorkerThreadsManager workerThreadsManager, Dto.RepairInputDto? input) - => DoRepair(backup, true, workerThreadsManager, input); + private static Dto.TaskStartedDto ExecuteRepairUpdate(IBackup backup, IQueueRunnerService queueRunnerService, Dto.RepairInputDto? input) + => DoRepair(backup, true, queueRunnerService, input); - private static Dto.TaskStartedDto ExecuteVacuum(IBackup backup, IWorkerThreadsManager workerThreadsManager) - => new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateTask(DuplicatiOperation.Vacuum, backup))); + private static Dto.TaskStartedDto ExecuteVacuum(IBackup backup, IQueueRunnerService queueRunnerService) + => new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateTask(DuplicatiOperation.Vacuum, backup))); - private static Dto.TaskStartedDto ExecuteVerify(IBackup backup, IWorkerThreadsManager workerThreadsManager) - => new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateTask(DuplicatiOperation.Verify, backup))); + private static Dto.TaskStartedDto ExecuteVerify(IBackup backup, IQueueRunnerService queueRunnerService) + => new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateTask(DuplicatiOperation.Verify, backup))); - private static Dto.TaskStartedDto ExecuteCompact(IBackup backup, IWorkerThreadsManager workerThreadsManager) - => new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateTask(DuplicatiOperation.Compact, backup))); + private static Dto.TaskStartedDto ExecuteCompact(IBackup backup, IQueueRunnerService queueRunnerService) + => new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateTask(DuplicatiOperation.Compact, backup))); - private static Dto.TaskStartedDto DoRepair(IBackup backup, bool repairUpdate, IWorkerThreadsManager workerThreadsManager, Dto.RepairInputDto? input) + private static Dto.TaskStartedDto DoRepair(IBackup backup, bool repairUpdate, IQueueRunnerService queueRunnerService, Dto.RepairInputDto? input) { // These are all props on the input object - var extra = new Dictionary(); + var extra = new Dictionary(); if (input != null) { if (input.only_paths.HasValue) @@ -168,23 +167,23 @@ public class BackupPost : IEndpointV1 var filters = input?.paths ?? []; - return new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateTask(repairUpdate ? DuplicatiOperation.RepairUpdate : DuplicatiOperation.Repair, backup, extra, filters))); + return new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateTask(repairUpdate ? DuplicatiOperation.RepairUpdate : DuplicatiOperation.Repair, backup, extra, filters))); } - private static Dto.TaskStartedDto ExecuteRunBackup(IBackup backup, IWorkerThreadsManager workerThreadsManager) + private static Dto.TaskStartedDto ExecuteRunBackup(IBackup backup, IQueueRunnerService queueRunnerService) { - var t = workerThreadsManager.WorkerThread?.CurrentTask; - var bt = t?.Backup; + var t = queueRunnerService.GetCurrentTask(); + var bt = t?.BackupID; // Already running - if (bt != null && backup.ID == bt.ID) + if (bt != null && backup.ID == bt) return new Dto.TaskStartedDto("OK", t!.TaskID); - t = workerThreadsManager.WorkerThread?.CurrentTasks.FirstOrDefault(x => x?.Backup != null && x.Backup.ID == backup.ID); + t = queueRunnerService.GetCurrentTasks().FirstOrDefault(x => x.BackupID == backup.ID); if (t != null) return new Dto.TaskStartedDto("OK", t.TaskID); - return new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateTask(DuplicatiOperation.Backup, backup), true)); + return new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateTask(DuplicatiOperation.Backup, backup), true)); } private class WrappedBackup : Server.Database.Backup diff --git a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupPutDelete.cs b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupPutDelete.cs index 8bb4ce4b4..77e763e42 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupPutDelete.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupPutDelete.cs @@ -18,8 +18,6 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System.Text.Json; -using Duplicati.Library.RestAPI.Abstractions; using Duplicati.Server; using Duplicati.Server.Database; using Duplicati.Server.Serialization; @@ -38,9 +36,9 @@ public class BackupPutDelete : IEndpointV1 => ExecutePut(GetBackup(connection, id), connection, input)) .RequireAuthorization(); - group.MapDelete("/backup/{id}", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromServices] LiveControls liveControls, [FromServices] IHttpContextAccessor httpContextAccessor, [FromRoute] string id, [FromQuery(Name = "delete-remote-files")] bool? delete_remote_files, [FromQuery(Name = "delete-local-db")] bool? delete_local_db, [FromQuery] bool? force) => + group.MapDelete("/backup/{id}", ([FromServices] Connection connection, [FromServices] ISchedulerService schedulerService, [FromServices] IQueueRunnerService queueRunnerService, [FromServices] LiveControls liveControls, [FromServices] IHttpContextAccessor httpContextAccessor, [FromRoute] string id, [FromQuery(Name = "delete-remote-files")] bool? delete_remote_files, [FromQuery(Name = "delete-local-db")] bool? delete_local_db, [FromQuery] bool? force) => { - var res = ExecuteDelete(GetBackup(connection, id), workerThreadsManager, liveControls, delete_remote_files ?? false, delete_local_db, force ?? false); + var res = ExecuteDelete(GetBackup(connection, id), schedulerService, queueRunnerService, liveControls, delete_remote_files ?? false, delete_local_db, force ?? false); if (res.Status != "OK" && httpContextAccessor.HttpContext != null) httpContextAccessor.HttpContext.Response.StatusCode = 500; return res; @@ -132,67 +130,56 @@ public class BackupPutDelete : IEndpointV1 } } - private static Dto.DeleteBackupOutputDto ExecuteDelete(IBackup backup, IWorkerThreadsManager workerThreadsManager, LiveControls liveControls, bool delete_remote_files, bool? delete_local_db, bool force) + private static Dto.DeleteBackupOutputDto ExecuteDelete(IBackup backup, ISchedulerService schedulerService, IQueueRunnerService queueRunnerService, LiveControls liveControls, bool delete_remote_files, bool? delete_local_db, bool force) { - if (workerThreadsManager.WorkerThread!.Active) + try { - try + var nt = queueRunnerService.GetCurrentTask(); + if (backup.ID == nt?.BackupID) { - //TODO: It's not safe to access the values like this, - //because the runner thread might interfere - var nt = workerThreadsManager.WorkerThread.CurrentTask; - if (backup.Equals(nt?.Backup)) + if (!force) + return new Dto.DeleteBackupOutputDto("failed", "backup-in-progress", nt?.TaskID); + + + bool hasPaused = liveControls.State != LiveControls.LiveControlState.Paused; + if (hasPaused) + liveControls.Pause(true); + nt.Abort(); + + for (int i = 0; i < 10; i++) { - if (!force) - return new Dto.DeleteBackupOutputDto("failed", "backup-in-progress", nt?.TaskID); - - - bool hasPaused = liveControls.State != LiveControls.LiveControlState.Paused; - if (hasPaused) - liveControls.Pause(true); - nt.Abort(); - - for (int i = 0; i < 10; i++) - if (workerThreadsManager.WorkerThread.Active) - { - var t = workerThreadsManager.WorkerThread.CurrentTask; - if (backup.Equals(t == null ? null : t.Backup)) - Thread.Sleep(1000); - else - break; - } - else - break; - - if (workerThreadsManager.WorkerThread.Active) - { - var t = workerThreadsManager.WorkerThread.CurrentTask; - if (backup.Equals(t == null ? null : t.Backup)) - { - if (hasPaused) - liveControls.Resume(); - - return new Dto.DeleteBackupOutputDto("failed", "backup-unstoppable", t?.TaskID); - } - } + var tt = queueRunnerService.GetCurrentTask(); + if (backup.ID == tt?.BackupID) + Thread.Sleep(1000); + else + break; + } + var t = queueRunnerService.GetCurrentTask(); + if (backup.ID == t?.BackupID) + { if (hasPaused) liveControls.Resume(); + + return new Dto.DeleteBackupOutputDto("failed", "backup-unstoppable", t?.TaskID); } - } - catch (Exception ex) - { - return new Dto.DeleteBackupOutputDto("error", ex.Message, null); + + if (hasPaused) + liveControls.Resume(); } } + catch (Exception ex) + { + return new Dto.DeleteBackupOutputDto("error", ex.Message, null); + } - var extra = new Dictionary(); + var extra = new Dictionary(); if (delete_local_db.HasValue) extra["delete-local-db"] = delete_local_db.Value.ToString(); if (delete_remote_files) extra["delete-remote-files"] = "true"; - return new Dto.DeleteBackupOutputDto("OK", null, workerThreadsManager.AddTask(Runner.CreateTask(DuplicatiOperation.Delete, backup, extra))); + return new Dto.DeleteBackupOutputDto("OK", null, queueRunnerService.AddTask(Runner.CreateDeleteTask(backup, extra, (_) => schedulerService.Reschedule()))); } } diff --git a/Duplicati/WebserverCore/Endpoints/V1/BackupDefaults.cs b/Duplicati/WebserverCore/Endpoints/V1/BackupDefaults.cs index 918eb7a46..85b1d162f 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/BackupDefaults.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/BackupDefaults.cs @@ -60,7 +60,7 @@ public class BackupDefaults : IEndpointV1 { // Add built-in defaults Newtonsoft.Json.Linq.JObject? n; - using (var s = new StreamReader(typeof(FIXMEGlobal).Assembly.GetManifestResourceStream(typeof(FIXMEGlobal), "newbackup.json")!)) + using (var s = new StreamReader(typeof(Server.Runner).Assembly.GetManifestResourceStream(typeof(Server.Runner), "newbackup.json")!)) n = (Newtonsoft.Json.Linq.JObject)Newtonsoft.Json.JsonConvert.DeserializeObject(s.ReadToEnd())!; o.Merge(n); diff --git a/Duplicati/WebserverCore/Endpoints/V1/LogData.cs b/Duplicati/WebserverCore/Endpoints/V1/LogData.cs index 13c39324a..69c4c929e 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/LogData.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/LogData.cs @@ -19,7 +19,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Crashlog; -using Duplicati.Library.RestAPI; using Duplicati.Server.Database; using Duplicati.WebserverCore.Abstractions; using Duplicati.WebserverCore.Dto; @@ -31,8 +30,8 @@ public class LogData : IEndpointV1 { public static void Map(RouteGroupBuilder group) { - group.MapGet("/logdata/poll", ([FromServices] Connection connection, [FromQuery] Library.Logging.LogMessageType level, [FromQuery] long id, [FromQuery] long? offset, [FromQuery] int? pagesize) - => ExecuteLogPoll(level, id, offset ?? 0, pagesize ?? 100)) + group.MapGet("/logdata/poll", ([FromServices] ILogWriteHandler logWriteHandler, [FromQuery] Library.Logging.LogMessageType level, [FromQuery] long id, [FromQuery] int? pagesize) + => ExecuteLogPoll(logWriteHandler, level, id, pagesize ?? 100)) .RequireAuthorization(); group.MapGet("/logdata/log", ([FromServices] Connection connection, [FromQuery] long? offset, [FromQuery] int? pagesize) @@ -44,10 +43,10 @@ public class LogData : IEndpointV1 .RequireAuthorization(); } - private static LogEntry[] ExecuteLogPoll(Library.Logging.LogMessageType level, long id, long offset, int pagesize) + private static LogEntry[] ExecuteLogPoll(ILogWriteHandler logWriteHandler, Library.Logging.LogMessageType level, long id, int pagesize) { pagesize = Math.Max(1, Math.Min(500, pagesize)); - return FIXMEGlobal.LogHandler.AfterID(id, level, pagesize).Select(x => Dto.LogEntry.FromInternalEntry(x)).ToArray(); + return logWriteHandler.AfterID(id, level, pagesize).Select(x => Dto.LogEntry.FromInternalEntry(x)).ToArray(); } private static List>? ExecuteGetLog(Connection connection, long? offset, long pagesize) diff --git a/Duplicati/WebserverCore/Endpoints/V1/ProgressState.cs b/Duplicati/WebserverCore/Endpoints/V1/ProgressState.cs index b24980e9e..b178dd718 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/ProgressState.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/ProgressState.cs @@ -18,9 +18,9 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using Duplicati.Library.RestAPI; using Duplicati.WebserverCore.Abstractions; using Duplicati.WebserverCore.Exceptions; +using Microsoft.AspNetCore.Mvc; namespace Duplicati.WebserverCore.Endpoints.V1; @@ -28,15 +28,16 @@ public class ProgressState : IEndpointV1 { public static void Map(RouteGroupBuilder group) { - group.MapGet("/progressstate", Execute) + group.MapGet("/progressstate", ([FromServices] IProgressStateProviderService progressStateProviderService) => Execute(progressStateProviderService)) .RequireAuthorization(); } - private static Server.Serialization.Interface.IProgressEventData Execute() + private static Server.Serialization.Interface.IProgressEventData Execute(IProgressStateProviderService progressStateProviderService) { - if (FIXMEGlobal.GenerateProgressState == null) + var pgState = progressStateProviderService.GenerateProgressState; + if (pgState == null) throw new NotFoundException("No active backup"); - return FIXMEGlobal.GenerateProgressState(); + return pgState(); } } diff --git a/Duplicati/WebserverCore/Endpoints/V1/RemoteControl.cs b/Duplicati/WebserverCore/Endpoints/V1/RemoteControl.cs index b8e57796b..19af1cc1e 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/RemoteControl.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/RemoteControl.cs @@ -18,7 +18,6 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using Duplicati.Library.RestAPI; using Duplicati.WebserverCore.Abstractions; using Duplicati.WebserverCore.Exceptions; using Microsoft.AspNetCore.Mvc; @@ -32,9 +31,12 @@ public class RemoteControl : IEndpointV1 group.MapGet("/remotecontrol/status", ([FromServices] IRemoteControllerRegistration registration, [FromServices] IRemoteController remoteController) => GetStatus(registration, remoteController)) .RequireAuthorization(); + } + public static void MapConditional(RouteGroupBuilder group, string origin) + { // Don't allow these in agent-mode - if (FIXMEGlobal.Origin == "Agent") + if (origin == "Agent") return; group.MapPost("/remotecontrol/enable", ([FromServices] IRemoteControllerRegistration registration, [FromServices] IRemoteController remoteController) diff --git a/Duplicati/WebserverCore/Endpoints/V1/RemoteOperation.cs b/Duplicati/WebserverCore/Endpoints/V1/RemoteOperation.cs index 8771de22d..ed8bf15fa 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/RemoteOperation.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/RemoteOperation.cs @@ -20,8 +20,8 @@ // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Interface; using Duplicati.Library.Main; -using Duplicati.Library.RestAPI; using Duplicati.Server; +using Duplicati.Server.Database; using Duplicati.WebserverCore.Abstractions; using Duplicati.WebserverCore.Exceptions; using Microsoft.AspNetCore.Mvc; @@ -38,8 +38,8 @@ namespace Duplicati.WebserverCore.Endpoints.V1 => ExecuteDbPath(input.path)) .RequireAuthorization(); - group.MapPost("/remoteoperation/test", ([FromQuery] bool? autocreate, [FromBody] RemoteOperationInput input, CancellationToken cancelToken) - => ExecuteTest(input.path, autocreate ?? false, cancelToken)) + group.MapPost("/remoteoperation/test", ([FromServices] Connection connection, [FromServices] IApplicationSettings applicationSettings, [FromQuery] bool? autocreate, [FromBody] RemoteOperationInput input, CancellationToken cancelToken) + => ExecuteTest(connection, applicationSettings, input.path, autocreate ?? false, cancelToken)) .RequireAuthorization(); group.MapPost("/remoteoperation/create", ([FromBody] RemoteOperationInput input, CancellationToken cancelToken) @@ -53,11 +53,11 @@ namespace Duplicati.WebserverCore.Endpoints.V1 return new Dto.GetDbPathDto(!string.IsNullOrWhiteSpace(path), path); } - private static Dictionary ParseUrlOptions(Library.Utility.Uri uri) + private static Dictionary ParseUrlOptions(Connection connection, Library.Utility.Uri uri) { var qp = uri.QueryParameters; - var opts = Runner.GetCommonOptions(); + var opts = Runner.GetCommonOptions(connection); foreach (var k in qp.Keys.Cast()) opts[k] = qp[k]; @@ -87,13 +87,13 @@ namespace Duplicati.WebserverCore.Endpoints.V1 } } - private static async Task GetBackend(string url, CancellationToken cancelToken) + private static async Task GetBackend(Connection connection, IApplicationSettings applicationSettings, string url, CancellationToken cancelToken) { var uri = new Library.Utility.Uri(url); - var opts = ParseUrlOptions(uri); + var opts = ParseUrlOptions(connection, uri); var tmp = new[] { uri }; - await SecretProviderHelper.ApplySecretProviderAsync([], tmp, opts, Library.Utility.TempFolder.SystemTempPath, FIXMEGlobal.SecretProvider, cancelToken); + await SecretProviderHelper.ApplySecretProviderAsync([], tmp, opts, Library.Utility.TempFolder.SystemTempPath, applicationSettings.SecretProvider, cancelToken); url = tmp[0].ToString(); var modules = ConfigureModules(opts); @@ -101,13 +101,13 @@ namespace Duplicati.WebserverCore.Endpoints.V1 return new TupleDisposeWrapper(backend, modules); } - private static async Task ExecuteTest(string url, bool autoCreate, CancellationToken cancelToken) + private static async Task ExecuteTest(Connection connection, IApplicationSettings applicationSettings, string url, bool autoCreate, CancellationToken cancelToken) { TupleDisposeWrapper? wrapper = null; try { - wrapper = await GetBackend(url, cancelToken); + wrapper = await GetBackend(connection, applicationSettings, url, cancelToken); using (var b = wrapper.Backend) { diff --git a/Duplicati/WebserverCore/Endpoints/V1/Tasks.cs b/Duplicati/WebserverCore/Endpoints/V1/Tasks.cs index 6a466c4b9..1da85989d 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Tasks.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Tasks.cs @@ -34,57 +34,55 @@ public class Tasks : IEndpointV1 } public static void Map(RouteGroupBuilder group) { - group.MapGet("/tasks", Execute).RequireAuthorization(); - group.MapGet("/task/{taskid}", ([FromRoute] long taskId) => ExecuteGet(taskId)).RequireAuthorization(); - group.MapPost("/task/{taskid}/stop", ([FromRoute] long taskId) => ExecutePost(taskId, TaskStopState.Stop)).RequireAuthorization(); - group.MapPost("/task/{taskid}/abort", ([FromRoute] long taskId) => ExecutePost(taskId, TaskStopState.Abort)).RequireAuthorization(); + group.MapGet("/tasks", ([FromServices] IQueueRunnerService queueRunnerService) => Execute(queueRunnerService)).RequireAuthorization(); + group.MapGet("/task/{taskid}", ([FromRoute] long taskId, [FromServices] IQueueRunnerService queueRunnerService) => ExecuteGet(queueRunnerService, taskId)).RequireAuthorization(); + group.MapPost("/task/{taskid}/stop", ([FromRoute] long taskId, [FromServices] IQueueRunnerService queueRunnerService) => ExecutePost(queueRunnerService, taskId, TaskStopState.Stop)).RequireAuthorization(); + group.MapPost("/task/{taskid}/abort", ([FromRoute] long taskId, [FromServices] IQueueRunnerService queueRunnerService) => ExecutePost(queueRunnerService, taskId, TaskStopState.Abort)).RequireAuthorization(); } - private static IEnumerable Execute() + private static IEnumerable Execute(IQueueRunnerService queueRunnerService) { - var cur = FIXMEGlobal.WorkThread.CurrentTask; - var n = FIXMEGlobal.WorkThread.CurrentTasks; + var cur = queueRunnerService.GetCurrentTask(); + var n = queueRunnerService.GetCurrentTasks(); if (cur != null) n.Insert(0, cur); - return n; + return n.OfType(); } - private static Dto.GetTaskStateDto ExecuteGet(long taskid) + private static Dto.GetTaskStateDto ExecuteGet(IQueueRunnerService queueRunnerService, long taskid) { - var task = FIXMEGlobal.WorkThread.CurrentTask; - var tasks = FIXMEGlobal.WorkThread.CurrentTasks; + var task = queueRunnerService.GetCurrentTask(); + var tasks = queueRunnerService.GetCurrentTasks(); if (task != null && task.TaskID == taskid) - return new Dto.GetTaskStateDto("Running", taskid); + return new Dto.GetTaskStateDto("Running", taskid, task.TaskStarted, task.TaskFinished); if (tasks.FirstOrDefault(x => x.TaskID == taskid) == null) { - KeyValuePair[] matches; - lock (FIXMEGlobal.MainLock) - matches = FIXMEGlobal.TaskResultCache.Where(x => x.Key == taskid).ToArray(); - - if (matches.Length == 0) + var res = queueRunnerService.GetCachedTaskResults(taskid); + if (res == null) throw new NotFoundException("No such task found"); - return new Dto.GetTaskStateDto( - Status: matches[0].Value == null ? "Completed" : "Failed", + Status: res.Exception == null ? "Completed" : "Failed", ID: taskid, - ErrorMessage: matches[0].Value == null ? null : matches[0].Value.Message, - Exception: matches[0].Value == null ? null : matches[0].Value.ToString() + TaskStarted: res?.TaskStarted, + TaskFinished: res?.TaskFinished, + ErrorMessage: res?.Exception?.Message, + Exception: res?.Exception?.ToString() ); } - return new Dto.GetTaskStateDto("Waiting", taskid); + return new Dto.GetTaskStateDto("Waiting", taskid, null, null); } - private static void ExecutePost(long taskid, TaskStopState stopState) + private static void ExecutePost(IQueueRunnerService queueRunnerService, long taskid, TaskStopState stopState) { - var task = FIXMEGlobal.WorkThread.CurrentTask; - var tasks = FIXMEGlobal.WorkThread.CurrentTasks; + var task = queueRunnerService.GetCurrentTask(); + var tasks = queueRunnerService.GetCurrentTasks(); if (task != null) tasks.Insert(0, task); diff --git a/Duplicati/WebserverCore/Endpoints/V1/Updates.cs b/Duplicati/WebserverCore/Endpoints/V1/Updates.cs index 86e252dbf..029f60750 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Updates.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Updates.cs @@ -18,8 +18,9 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using Duplicati.Library.RestAPI; +using Duplicati.Server; using Duplicati.WebserverCore.Abstractions; +using Microsoft.AspNetCore.Mvc; namespace Duplicati.WebserverCore.Endpoints.V1; @@ -27,9 +28,11 @@ public class Updates : IEndpointV1 { public static void Map(RouteGroupBuilder group) { - group.MapPost("/updates/check", Execute).RequireAuthorization(); + group.MapPost("/updates/check", + ([FromServices] UpdatePollThread updatePollThread) => Execute(updatePollThread)) + .RequireAuthorization(); } - private static void Execute() - => FIXMEGlobal.UpdatePoller.CheckNow(); + private static void Execute(UpdatePollThread updatePollThread) + => updatePollThread.CheckNow(); } \ No newline at end of file diff --git a/Duplicati/WebserverCore/Endpoints/V1/WebModules.cs b/Duplicati/WebserverCore/Endpoints/V1/WebModules.cs index 7c35c171f..4ef47e40e 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/WebModules.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/WebModules.cs @@ -20,8 +20,8 @@ // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Interface; using Duplicati.Library.Main; -using Duplicati.Library.RestAPI; using Duplicati.Server; +using Duplicati.Server.Database; using Duplicati.WebserverCore.Abstractions; using Duplicati.WebserverCore.Exceptions; using Microsoft.AspNetCore.Mvc; @@ -33,22 +33,24 @@ public record WebModules : IEndpointV1 public static void Map(RouteGroupBuilder group) { group.MapGet("/webmodules", ExecuteGet).RequireAuthorization(); - group.MapPost("/webmodule/{modulekey}", ([FromRoute] string modulekey, [FromBody] Dictionary options, CancellationToken cancellationToken) => ExecutePost(modulekey, options, cancellationToken)).RequireAuthorization(); + group.MapPost("/webmodule/{modulekey}", ([FromServices] Connection connection, [FromServices] IApplicationSettings applicationSettings, [FromRoute] string modulekey, [FromBody] Dictionary options, CancellationToken cancellationToken) + => ExecutePost(connection, applicationSettings, modulekey, options, cancellationToken)) + .RequireAuthorization(); } private static IEnumerable ExecuteGet() => Library.DynamicLoader.WebLoader.Modules; - private static async Task ExecutePost(string modulekey, Dictionary inputOptions, CancellationToken cancellationToken) + private static async Task ExecutePost(Connection connection, IApplicationSettings applicationSettings, string modulekey, Dictionary inputOptions, CancellationToken cancellationToken) { var m = Library.DynamicLoader.WebLoader.Modules.FirstOrDefault(x => x.Key.Equals(modulekey, StringComparison.OrdinalIgnoreCase)) ?? throw new NotFoundException("No such module found"); - var options = Runner.GetCommonOptions(); + var options = Runner.GetCommonOptions(connection); foreach (var k in inputOptions.Keys) options[k] = inputOptions[k]; - await SecretProviderHelper.ApplySecretProviderAsync([], [], options, Library.Utility.TempFolder.SystemTempPath, FIXMEGlobal.SecretProvider, cancellationToken); + await SecretProviderHelper.ApplySecretProviderAsync([], [], options, Library.Utility.TempFolder.SystemTempPath, applicationSettings.SecretProvider, cancellationToken); return new Dto.WebModuleOutputDto( Status: "OK", diff --git a/Duplicati/WebserverCore/Endpoints/V2/BackupListing.cs b/Duplicati/WebserverCore/Endpoints/V2/BackupListing.cs index d013f73d5..8466abffe 100644 --- a/Duplicati/WebserverCore/Endpoints/V2/BackupListing.cs +++ b/Duplicati/WebserverCore/Endpoints/V2/BackupListing.cs @@ -20,11 +20,13 @@ // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Interface; +using Duplicati.Library.RestAPI; using Duplicati.Server; using Duplicati.Server.Database; using Duplicati.Server.Serialization.Interface; using Duplicati.WebserverCore.Abstractions; using Duplicati.WebserverCore.Exceptions; +using Duplicati.WebserverCore.Services; using Microsoft.AspNetCore.Mvc; namespace Duplicati.WebserverCore.Endpoints.V2.Backup; @@ -33,33 +35,33 @@ public class BackupListing : IEndpointV2 { public static void Map(RouteGroupBuilder group) { - group.MapPost("/backup/list-filesets", ([FromServices] Connection connection, [FromBody] Dto.V2.ListFilesetsRequestDto input) - => ExecuteGetFilesets(GetBackup(connection, input.BackupId))) + group.MapPost("/backup/list-filesets", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromBody] Dto.V2.ListFilesetsRequestDto input) + => ExecuteGetFilesets(queueRunnerService, GetBackup(connection, input.BackupId))) .RequireAuthorization(); - group.MapPost("/backup/list-folder", ([FromServices] Connection connection, [FromBody] Dto.V2.ListFolderContentRequestDto input) - => ExecuteListFolder(GetBackup(connection, input.BackupId), input)) + group.MapPost("/backup/list-folder", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromBody] Dto.V2.ListFolderContentRequestDto input) + => ExecuteListFolder(queueRunnerService, GetBackup(connection, input.BackupId), input)) .RequireAuthorization(); - group.MapPost("/backup/list-versions", ([FromServices] Connection connection, [FromBody] Dto.V2.ListFileVersionsRequestDto input) - => ExecuteListVersions(GetBackup(connection, input.BackupId), input)) + group.MapPost("/backup/list-versions", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromBody] Dto.V2.ListFileVersionsRequestDto input) + => ExecuteListVersions(queueRunnerService, GetBackup(connection, input.BackupId), input)) .RequireAuthorization(); - group.MapPost("/backup/search", ([FromServices] Connection connection, [FromBody] Dto.V2.SearchEntriesRequestDto input) - => ExecuteSearch(GetBackup(connection, input.BackupId), input)) + group.MapPost("/backup/search", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromBody] Dto.V2.SearchEntriesRequestDto input) + => ExecuteSearch(queueRunnerService, GetBackup(connection, input.BackupId), input)) .RequireAuthorization(); } private static IBackup GetBackup(Connection connection, string id) => connection.GetBackup(id) ?? throw new NotFoundException("Backup not found"); - private static Dto.V2.ListFolderContentResponseDto ExecuteListFolder(IBackup bk, Dto.V2.ListFolderContentRequestDto input) + private static Dto.V2.ListFolderContentResponseDto ExecuteListFolder(IQueueRunnerService queueRunnerService, IBackup bk, Dto.V2.ListFolderContentRequestDto input) { var time = string.IsNullOrWhiteSpace(input.Time) ? new DateTime(0) : Library.Utility.Timeparser.ParseTimeInterval(input.Time, DateTime.Now); - var r = Runner.Run(Runner.CreateListFolderContents(bk, input.Paths, time, input.PageSize ?? 1000, input.Page ?? 0), false) as IListFolderResults; + var r = queueRunnerService.RunImmediately(Runner.CreateListFolderContents(bk, input.Paths, time, input.PageSize ?? 1000, input.Page ?? 0)) as IListFolderResults; if (r == null) throw new ServerErrorException("No result from list operation"); @@ -78,16 +80,16 @@ public class BackupListing : IEndpointV2 r.Entries.TotalCount); } - private static Dto.V2.ListFilesetsResponseDto ExecuteGetFilesets(IBackup bk) + private static Dto.V2.ListFilesetsResponseDto ExecuteGetFilesets(IQueueRunnerService queueRunnerService, IBackup bk) { - var extra = new Dictionary(); + var extra = new Dictionary(); // Retries will hang the http request extra["number-of-retries"] = "0"; try { - var r = Runner.Run(Runner.CreateListFilesetsTask(bk, extra), false) as IListFilesetResults; + var r = queueRunnerService.RunImmediately(Runner.CreateListFilesetsTask(bk, extra)) as IListFilesetResults; if (r == null) throw new ServerErrorException("No result from list operation"); @@ -112,9 +114,9 @@ public class BackupListing : IEndpointV2 } } - private static Dto.V2.ListFileVersionsOutputDto ExecuteListVersions(IBackup bk, Dto.V2.ListFileVersionsRequestDto input) + private static Dto.V2.ListFileVersionsOutputDto ExecuteListVersions(IQueueRunnerService queueRunnerService, IBackup bk, Dto.V2.ListFileVersionsRequestDto input) { - var r = Runner.Run(Runner.ListFileVersionsTask(bk, input.Paths, input.PageSize ?? 1000, input.Page ?? 0), false) as IListFileVersionsResults; + var r = queueRunnerService.RunImmediately(Runner.ListFileVersionsTask(bk, input.Paths, input.PageSize ?? 1000, input.Page ?? 0)) as IListFileVersionsResults; if (r == null) throw new ServerErrorException("No result from list operation"); @@ -135,13 +137,13 @@ public class BackupListing : IEndpointV2 r.FileVersions.TotalCount); } - private static Dto.V2.SearchEntriesResponseDto ExecuteSearch(IBackup bk, Dto.V2.SearchEntriesRequestDto input) + private static Dto.V2.SearchEntriesResponseDto ExecuteSearch(IQueueRunnerService queueRunnerService, IBackup bk, Dto.V2.SearchEntriesRequestDto input) { var time = string.IsNullOrWhiteSpace(input.Time) ? new DateTime(0) : Library.Utility.Timeparser.ParseTimeInterval(input.Time, DateTime.Now); - var r = Runner.Run(Runner.CreateSearchEntriesTask(bk, input.Filters, input.Paths, time, input.PageSize ?? 1000, input.Page ?? 0), false) as ISearchFilesResults; + var r = queueRunnerService.RunImmediately(Runner.CreateSearchEntriesTask(bk, input.Filters, input.Paths, time, input.PageSize ?? 1000, input.Page ?? 0)) as ISearchFilesResults; if (r == null) throw new ServerErrorException("No result from list operation"); diff --git a/Duplicati/WebserverCore/Endpoints/V2/DestinationVerify.cs b/Duplicati/WebserverCore/Endpoints/V2/DestinationVerify.cs index 6e63db05b..504043fdb 100644 --- a/Duplicati/WebserverCore/Endpoints/V2/DestinationVerify.cs +++ b/Duplicati/WebserverCore/Endpoints/V2/DestinationVerify.cs @@ -22,7 +22,6 @@ using Duplicati.Library.Interface; using Duplicati.Library.Main; using Duplicati.Library.Main.Volumes; -using Duplicati.Library.RestAPI; using Duplicati.Server; using Duplicati.Server.Database; using Duplicati.WebserverCore.Abstractions; @@ -35,8 +34,8 @@ public class DestinationVerify : IEndpointV2 { public static void Map(RouteGroupBuilder group) { - group.MapPost("/destination/test", ([FromServices] Connection connection, [FromBody] Dto.V2.DestinationTestRequestDto input, CancellationToken cancelToken) - => ExecuteTest(input, cancelToken)) + group.MapPost("/destination/test", ([FromServices] Connection connection, [FromServices] IApplicationSettings applicationSettings, [FromBody] Dto.V2.DestinationTestRequestDto input, CancellationToken cancelToken) + => ExecuteTest(connection, applicationSettings, input, cancelToken)) .RequireAuthorization(); } @@ -51,11 +50,11 @@ public class DestinationVerify : IEndpointV2 } } - private static Dictionary ParseUrlOptions(Library.Utility.Uri uri) + private static Dictionary ParseUrlOptions(Connection connection, Library.Utility.Uri uri) { var qp = uri.QueryParameters; - var opts = Runner.GetCommonOptions(); + var opts = Runner.GetCommonOptions(connection); foreach (var k in qp.Keys.Cast()) opts[k] = qp[k]; @@ -75,13 +74,13 @@ public class DestinationVerify : IEndpointV2 return modules; } - private static async Task GetBackend(string url, CancellationToken cancelToken) + private static async Task GetBackend(Connection connection, IApplicationSettings applicationSettings, string url, CancellationToken cancelToken) { var uri = new Library.Utility.Uri(url); - var opts = ParseUrlOptions(uri); + var opts = ParseUrlOptions(connection, uri); var tmp = new[] { uri }; - await SecretProviderHelper.ApplySecretProviderAsync([], tmp, opts, Library.Utility.TempFolder.SystemTempPath, FIXMEGlobal.SecretProvider, cancelToken); + await SecretProviderHelper.ApplySecretProviderAsync([], tmp, opts, Library.Utility.TempFolder.SystemTempPath, applicationSettings.SecretProvider, cancelToken); url = tmp[0].ToString(); var modules = ConfigureModules(opts); @@ -90,13 +89,13 @@ public class DestinationVerify : IEndpointV2 } - private static async Task ExecuteTest(DestinationTestRequestDto input, CancellationToken cancelToken) + private static async Task ExecuteTest(Connection connection, IApplicationSettings applicationSettings, DestinationTestRequestDto input, CancellationToken cancelToken) { TupleDisposeWrapper? wrapper = null; try { - wrapper = await GetBackend(input.DestinationUrl, cancelToken); + wrapper = await GetBackend(connection, applicationSettings, input.DestinationUrl, cancelToken); using (var b = wrapper.Backend) { diff --git a/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs b/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs index 7f3e80ad3..39065eafd 100644 --- a/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs +++ b/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs @@ -20,7 +20,6 @@ // DEALINGS IN THE SOFTWARE. using Duplicati.Library.IO; using Duplicati.Library.RestAPI; -using Duplicati.Library.RestAPI.Abstractions; using Duplicati.Server; using Duplicati.Server.Database; using Duplicati.Server.Serialization; @@ -36,7 +35,7 @@ namespace Duplicati.WebserverCore.Extensions; public static class ServiceCollectionsExtensions { - public static IServiceCollection AddDuplicati(this IServiceCollection services, Connection connection) + public static IServiceCollection AddDuplicati(this IServiceCollection services, Connection connection, ILogWriteHandler logWriteHandler, IApplicationSettings applicationSettings) { //old part services @@ -44,7 +43,6 @@ public static class ServiceCollectionsExtensions .AddSingleton(Serializer.JsonSettings) .AddSingleton() .AddSingleton() - .AddSingleton() .AddSingleton(connection); @@ -57,8 +55,7 @@ public static class ServiceCollectionsExtensions .AddTransient() .AddTransient() .AddSingleton() - .AddSingleton() - .AddSingleton() + .AddSingleton() .AddSingleton() .AddTransient() .AddSingleton() @@ -68,7 +65,11 @@ public static class ServiceCollectionsExtensions .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton(); + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton(logWriteHandler) + .AddSingleton(applicationSettings); return services; } diff --git a/Duplicati/WebserverCore/Extensions/WebApplicationExtensions.cs b/Duplicati/WebserverCore/Extensions/WebApplicationExtensions.cs index 1b92eddac..ede913a4a 100644 --- a/Duplicati/WebserverCore/Extensions/WebApplicationExtensions.cs +++ b/Duplicati/WebserverCore/Extensions/WebApplicationExtensions.cs @@ -56,6 +56,10 @@ public static class WebApplicationExtensions methodMap!.Invoke(null, [group]); } + // Special handling for endpoints that are not supported in all modes + var applicationSettings = application.Services.GetRequiredService(); + Endpoints.V1.RemoteControl.MapConditional(group, applicationSettings.Origin); + return application; } diff --git a/Duplicati/WebserverCore/Services/ApplicationSettings.cs b/Duplicati/WebserverCore/Services/ApplicationSettings.cs new file mode 100644 index 000000000..6e02163bc --- /dev/null +++ b/Duplicati/WebserverCore/Services/ApplicationSettings.cs @@ -0,0 +1,46 @@ + +using Duplicati.Library.AutoUpdater; +using Duplicati.Library.Interface; +using Duplicati.WebserverCore.Abstractions; +namespace Duplicati.WebserverCore.Services; + +/// +/// Application settings for the Duplicati server +/// +public class ApplicationSettings : IApplicationSettings +{ + /// + /// The application exit event + /// + private readonly ManualResetEvent _applicationExitEvent = new ManualResetEvent(false); + /// + /// The folder where Duplicati data is stored + /// + private readonly string _dataFolder; + + /// + /// Initializes a new instance of the class. + /// + public ApplicationSettings() + { + _dataFolder = DataFolderManager.GetDataFolder(DataFolderManager.AccessMode.ReadWritePermissionSet); + } + + /// + public bool SettingsEncryptionKeyProvidedExternally { get; set; } + + /// + public Action? StartOrStopUsageReporter { get; set; } + + /// + public string DataFolder => _dataFolder; + + /// + public string Origin { get; set; } = "Server"; + + /// + public ManualResetEvent ApplicationExitEvent => _applicationExitEvent; + + /// + public ISecretProvider? SecretProvider { get; set; } +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Services/CommandlineRunService.cs b/Duplicati/WebserverCore/Services/CommandlineRunService.cs index 81bff62b4..741bf31c1 100644 --- a/Duplicati/WebserverCore/Services/CommandlineRunService.cs +++ b/Duplicati/WebserverCore/Services/CommandlineRunService.cs @@ -19,35 +19,27 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.Text; -using Duplicati.Library.RestAPI; -using Duplicati.Library.RestAPI.Abstractions; using Duplicati.Server; using Duplicati.WebserverCore.Abstractions; namespace Duplicati.WebserverCore.Services; -public class CommandlineRunService(IWorkerThreadsManager workerThreadsManager) : ICommandlineRunService +public class CommandlineRunService(IQueueRunnerService queueRunnerService, ILogWriteHandler logWriteHandler) : ICommandlineRunService { private static readonly string LOGTAG = Library.Logging.Log.LogTagFromType(); - private class LogWriter : TextWriter + private class LogWriter(ActiveRun target, ILogWriteHandler logWriteHandler) : TextWriter { - private readonly ActiveRun m_target; - private readonly StringBuilder m_sb = new StringBuilder(); + private readonly StringBuilder sb = new StringBuilder(); private int m_newlinechars = 0; - public LogWriter(ActiveRun target) - { - m_target = target; - } - public override Encoding Encoding => Encoding.UTF8; public override void Write(char value) { - lock (m_target.Lock) + lock (target.Lock) { - m_sb.Append(value); + sb.Append(value); if (NewLine[m_newlinechars] == value) { m_newlinechars++; @@ -62,34 +54,34 @@ public class CommandlineRunService(IWorkerThreadsManager workerThreadsManager) : public override void WriteLine(string? value) { value ??= string.Empty; - lock (m_target.Lock) + lock (target.Lock) { - m_target.LastAccess = DateTime.Now; + target.LastAccess = DateTime.Now; //Avoid writing the log if it does not exist - if (m_target.IsLogDisposed) + if (target.IsLogDisposed) { - FIXMEGlobal.LogHandler.WriteMessage(new Library.Logging.LogEntry("Attempted to write message after closing: {0}", new object[] { value }, Library.Logging.LogMessageType.Warning, LOGTAG, "CommandLineOutputAfterLogClosed", null)); + logWriteHandler.WriteMessage(new Library.Logging.LogEntry("Attempted to write message after closing: {0}", new object[] { value }, Library.Logging.LogMessageType.Warning, LOGTAG, "CommandLineOutputAfterLogClosed", null)); return; } try { - if (m_sb.Length != 0) + if (sb.Length != 0) { - m_target.Log.Add(m_sb + value); - m_sb.Length = 0; + target.Log.Add(sb + value); + sb.Length = 0; m_newlinechars = 0; } else { - m_target.Log.Add(value); + target.Log.Add(value); } } catch (Exception ex) { // This can happen on a very unlucky race where IsLogDisposed is set right after the check - FIXMEGlobal.LogHandler.WriteMessage(new Library.Logging.LogEntry("Failed to forward commandline message: {0}", new object[] { value }, Library.Logging.LogMessageType.Warning, LOGTAG, "CommandLineOutputAfterLogClosed", ex)); + logWriteHandler.WriteMessage(new Library.Logging.LogEntry("Failed to forward commandline message: {0}", new object[] { value }, Library.Logging.LogMessageType.Warning, LOGTAG, "CommandLineOutputAfterLogClosed", ex)); } } } @@ -141,7 +133,7 @@ public class CommandlineRunService(IWorkerThreadsManager workerThreadsManager) : public string StartTask(string[] args) { var k = new ActiveRun(); - k.Writer = new LogWriter(k); + k.Writer = new LogWriter(k, logWriteHandler); m_activeItems[k.ID] = k; StartCleanupTask(); @@ -180,7 +172,7 @@ public class CommandlineRunService(IWorkerThreadsManager workerThreadsManager) : } }); - workerThreadsManager.AddTask(k.Task); + queueRunnerService.AddTask(k.Task); return k.ID; } diff --git a/Duplicati/WebserverCore/Services/WorkerThreadsManager.cs b/Duplicati/WebserverCore/Services/LoginProvider copy.cs similarity index 51% rename from Duplicati/WebserverCore/Services/WorkerThreadsManager.cs rename to Duplicati/WebserverCore/Services/LoginProvider copy.cs index 829837f4f..a3b9731e4 100644 --- a/Duplicati/WebserverCore/Services/WorkerThreadsManager.cs +++ b/Duplicati/WebserverCore/Services/LoginProvider copy.cs @@ -18,44 +18,18 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -#nullable enable -using Duplicati.Library.IO; -using Duplicati.Library.RestAPI; -using Duplicati.Library.RestAPI.Abstractions; -using Duplicati.Library.Utility; -using Duplicati.Server; +using Duplicati.Server.Serialization.Interface; using Duplicati.WebserverCore.Abstractions; namespace Duplicati.WebserverCore.Services; -public class WorkerThreadsManager(ILiveControls liveControls, IScheduler scheduler) : IWorkerThreadsManager +/// +/// Provides a way to generate progress state. +/// +public class ProgressStateProviderService : IProgressStateProviderService { - public WorkerThread? WorkerThread { get; private set; } - - public void Spawn(Action item) - { - WorkerThread = new WorkerThread(item, liveControls.IsPaused); - scheduler.Init(WorkerThread); - } - - public Tuple? CurrentTask - { - get - { - var t = WorkerThread?.CurrentTask; - return t == null ? null : new Tuple(t.TaskID, t.Backup.ID); - } - } - - public void UpdateThrottleSpeeds(string? uploadSpeed, string? downloadSpeed) - { - WorkerThread?.CurrentTask?.UpdateThrottleSpeed(uploadSpeed, downloadSpeed); - } - - public long AddTask(Runner.IRunnerData data, bool skipQueue = false) - { - WorkerThread!.AddTask(data, skipQueue); - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); - return data.TaskID; - } -} \ No newline at end of file + /// + /// The current progress state generator function. + /// + public Func? GenerateProgressState { get; set; } +} diff --git a/Duplicati/WebserverCore/Services/QueueRunnerService.cs b/Duplicati/WebserverCore/Services/QueueRunnerService.cs new file mode 100644 index 000000000..f30fd70c0 --- /dev/null +++ b/Duplicati/WebserverCore/Services/QueueRunnerService.cs @@ -0,0 +1,216 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using Duplicati.Library.Interface; +using Duplicati.Library.RestAPI; +using Duplicati.Library.Utility; +using Duplicati.Server; +using Duplicati.Server.Database; +using Duplicati.Server.Serialization.Interface; +using Duplicati.WebserverCore.Abstractions; + +namespace Duplicati.WebserverCore.Services; + +/// +/// Simple queue that will run the given task +/// +public class QueueRunnerService( + Connection connection, + EventPollNotify eventPollNotify, + INotificationUpdateService notificationUpdateService, + IProgressStateProviderService progressStateProviderService, + IApplicationSettings applicationSettings) : IQueueRunnerService +{ + private readonly object _lock = new(); + /// + /// A thread-safe dictionary to store cached task results. + /// + private readonly Dictionary _taskCache = new(); + + /// + /// The maximum number of completed task results to keep in memory + /// + private static readonly int MAX_TASK_RESULT_CACHE_SIZE = 100; + + private readonly List _tasks = new(); + private (Task? Task, IQueuedTask? QueuedTask) _current; + private bool _isPaused; + private bool _isTerminated; + + public long AddTask(IQueuedTask task) + => AddTask(task, false); + + public long AddTask(IQueuedTask task, bool skipQueue) + { + lock (_lock) + if (skipQueue) + _tasks.Insert(0, task); + else + _tasks.Add(task); + + eventPollNotify.SignalNewEvent(); + StartNextTask(); + return task.TaskID; + } + + public bool GetIsActive() + => _current.Task != null; + + public IQueuedTask? GetCurrentTask() + => _current.QueuedTask; + + public List GetCurrentTasks() + { + lock (_lock) + return [.. _tasks]; + } + + public void Pause() + { + lock (_lock) + _isPaused = true; + } + + public void Resume() + { + lock (_lock) + _isPaused = false; + + StartNextTask(); + } + + public void Terminate(bool wait) + { + _isTerminated = true; + if (wait) + { + var task = _current.Task; + if (task != null) + task.Await(); + } + } + + private void StartNextTask() + { + lock (_lock) + { + if (_isTerminated || _isPaused || (_current.Task != null && !_current.Task.IsCompleted)) + return; + + // Clean up completed tasks + if (_current.Task != null && _current.Task.IsCompleted) + _current = (null, null); + + if (_tasks.Count == 0) + return; + + var nextTask = _tasks[0]; + _tasks.RemoveAt(0); + _current = (Task.Run(() => RunTask(nextTask), CancellationToken.None), nextTask); + } + } + + private async Task RunTask(IQueuedTask task) + { + var completed = false; + try + { + eventPollNotify.SignalNewEvent(); + task.TaskStarted = DateTime.UtcNow; + if (task.OnStarting != null) + await task.OnStarting().ConfigureAwait(false); + + Runner.Run(connection, eventPollNotify, notificationUpdateService, progressStateProviderService, applicationSettings, task, true); + + // If the task is completed, don't call OnFinished again + completed = true; + AddTaskResult(new CachedTaskResult(task.TaskID, task.BackupID, task.TaskStarted, task.TaskFinished ?? DateTime.Now, null)); + if (task.OnFinished != null) + await task.OnFinished(null).ConfigureAwait(false); + } + catch (Exception ex) + { + connection.LogError(task.BackupID, "Error in worker", ex); + if (!completed) + { + AddTaskResult(new CachedTaskResult(task.TaskID, task.BackupID, task.TaskStarted, task.TaskFinished ?? DateTime.Now, ex)); + if (task.OnFinished != null) + await task.OnFinished(ex).ConfigureAwait(false); + } + } + finally + { + task.TaskFinished = DateTime.UtcNow; + lock (_lock) + _current = (null, null); + eventPollNotify.SignalNewEvent(); + StartNextTask(); + } + } + + public IList> GetQueueWithIds() + { + return (from n in GetCurrentTasks() + where n.BackupID != null + select new Tuple(n.TaskID, n.BackupID)).ToList(); + } + + + /// + public CachedTaskResult? GetCachedTaskResults(long taskID) + { + lock (_lock) + { + _taskCache.TryGetValue(taskID, out var result); + return result; + } + } + + private void AddTaskResult(CachedTaskResult taskResult) + { + lock (_lock) + { + // If the task result is already in the cache, remove it + if (_taskCache.TryGetValue(taskResult.TaskID, out var existingResult)) + { + // If the stored task result has an exception, do not overwrite it + if (existingResult.Exception != null) + return; + } + + // Add/update the new task result in the cache + _taskCache[taskResult.TaskID] = taskResult; + + // If the cache size exceeds the maximum, remove the oldest entry + while (_taskCache.Count >= MAX_TASK_RESULT_CACHE_SIZE) + { + var oldestTaskID = _taskCache.Keys.Min(); + _taskCache.Remove(oldestTaskID); + } + } + } + + /// + public IBasicResults? RunImmediately(IQueuedTask task) + { + return Runner.Run(connection, eventPollNotify, notificationUpdateService, progressStateProviderService, applicationSettings, task, false); + } +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Services/RemoteControllerHandler.cs b/Duplicati/WebserverCore/Services/RemoteControllerHandler.cs index 6664e2918..371c987e6 100644 --- a/Duplicati/WebserverCore/Services/RemoteControllerHandler.cs +++ b/Duplicati/WebserverCore/Services/RemoteControllerHandler.cs @@ -22,7 +22,6 @@ using System.Net.Http.Headers; using Duplicati.Library.Logging; using Duplicati.Library.RemoteControl; -using Duplicati.Library.RestAPI; using Duplicati.Server.Database; using Duplicati.WebserverCore.Abstractions; using Newtonsoft.Json; @@ -35,7 +34,7 @@ namespace Duplicati.WebserverCore.Services; /// The connection to the database /// The HTTP client factory /// The JWT token provider -public class RemoteControllerHandler(Connection connection, IHttpClientFactory httpClientFactory, IJWTTokenProvider jwtTokenProvider) : IRemoteControllerHandler +public class RemoteControllerHandler(Connection connection, IHttpClientFactory httpClientFactory, IJWTTokenProvider jwtTokenProvider, IApplicationSettings applicationSettings) : IRemoteControllerHandler { /// /// The log tag for this class. @@ -64,7 +63,7 @@ public class RemoteControllerHandler(Connection connection, IHttpClientFactory h CertificateUrl = data.CertificateUrl }); - if (!FIXMEGlobal.SettingsEncryptionKeyProvidedExternally) + if (!applicationSettings.SettingsEncryptionKeyProvidedExternally) { // TODO: Implement changing the encryption key // if (!string.IsNullOrWhiteSpace(data.LocalEncryptionKey) && data.LocalEncryptionKey != connection.ApplicationSettings.SettingsEncryptionKey) @@ -86,10 +85,10 @@ public class RemoteControllerHandler(Connection connection, IHttpClientFactory h try { if (string.Equals(message.ControlRequestMessage.Command, ControlRequestMessage.ConfigureReportUrlSet, StringComparison.OrdinalIgnoreCase)) - FIXMEGlobal.DataConnection.ApplicationSettings.AdditionalReportUrl = message.ControlRequestMessage.Parameters.GetValueOrDefault(ControlRequestMessage.ConfigureReportUrlParameter); + connection.ApplicationSettings.AdditionalReportUrl = message.ControlRequestMessage.Parameters.GetValueOrDefault(ControlRequestMessage.ConfigureReportUrlParameter); if (string.Equals(message.ControlRequestMessage.Command, ControlRequestMessage.ConfigureReportUrlGet, StringComparison.OrdinalIgnoreCase)) - result = new Dictionary { { ControlRequestMessage.ConfigureReportUrlParameter, FIXMEGlobal.DataConnection.ApplicationSettings.AdditionalReportUrl } }; + result = new Dictionary { { ControlRequestMessage.ConfigureReportUrlParameter, connection.ApplicationSettings.AdditionalReportUrl } }; } catch (Exception ex) { diff --git a/Duplicati/WebserverCore/Services/SchedulerService.cs b/Duplicati/WebserverCore/Services/SchedulerService.cs index 15296edf2..381fe38d9 100644 --- a/Duplicati/WebserverCore/Services/SchedulerService.cs +++ b/Duplicati/WebserverCore/Services/SchedulerService.cs @@ -19,31 +19,33 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using Duplicati.Library.Utility; +using Duplicati.Library.RestAPI; using Duplicati.Server; +using Duplicati.Server.Database; using Duplicati.Server.Serialization.Interface; using Duplicati.WebserverCore.Abstractions; namespace WebserverCore.Services; -public class SchedulerService : IScheduler +public class SchedulerService : ISchedulerService { - private readonly Duplicati.Server.Scheduler scheduler; - public SchedulerService(Duplicati.Server.Scheduler scheduler) + private readonly Scheduler scheduler; + public SchedulerService(Connection connection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IQueueRunnerService queueRunnerService) { - this.scheduler = scheduler; + this.scheduler = new Scheduler(connection, queueRunnerService); + var lastScheduleId = notificationUpdateService.LastDataUpdateId; + eventPollNotify.NewEvent += (sender, e) => + { + if (lastScheduleId != notificationUpdateService.LastDataUpdateId) + { + lastScheduleId = notificationUpdateService.LastDataUpdateId; + Reschedule(); + } + }; } public List> Schedule => scheduler.Schedule; - public List WorkerQueue => scheduler.WorkerQueue; - - public void SubScribeToNewSchedule(Action handler) - => scheduler.NewSchedule += (_, _) => handler(); - - public IList> GetSchedulerQueueIds() - => scheduler.GetSchedulerQueueIds(); - public IList> GetProposedSchedule() => scheduler.GetProposedSchedule(); @@ -52,7 +54,4 @@ public class SchedulerService : IScheduler public void Terminate(bool wait) => scheduler.Terminate(wait); - - public void Init(WorkerThread worker) - => scheduler.Init(worker); } diff --git a/Duplicati/WebserverCore/Services/StatusService.cs b/Duplicati/WebserverCore/Services/StatusService.cs index 7c3845b8b..336ea261e 100644 --- a/Duplicati/WebserverCore/Services/StatusService.cs +++ b/Duplicati/WebserverCore/Services/StatusService.cs @@ -19,7 +19,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.RestAPI; -using Duplicati.Library.RestAPI.Abstractions; using Duplicati.Server; using Duplicati.Server.Serialization; using Duplicati.WebserverCore.Abstractions; @@ -31,22 +30,24 @@ public class StatusService( LiveControls liveControls, UpdatePollThread updatePollThread, IUpdateService updateService, - IWorkerThreadsManager workerThreadsManager, + IQueueRunnerService queueRunnerService, ISettingsService settingsService, - IScheduler scheduler, + ISchedulerService scheduler, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService) : IStatusService { public ServerStatusDto GetStatus() { + var task = queueRunnerService.GetCurrentTask(); + var status = new ServerStatusDto { UpdatedVersion = GetUpdatedVersion(), UpdaterState = updatePollThread.ThreadState, UpdateDownloadProgress = updatePollThread.DownloadProgess, - ActiveTask = workerThreadsManager.CurrentTask, - SchedulerQueueIds = scheduler.GetSchedulerQueueIds(), + ActiveTask = task == null ? null : new Tuple(task.TaskID, task.BackupID), + SchedulerQueueIds = queueRunnerService.GetQueueWithIds(), ProposedSchedule = scheduler.GetProposedSchedule(), LastEventID = eventPollNotify.EventNo, LastDataUpdateID = notificationUpdateService.LastDataUpdateId, @@ -72,7 +73,8 @@ public class StatusService( private SuggestedStatusIcon MapStateToIcon() { - if (workerThreadsManager.CurrentTask == null) + var task = queueRunnerService.GetCurrentTask(); + if (task == null) { if (liveControls.State == LiveControls.LiveControlState.Paused) return SuggestedStatusIcon.Paused; diff --git a/Duplicati/WebserverCore/Services/SystemInfoProvider.cs b/Duplicati/WebserverCore/Services/SystemInfoProvider.cs index 034464f08..b4e0e7b2d 100644 --- a/Duplicati/WebserverCore/Services/SystemInfoProvider.cs +++ b/Duplicati/WebserverCore/Services/SystemInfoProvider.cs @@ -21,8 +21,8 @@ using System.Globalization; using Duplicati.Library.AutoUpdater; using Duplicati.Library.Localization; -using Duplicati.Library.RestAPI; using Duplicati.Server; +using Duplicati.Server.Database; using Duplicati.Server.Serialization.Interface; using Duplicati.WebserverCore.Abstractions; using Duplicati.WebserverCore.Dto; @@ -32,7 +32,7 @@ namespace Duplicati.WebserverCore.Services; /// /// Produces system information. /// -public class SystemInfoProvider : ISystemInfoProvider +public class SystemInfoProvider(IApplicationSettings applicationSettings) : ISystemInfoProvider { /// /// The API extensions that are available @@ -42,7 +42,9 @@ public class SystemInfoProvider : ISystemInfoProvider "v2:backup:list-folder", "v2:backup:list-versions", "v2:backup:search", - "v2:destination:test" + "v2:destination:test", + "v1:gettask:taskstarted", + "v1:gettask:taskfinished", ]; /// @@ -85,11 +87,6 @@ public class SystemInfoProvider : ISystemInfoProvider /// public required string RemoteControlRegistrationUrl { get; init; } - /// - /// Gets or sets the started by. - /// - public required string StartedBy { get; init; } - /// /// Gets or sets the default update channel. /// @@ -229,12 +226,11 @@ public class SystemInfoProvider : ISystemInfoProvider => new StaticSystemInformation { APIVersion = 1, - PasswordPlaceholder = FIXMEGlobal.PASSWORD_PLACEHOLDER, + PasswordPlaceholder = Connection.PASSWORD_PLACEHOLDER, ServerVersion = UpdaterManager.SelfVersion.Version, ServerVersionName = License.VersionNumbers.VERSION_NAME, ServerVersionType = UpdaterManager.SelfVersion.ReleaseType, RemoteControlRegistrationUrl = Library.RemoteControl.RegisterForRemote.DefaultRegisterationUrl, - StartedBy = FIXMEGlobal.Origin, DefaultUpdateChannel = AutoUpdateSettings.DefaultUpdateChannel.ToString(), DefaultUsageReportLevel = Library.UsageReporter.Reporter.DefaultReportLevel, OSType = UpdaterManager.OperatingSystemName, @@ -313,7 +309,7 @@ public class SystemInfoProvider : ISystemInfoProvider ServerVersionName = systeminfo.ServerVersionName, ServerVersionType = systeminfo.ServerVersionType, RemoteControlRegistrationUrl = systeminfo.RemoteControlRegistrationUrl, - StartedBy = systeminfo.StartedBy, + StartedBy = applicationSettings.Origin, DefaultUpdateChannel = systeminfo.DefaultUpdateChannel, DefaultUsageReportLevel = systeminfo.DefaultUsageReportLevel, ServerTime = DateTime.Now, diff --git a/LiveTests/Duplicati.Backend.Tests/OneDrive/OneDriveTests.cs b/LiveTests/Duplicati.Backend.Tests/OneDrive/OneDriveTests.cs new file mode 100644 index 000000000..e45478dd3 --- /dev/null +++ b/LiveTests/Duplicati.Backend.Tests/OneDrive/OneDriveTests.cs @@ -0,0 +1,46 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +namespace Duplicati.Backend.Tests.OneDrive; + +/// +/// OneDrive Tests +/// +[TestClass] +public sealed class OneDriveTests : BaseTest +{ + [TestMethod] + public Task TestOneDrive() + { + CheckRequiredEnvironment(["TESTCREDENTIAL_ONEDRIVE_FOLDER", "TESTCREDENTIAL_ONEDRIVE_AUTHID"]); + + var exitCode = CommandLine.BackendTester.Program.Main( + new[] + { + $"onedrivev2://{Environment.GetEnvironmentVariable("TESTCREDENTIAL_ONEDRIVE_FOLDER")}/?authid={Uri.EscapeDataString(Environment.GetEnvironmentVariable("TESTCREDENTIAL_ONEDRIVE_AUTHID")!)}", + + }.Concat(Parameters.GlobalTestParameters).ToArray()); + + if (exitCode != 0) Assert.Fail("BackendTester is returning non-zero exit code, check logs for details"); + + return Task.CompletedTask; + } +} \ No newline at end of file