From a41c330b0690361d6a4a285802885129bc058455 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 29 Apr 2025 16:26:42 +0200 Subject: [PATCH 01/24] Added OneDrive to CI tests --- .github/workflows/backendtests.yml | 55 ++++++++++++++++++- .../OneDrive/OneDriveTests.cs | 46 ++++++++++++++++ 2 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 LiveTests/Duplicati.Backend.Tests/OneDrive/OneDriveTests.cs diff --git a/.github/workflows/backendtests.yml b/.github/workflows/backendtests.yml index 68a0fb9a7..6ac7f7777 100644 --- a/.github/workflows/backendtests.yml +++ b/.github/workflows/backendtests.yml @@ -17,6 +17,7 @@ jobs: box_secrets_available: "${{ steps.check_box.outputs.available }}" filen_secrets_available: "${{ steps.check_filen.outputs.available }}" filejump_secrets_available: "${{ steps.check_filejump.outputs.available }}" + onedrive_secrets_available: "${{ steps.check_onedrive.outputs.available }}" steps: - id: check_pcloud name: Check pCloud secrets @@ -156,7 +157,7 @@ jobs: name: Check Filejump secrets shell: bash run: | - echo "Starting Filen.io secrets check..." + echo "Starting Filenjump secrets check..." if [[ -n "${{ secrets.TESTCREDENTIAL_FILEJUMP_FOLDER }}" ]] && \ [[ -n "${{ secrets.TESTCREDENTIAL_FILEJUMP_TOKEN }}" ]]; then echo "All Filejump secrets found" @@ -166,6 +167,20 @@ jobs: echo "available=false" >> $GITHUB_OUTPUT fi echo "Filejump check completed" + - id: check_onedrive + name: Check OneDrive secrets + shell: bash + run: | + echo "Starting OneDrive secrets check..." + if [[ -n "${{ secrets.TESTCREDENTIAL_ONEDRIVE_FOLDER }}" ]] && \ + [[ -n "${{ secrets.TESTCREDENTIAL_ONEDRIVE_AUTHID }}" ]]; then + echo "All OneDrive secrets found" + echo "available=true" >> $GITHUB_OUTPUT + else + echo "Missing some OneDrive secrets" + echo "available=false" >> $GITHUB_OUTPUT + fi + echo "OneDrive check completed" - name: Debug Output shell: bash run: | @@ -178,6 +193,8 @@ jobs: echo "B2 Available: ${{ steps.check_b2.outputs.available }}" echo "Box.com Available: ${{ steps.check_box.outputs.available }}" echo "Filen.io Available: ${{ steps.check_filen.outputs.available }}" + echo "Filejump Available: ${{ steps.check_filejump.outputs.available }}" + echo "OneDrive Available: ${{ steps.check_onedrive.outputs.available }}" test_ftp: needs: check_secrets if: needs.check_secrets.outputs.testcontainers_secrets_available == 'true' @@ -662,3 +679,39 @@ jobs: dotnet test --no-build --filter="ClassName~Filejump.FilejumpTests" --logger:"console;verbosity=detailed" LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + test_onedrive: + needs: check_secrets + if: needs.check_secrets.outputs.onedrive_secrets_available == 'true' + name: OneDrive Tests + runs-on: "${{ matrix.os }}" + strategy: + max-parallel: 1 + fail-fast: false + matrix: + os: + - ubuntu-latest + - windows-latest + - macos-latest + steps: + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.x + - name: Checkout source + uses: actions/checkout@v4 + - name: Restore NuGet dependencies + run: >- + dotnet restore + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + - name: Build project + run: >- + dotnet build --no-restore + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + - name: Run OneDrive tests + env: + TESTCREDENTIAL_ONEDRIVE_FOLDER: "${{ secrets.TESTCREDENTIAL_ONEDRIVE_FOLDER }}" + TESTCREDENTIAL_ONEDRIVE_AUTHID: "${{ secrets.TESTCREDENTIAL_ONEDRIVE_AUTHID }}" + run: >- + dotnet test --no-build --filter="ClassName~OneDrive.OneDriveTests" + --logger:"console;verbosity=detailed" + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln diff --git a/LiveTests/Duplicati.Backend.Tests/OneDrive/OneDriveTests.cs b/LiveTests/Duplicati.Backend.Tests/OneDrive/OneDriveTests.cs new file mode 100644 index 000000000..e45478dd3 --- /dev/null +++ b/LiveTests/Duplicati.Backend.Tests/OneDrive/OneDriveTests.cs @@ -0,0 +1,46 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +namespace Duplicati.Backend.Tests.OneDrive; + +/// +/// OneDrive Tests +/// +[TestClass] +public sealed class OneDriveTests : BaseTest +{ + [TestMethod] + public Task TestOneDrive() + { + CheckRequiredEnvironment(["TESTCREDENTIAL_ONEDRIVE_FOLDER", "TESTCREDENTIAL_ONEDRIVE_AUTHID"]); + + var exitCode = CommandLine.BackendTester.Program.Main( + new[] + { + $"onedrivev2://{Environment.GetEnvironmentVariable("TESTCREDENTIAL_ONEDRIVE_FOLDER")}/?authid={Uri.EscapeDataString(Environment.GetEnvironmentVariable("TESTCREDENTIAL_ONEDRIVE_AUTHID")!)}", + + }.Concat(Parameters.GlobalTestParameters).ToArray()); + + if (exitCode != 0) Assert.Fail("BackendTester is returning non-zero exit code, check logs for details"); + + return Task.CompletedTask; + } +} \ No newline at end of file From c008c0e1e7401e3292fdf91208dabc80c4ab488b Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 29 Apr 2025 17:13:40 +0200 Subject: [PATCH 02/24] Fixed a typo --- .github/workflows/backendtests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backendtests.yml b/.github/workflows/backendtests.yml index 6ac7f7777..4d61a8748 100644 --- a/.github/workflows/backendtests.yml +++ b/.github/workflows/backendtests.yml @@ -157,7 +157,7 @@ jobs: name: Check Filejump secrets shell: bash run: | - echo "Starting Filenjump secrets check..." + echo "Starting Filejump secrets check..." if [[ -n "${{ secrets.TESTCREDENTIAL_FILEJUMP_FOLDER }}" ]] && \ [[ -n "${{ secrets.TESTCREDENTIAL_FILEJUMP_TOKEN }}" ]]; then echo "All Filejump secrets found" From 4ecb85d2def32a7e84ae81517fabcc478d374314 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 2 May 2025 15:11:34 +0200 Subject: [PATCH 03/24] Added task metadata This adds the task start and finish time. The tracking of completed tasks has been moved to a separate service to reduce the use of FIXMEGlobal. --- Duplicati/Library/RestAPI/FIXMEGlobal.cs | 13 ---- Duplicati/Library/RestAPI/Runner.cs | 13 +++- Duplicati/Server/Program.cs | 50 +++---------- .../Abstractions/ITaskCacheService.cs | 53 +++++++++++++ .../WebserverCore/Dto/GetTaskStateDto.cs | 4 +- Duplicati/WebserverCore/Endpoints/V1/Tasks.cs | 24 +++--- .../ServiceCollectionsExtensions.cs | 3 +- .../Services/SystemInfoProvider.cs | 4 +- .../Services/TaskCacheResults.cs | 75 +++++++++++++++++++ 9 files changed, 170 insertions(+), 69 deletions(-) create mode 100644 Duplicati/WebserverCore/Abstractions/ITaskCacheService.cs create mode 100644 Duplicati/WebserverCore/Services/TaskCacheResults.cs diff --git a/Duplicati/Library/RestAPI/FIXMEGlobal.cs b/Duplicati/Library/RestAPI/FIXMEGlobal.cs index 8b6253cda..9d3d66839 100644 --- a/Duplicati/Library/RestAPI/FIXMEGlobal.cs +++ b/Duplicati/Library/RestAPI/FIXMEGlobal.cs @@ -21,7 +21,6 @@ using Duplicati.Server; using System; -using System.Collections.Generic; using Duplicati.Library.RestAPI.Abstractions; using Duplicati.Library.Utility; using Duplicati.WebserverCore.Abstractions; @@ -110,18 +109,6 @@ namespace Duplicati.Library.RestAPI /// public static System.Threading.ManualResetEvent ApplicationExitEvent; - - /// - /// List of completed task results - /// - public static readonly List> TaskResultCache = new List>(); - - - /// - /// This is the lock to be used before manipulating the shared resources - /// - public static readonly object MainLock = new object(); - /// /// The shared secret provider from the server invocation /// diff --git a/Duplicati/Library/RestAPI/Runner.cs b/Duplicati/Library/RestAPI/Runner.cs index c55fe657d..aa73ceabc 100644 --- a/Duplicati/Library/RestAPI/Runner.cs +++ b/Duplicati/Library/RestAPI/Runner.cs @@ -45,6 +45,8 @@ namespace Duplicati.Server void Resume(); void UpdateThrottleSpeed(string uploadSpeed, string downloadSpeed); void SetController(Duplicati.Library.Main.Controller controller); + DateTime? TaskStarted { get; set; } + DateTime? TaskFinished { get; set; } } private class RunnerData : IRunnerData @@ -63,6 +65,9 @@ namespace Duplicati.Server public int PageSize { get; internal set; } = 0; public int PageOffset { get; internal set; } = 0; + public DateTime? TaskStarted { get; set; } + public DateTime? TaskFinished { get; set; } + internal Duplicati.Library.Main.Controller Controller { get; set; } public void SetController(Duplicati.Library.Main.Controller controller) @@ -471,6 +476,7 @@ namespace Duplicati.Server public static Duplicati.Library.Interface.IBasicResults Run(IRunnerData data, bool fromQueue) { + data.TaskStarted = DateTime.Now; if (data is CustomRunnerTask task) { try @@ -485,6 +491,10 @@ namespace Duplicati.Server { FIXMEGlobal.DataConnection.LogError(string.Empty, "Failed while executing custom task", ex); } + finally + { + data.TaskFinished = DateTime.Now; + } return null; } @@ -707,7 +717,8 @@ namespace Duplicati.Server } finally { - ((RunnerData)data).Controller = null; + data.SetController(null); + data.TaskFinished = DateTime.Now; } } diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index fc1f7809a..55c734049 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -99,26 +99,11 @@ namespace Duplicati.Server /// public static Database.Connection DataConnection { get => FIXMEGlobal.DataConnection; set => FIXMEGlobal.DataConnection = value; } - /// - /// This is the lock to be used before manipulating the shared resources - /// - public static object MainLock { get => FIXMEGlobal.MainLock; } - /// /// This is the scheduling thread /// public static IScheduler Scheduler { get => FIXMEGlobal.Scheduler; } - /// - /// List of completed task results - /// - public static List> TaskResultCache { get => FIXMEGlobal.TaskResultCache; } - - /// - /// The maximum number of completed task results to keep in memory - /// - private static readonly int MAX_TASK_RESULT_CACHE_SIZE = 100; - /// /// The thread running the ping-pong handler /// @@ -450,13 +435,22 @@ namespace Duplicati.Server private static void SetWorkerThread() { FIXMEGlobal.WorkerThreadsManager.Spawn(x => { Runner.Run(x, true); }); - FIXMEGlobal.WorkThread.StartingWork += (worker, task) => { SignalNewEvent(null, null); }; - FIXMEGlobal.WorkThread.CompletedWork += (worker, task) => { SignalNewEvent(null, null); }; + FIXMEGlobal.WorkThread.StartingWork += (worker, task) => + { + SignalNewEvent(null, null); + task.TaskStarted = DateTime.Now; + }; + FIXMEGlobal.WorkThread.CompletedWork += (worker, task) => + { + SignalNewEvent(null, null); + FIXMEGlobal.Provider.GetRequiredService()?.AddTaskResult(new CachedTaskResult(task.TaskID, task.BackupID, task.TaskStarted, task.TaskFinished ?? DateTime.Now, null)); + }; FIXMEGlobal.WorkThread.WorkQueueChanged += (worker) => { SignalNewEvent(null, null); }; FIXMEGlobal.Scheduler.SubScribeToNewSchedule(() => SignalNewEvent(null, null)); FIXMEGlobal.WorkThread.OnError += (worker, task, exception) => { DataConnection.LogError(task?.BackupID, "Error in worker", exception); + FIXMEGlobal.Provider.GetRequiredService()?.AddTaskResult(new CachedTaskResult(task.TaskID, task.BackupID, task.TaskStarted, task.TaskFinished ?? DateTime.Now, exception)); }; var lastScheduleId = FIXMEGlobal.NotificationUpdateService.LastDataUpdateId; @@ -466,28 +460,6 @@ namespace Duplicati.Server lastScheduleId = FIXMEGlobal.NotificationUpdateService.LastDataUpdateId; Scheduler.Reschedule(); }; - - void RegisterTaskResult(long id, Exception ex) - { - lock (MainLock) - { - // If the new results says it crashed, we store that instead of success - if (TaskResultCache.Count > 0 && TaskResultCache.Last().Key == id) - { - if (ex != null && TaskResultCache.Last().Value == null) - TaskResultCache.RemoveAt(TaskResultCache.Count - 1); - else - return; - } - - TaskResultCache.Add(new KeyValuePair(id, ex)); - while (TaskResultCache.Count > MAX_TASK_RESULT_CACHE_SIZE) - TaskResultCache.RemoveAt(0); - } - } - - FIXMEGlobal.WorkThread.CompletedWork += (worker, task) => { RegisterTaskResult(task.TaskID, null); }; - FIXMEGlobal.WorkThread.OnError += (worker, task, exception) => { RegisterTaskResult(task.TaskID, exception); }; } private static void SetPurgeTempFilesTimer(Dictionary commandlineOptions) diff --git a/Duplicati/WebserverCore/Abstractions/ITaskCacheService.cs b/Duplicati/WebserverCore/Abstractions/ITaskCacheService.cs new file mode 100644 index 000000000..a471cf8c3 --- /dev/null +++ b/Duplicati/WebserverCore/Abstractions/ITaskCacheService.cs @@ -0,0 +1,53 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using Duplicati.WebserverCore.Dto; + +namespace Duplicati.WebserverCore.Abstractions; + + +/// +/// A cached task result +/// +/// The task ID +/// The backup ID +/// The time the task started +/// The time the task finished +/// The exception that was thrown +public sealed record CachedTaskResult(long TaskID, string? BackupId, DateTime? TaskStarted, DateTime? TaskFinished, Exception? Exception); + + +/// +/// Interface for the task result cache service +/// +public interface ITaskCacheService +{ + /// + /// Gets the cached task results for a given task ID + /// + /// The task ID + /// The cached task result + CachedTaskResult? GetCachedTaskResults(long taskID); + /// + /// Adds a task result to the cache + /// + /// The task result to add + void AddTaskResult(CachedTaskResult taskResult); +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Dto/GetTaskStateDto.cs b/Duplicati/WebserverCore/Dto/GetTaskStateDto.cs index 3a29ecb06..016aa49f8 100644 --- a/Duplicati/WebserverCore/Dto/GetTaskStateDto.cs +++ b/Duplicati/WebserverCore/Dto/GetTaskStateDto.cs @@ -25,6 +25,8 @@ namespace Duplicati.WebserverCore.Dto; /// /// The status /// The ID +/// When the task started +/// When the task finished /// The error message /// The exception -public record GetTaskStateDto(string Status, long ID, string? ErrorMessage = null, string? Exception = null); +public record GetTaskStateDto(string Status, long ID, DateTime? TaskStarted, DateTime? TaskFinished, string? ErrorMessage = null, string? Exception = null); diff --git a/Duplicati/WebserverCore/Endpoints/V1/Tasks.cs b/Duplicati/WebserverCore/Endpoints/V1/Tasks.cs index 6a466c4b9..a1a72cca2 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Tasks.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Tasks.cs @@ -35,7 +35,7 @@ public class Tasks : IEndpointV1 public static void Map(RouteGroupBuilder group) { group.MapGet("/tasks", Execute).RequireAuthorization(); - group.MapGet("/task/{taskid}", ([FromRoute] long taskId) => ExecuteGet(taskId)).RequireAuthorization(); + group.MapGet("/task/{taskid}", ([FromRoute] long taskId, [FromServices] ITaskCacheService taskCacheService) => ExecuteGet(taskCacheService, taskId)).RequireAuthorization(); group.MapPost("/task/{taskid}/stop", ([FromRoute] long taskId) => ExecutePost(taskId, TaskStopState.Stop)).RequireAuthorization(); group.MapPost("/task/{taskid}/abort", ([FromRoute] long taskId) => ExecutePost(taskId, TaskStopState.Abort)).RequireAuthorization(); @@ -52,33 +52,31 @@ public class Tasks : IEndpointV1 return n; } - private static Dto.GetTaskStateDto ExecuteGet(long taskid) + private static Dto.GetTaskStateDto ExecuteGet(ITaskCacheService taskCacheService, long taskid) { var task = FIXMEGlobal.WorkThread.CurrentTask; var tasks = FIXMEGlobal.WorkThread.CurrentTasks; if (task != null && task.TaskID == taskid) - return new Dto.GetTaskStateDto("Running", taskid); + return new Dto.GetTaskStateDto("Running", taskid, task.TaskStarted, task.TaskFinished); if (tasks.FirstOrDefault(x => x.TaskID == taskid) == null) { - KeyValuePair[] matches; - lock (FIXMEGlobal.MainLock) - matches = FIXMEGlobal.TaskResultCache.Where(x => x.Key == taskid).ToArray(); - - if (matches.Length == 0) + var res = taskCacheService.GetCachedTaskResults(taskid); + if (res == null) throw new NotFoundException("No such task found"); - return new Dto.GetTaskStateDto( - Status: matches[0].Value == null ? "Completed" : "Failed", + Status: res.Exception == null ? "Completed" : "Failed", ID: taskid, - ErrorMessage: matches[0].Value == null ? null : matches[0].Value.Message, - Exception: matches[0].Value == null ? null : matches[0].Value.ToString() + TaskStarted: res?.TaskStarted, + TaskFinished: res?.TaskFinished, + ErrorMessage: res?.Exception?.Message, + Exception: res?.Exception?.ToString() ); } - return new Dto.GetTaskStateDto("Waiting", taskid); + return new Dto.GetTaskStateDto("Waiting", taskid, null, null); } private static void ExecutePost(long taskid, TaskStopState stopState) diff --git a/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs b/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs index 7f3e80ad3..45264c9fe 100644 --- a/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs +++ b/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs @@ -68,7 +68,8 @@ public static class ServiceCollectionsExtensions .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton(); + .AddSingleton() + .AddSingleton(); return services; } diff --git a/Duplicati/WebserverCore/Services/SystemInfoProvider.cs b/Duplicati/WebserverCore/Services/SystemInfoProvider.cs index 034464f08..faacc34b9 100644 --- a/Duplicati/WebserverCore/Services/SystemInfoProvider.cs +++ b/Duplicati/WebserverCore/Services/SystemInfoProvider.cs @@ -42,7 +42,9 @@ public class SystemInfoProvider : ISystemInfoProvider "v2:backup:list-folder", "v2:backup:list-versions", "v2:backup:search", - "v2:destination:test" + "v2:destination:test", + "v1:gettask:taskstarted", + "v1:gettask:taskfinished", ]; /// diff --git a/Duplicati/WebserverCore/Services/TaskCacheResults.cs b/Duplicati/WebserverCore/Services/TaskCacheResults.cs new file mode 100644 index 000000000..0f8d171a3 --- /dev/null +++ b/Duplicati/WebserverCore/Services/TaskCacheResults.cs @@ -0,0 +1,75 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using Duplicati.WebserverCore.Abstractions; + +namespace Duplicati.WebserverCore.Services; + +/// +/// Service for caching task results. +/// +public class TaskCacheService : ITaskCacheService +{ + /// + /// A thread-safe dictionary to store cached task results. + /// + private readonly Dictionary _taskCache = new(); + + /// + /// The maximum number of completed task results to keep in memory + /// + private static readonly int MAX_TASK_RESULT_CACHE_SIZE = 100; + + /// + public CachedTaskResult? GetCachedTaskResults(long taskID) + { + lock (_taskCache) + { + _taskCache.TryGetValue(taskID, out var result); + return result; + } + } + + /// + public void AddTaskResult(CachedTaskResult taskResult) + { + lock (_taskCache) + { + // If the task result is already in the cache, remove it + if (_taskCache.TryGetValue(taskResult.TaskID, out var existingResult)) + { + // If the stored task result has an exception, do not overwrite it + if (existingResult.Exception != null) + return; + } + + // Add/update the new task result in the cache + _taskCache[taskResult.TaskID] = taskResult; + + // If the cache size exceeds the maximum, remove the oldest entry + while (_taskCache.Count >= MAX_TASK_RESULT_CACHE_SIZE) + { + var oldestTaskID = _taskCache.Keys.Min(); + _taskCache.Remove(oldestTaskID); + } + } + } + +} From 3430c73e33ea130c96f2d80f2b5b4955d117d229 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 2 May 2025 22:57:26 +0200 Subject: [PATCH 04/24] Update StreamUtil This makes smoother progress when throttling --- Duplicati/Library/Utility/Duplicati.Library.Utility.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj index 1fb23330e..a37f25bff 100644 --- a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj +++ b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj @@ -8,7 +8,7 @@ - + From 072f89c747d2fcda04a13898dce2bb1be759ae7d Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 3 May 2025 14:58:07 +0200 Subject: [PATCH 05/24] Remove WorkerThread class This removes the WorkerThread class and many of the complications around it. Instead of having a generic worker, there is now the `QueueRunnerService` which takes on all the resposibilities for handling queued tasks. This is one step towards removing `FIXMEGlobal` --- .../Abstractions/IQueueRunnerService.cs} | 62 ++- .../{IScheduler.cs => ISchedulerService.cs} | 25 +- .../Abstractions/IWorkerThreadsManager.cs | 37 -- .../Library/RestAPI/Database/Connection.cs | 75 ++-- .../RestAPI/Database/ServerSettings.cs | 15 +- Duplicati/Library/RestAPI/FIXMEGlobal.cs | 13 +- Duplicati/Library/RestAPI/Runner.cs | 237 ++++++----- Duplicati/Library/RestAPI/Scheduler.cs | 111 ++--- Duplicati/Library/Utility/Utility.cs | 2 +- Duplicati/Library/Utility/WorkerThread.cs | 401 ------------------ .../Interface/IQueuedTask.cs | 85 +++- Duplicati/Server/Program.cs | 57 +-- .../WebserverCore/Dto/ServerStatusDto.cs | 4 +- .../Endpoints/V1/Backup/BackupGet.cs | 22 +- .../Endpoints/V1/Backup/BackupPost.cs | 91 ++-- .../Endpoints/V1/Backup/BackupPutDelete.cs | 85 ++-- Duplicati/WebserverCore/Endpoints/V1/Tasks.cs | 30 +- .../ServiceCollectionsExtensions.cs | 7 +- .../Services/CommandlineRunService.cs | 5 +- .../Services/QueueRunnerService.cs | 208 +++++++++ .../Services/SchedulerService.cs | 30 +- .../WebserverCore/Services/StatusService.cs | 14 +- .../Services/TaskCacheResults.cs | 75 ---- .../Services/WorkerThreadsManager.cs | 61 --- 24 files changed, 709 insertions(+), 1043 deletions(-) rename Duplicati/{WebserverCore/Abstractions/ITaskCacheService.cs => Library/RestAPI/Abstractions/IQueueRunnerService.cs} (51%) rename Duplicati/Library/RestAPI/Abstractions/{IScheduler.cs => ISchedulerService.cs} (72%) delete mode 100644 Duplicati/Library/RestAPI/Abstractions/IWorkerThreadsManager.cs delete mode 100644 Duplicati/Library/Utility/WorkerThread.cs create mode 100644 Duplicati/WebserverCore/Services/QueueRunnerService.cs delete mode 100644 Duplicati/WebserverCore/Services/TaskCacheResults.cs delete mode 100644 Duplicati/WebserverCore/Services/WorkerThreadsManager.cs diff --git a/Duplicati/WebserverCore/Abstractions/ITaskCacheService.cs b/Duplicati/Library/RestAPI/Abstractions/IQueueRunnerService.cs similarity index 51% rename from Duplicati/WebserverCore/Abstractions/ITaskCacheService.cs rename to Duplicati/Library/RestAPI/Abstractions/IQueueRunnerService.cs index a471cf8c3..554ae4bae 100644 --- a/Duplicati/WebserverCore/Abstractions/ITaskCacheService.cs +++ b/Duplicati/Library/RestAPI/Abstractions/IQueueRunnerService.cs @@ -1,3 +1,4 @@ + // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // @@ -18,11 +19,14 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using Duplicati.WebserverCore.Dto; +#nullable enable + +using System; +using System.Collections.Generic; +using Duplicati.Server.Serialization.Interface; namespace Duplicati.WebserverCore.Abstractions; - /// /// A cached task result /// @@ -33,21 +37,63 @@ namespace Duplicati.WebserverCore.Abstractions; /// The exception that was thrown public sealed record CachedTaskResult(long TaskID, string? BackupId, DateTime? TaskStarted, DateTime? TaskFinished, Exception? Exception); - /// -/// Interface for the task result cache service +/// Class to encapsulate a thread that runs a list of queued operations /// -public interface ITaskCacheService +/// The type to operate on +public interface IQueueRunnerService { + /// + /// Returns a copy of the current tasks in the queue + /// + /// A list of queued tasks + List GetCurrentTasks(); + /// + /// Gets a flag indicating if the queue is currently executing a task + /// + /// True if the queue is executing a task, false otherwise + bool GetIsActive(); + /// + /// Returns the currently executing task in the queue + /// + /// The currently executing task, or null if no task is executing + IQueuedTask? GetCurrentTask(); + /// /// Gets the cached task results for a given task ID /// /// The task ID /// The cached task result CachedTaskResult? GetCachedTaskResults(long taskID); + /// - /// Adds a task result to the cache + /// Adds a task to the queue /// - /// The task result to add - void AddTaskResult(CachedTaskResult taskResult); + /// The task to add + long AddTask(IQueuedTask task); + /// + /// Adds a task to the queue, optionally skipping the queue + /// + /// The task to add + /// Whether to skip the queue + long AddTask(IQueuedTask task, bool skipQueue); + /// + /// Removes a task from the queue + /// + /// Whether to wait for the task to finish + void Terminate(bool wait); + /// + /// Resumes processing items in the queue + /// + void Resume(); + /// + /// Pauses processing items in the queue + /// + void Pause(); + + /// + /// Gets the IDs of the tasks in the worker queue + /// + /// A list of tuples containing the task ID and backup ID + IList> GetQueueWithIds(); } \ No newline at end of file diff --git a/Duplicati/Library/RestAPI/Abstractions/IScheduler.cs b/Duplicati/Library/RestAPI/Abstractions/ISchedulerService.cs similarity index 72% rename from Duplicati/Library/RestAPI/Abstractions/IScheduler.cs rename to Duplicati/Library/RestAPI/Abstractions/ISchedulerService.cs index 1140bf84a..398543cdb 100644 --- a/Duplicati/Library/RestAPI/Abstractions/IScheduler.cs +++ b/Duplicati/Library/RestAPI/Abstractions/ISchedulerService.cs @@ -20,25 +20,12 @@ // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; -using Duplicati.Library.Utility; -using Duplicati.Server; using Duplicati.Server.Serialization.Interface; namespace Duplicati.WebserverCore.Abstractions; -public interface IScheduler +public interface ISchedulerService { - /// - /// Initializes scheduler - /// - /// The worker thread - void Init(WorkerThread worker); - - /// - /// Gets the current ids in the scheduler queue - /// - IList> GetSchedulerQueueIds(); - /// /// Gets the current proposed schedule /// @@ -50,21 +37,11 @@ public interface IScheduler /// True if the call should block until the thread has exited, false otherwise void Terminate(bool wait); - /// - /// Subscribes to the event that is triggered when the schedule changes - /// - void SubScribeToNewSchedule(Action handler); - /// /// A snapshot copy of the current schedule list /// List> Schedule { get; } - /// - /// A snapshot copy of the current worker queue, that is items that are scheduled, but waiting for execution - /// - List WorkerQueue { get; } - /// /// Forces the scheduler to re-evaluate the order. /// Call this method if something changes diff --git a/Duplicati/Library/RestAPI/Abstractions/IWorkerThreadsManager.cs b/Duplicati/Library/RestAPI/Abstractions/IWorkerThreadsManager.cs deleted file mode 100644 index aceea26b9..000000000 --- a/Duplicati/Library/RestAPI/Abstractions/IWorkerThreadsManager.cs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. -#nullable enable -using System; -using Duplicati.Library.Utility; -using Duplicati.Server; - -namespace Duplicati.Library.RestAPI.Abstractions; - -public interface IWorkerThreadsManager -{ - void Spawn(Action item); - - Tuple? CurrentTask { get; } - WorkerThread? WorkerThread { get; } - void UpdateThrottleSpeeds(string? uploadSpeed, string? downloadSpeed); - - long AddTask(Runner.IRunnerData data, bool skipQueue = false); -} \ No newline at end of file diff --git a/Duplicati/Library/RestAPI/Database/Connection.cs b/Duplicati/Library/RestAPI/Database/Connection.cs index 90bb7bbda..2c4a8a91b 100644 --- a/Duplicati/Library/RestAPI/Database/Connection.cs +++ b/Duplicati/Library/RestAPI/Database/Connection.cs @@ -32,6 +32,7 @@ using Duplicati.Library.AutoUpdater; using System.Data; using Duplicati.Library.Main.Database; using System.Globalization; +using Microsoft.Extensions.DependencyInjection; #nullable enable @@ -47,6 +48,10 @@ namespace Duplicati.Server.Database private readonly Dictionary m_temporaryBackups = new Dictionary(); private readonly bool m_encryptSensitiveFields; private readonly EncryptedFieldHelper.KeyInstance? m_key; + private IServiceProvider? m_serviceProvider; + private INotificationUpdateService? m_notificationUpdateService; + private EventPollNotify? m_eventPollNotifyer; + private static readonly HashSet _encryptedFields = BackendLoader.Backends.SelectMany(x => x.SupportedCommands ?? []) .Concat(EncryptionLoader.Modules.SelectMany(x => x.SupportedCommands ?? [])) @@ -76,6 +81,22 @@ namespace Duplicati.Server.Database this.ApplicationSettings = new ServerSettings(this); } + /// + /// The service provider is used to resolve dependencies + /// + internal IServiceProvider? ServiceProvider => m_serviceProvider; + + /// + /// Set the service provider to be used for resolving dependencies + /// + /// The service provider + public void SetServiceProvider(IServiceProvider sp) + { + m_serviceProvider = sp; + m_notificationUpdateService = sp?.GetRequiredService(); + m_eventPollNotifyer = sp?.GetRequiredService(); + } + public bool IsEncryptingFields => m_encryptSensitiveFields; public void ReWriteAllFieldsIfEncryptionChanged() @@ -125,7 +146,7 @@ namespace Duplicati.Server.Database this.ApplicationSettings.PreloadSettingsHash = settingsHash; } - public void LogError(string backupid, string message, Exception ex) + public void LogError(string? backupid, string message, Exception ex) { lock (m_lock) { @@ -608,8 +629,8 @@ namespace Duplicati.Server.Database } } - FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); + m_notificationUpdateService?.IncrementLastDataUpdateId(); + m_eventPollNotifyer?.SignalNewEvent(); } private void AddOrUpdateBackup(IBackup item, bool updateSchedule, ISchedule? schedule) @@ -722,8 +743,8 @@ namespace Duplicati.Server.Database } tr.Commit(); - FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); + m_notificationUpdateService?.IncrementLastDataUpdateId(); + m_eventPollNotifyer?.SignalNewEvent(); } } } @@ -735,8 +756,8 @@ namespace Duplicati.Server.Database { AddOrUpdateSchedule(item, tr); tr.Commit(); - FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); + m_notificationUpdateService?.IncrementLastDataUpdateId(); + m_eventPollNotifyer?.SignalNewEvent(); } } @@ -800,8 +821,8 @@ namespace Duplicati.Server.Database } } - FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); + m_notificationUpdateService?.IncrementLastDataUpdateId(); + m_eventPollNotifyer?.SignalNewEvent(); } public void DeleteBackup(IBackup backup) @@ -820,8 +841,8 @@ namespace Duplicati.Server.Database lock (m_lock) DeleteFromDb("Schedule", ID); - FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); + m_notificationUpdateService?.IncrementLastDataUpdateId(); + m_eventPollNotifyer?.SignalNewEvent(); } public void DeleteSchedule(ISchedule schedule) @@ -906,21 +927,27 @@ namespace Duplicati.Server.Database return false; DeleteFromDb(typeof(Notification).Name, id); - FIXMEGlobal.DataConnection.ApplicationSettings.UnackedError = notifications.Any(x => x.ID != id && x.Type == Duplicati.Server.Serialization.NotificationType.Error); - FIXMEGlobal.DataConnection.ApplicationSettings.UnackedWarning = notifications.Any(x => x.ID != id && x.Type == Duplicati.Server.Serialization.NotificationType.Warning); + this.ApplicationSettings.UnackedError = notifications.Any(x => x.ID != id && x.Type == Duplicati.Server.Serialization.NotificationType.Error); + this.ApplicationSettings.UnackedWarning = notifications.Any(x => x.ID != id && x.Type == Duplicati.Server.Serialization.NotificationType.Warning); } - // Guard against dismissing notifications before the provider is initialized - if (FIXMEGlobal.Provider != null) - { - FIXMEGlobal.NotificationUpdateService.IncrementLastNotificationUpdateId(); - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); - } + m_notificationUpdateService?.IncrementLastNotificationUpdateId(); + m_eventPollNotifyer?.SignalNewEvent(); return true; } - public void RegisterNotification(Serialization.NotificationType type, string title, string message, Exception ex, string backupid, string action, string logid, string messageid, string logtag, Func conflicthandler) + public void RegisterNotification( + Serialization.NotificationType type, + string title, + string message, + Exception? ex, + string? backupid, + string action, + string? logid, + string? messageid, + string? logtag, + Func conflicthandler) { lock (m_lock) { @@ -949,13 +976,13 @@ namespace Duplicati.Server.Database OverwriteAndUpdateDb(null, null, [notification], false); if (type == Serialization.NotificationType.Error) - FIXMEGlobal.DataConnection.ApplicationSettings.UnackedError = true; + ApplicationSettings.UnackedError = true; else if (type == Serialization.NotificationType.Warning) - FIXMEGlobal.DataConnection.ApplicationSettings.UnackedWarning = true; + ApplicationSettings.UnackedWarning = true; } - FIXMEGlobal.NotificationUpdateService.IncrementLastNotificationUpdateId(); - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); + m_notificationUpdateService?.IncrementLastNotificationUpdateId(); + m_eventPollNotifyer?.SignalNewEvent(); } //Workaround to clean up the database after invalid settings update diff --git a/Duplicati/Library/RestAPI/Database/ServerSettings.cs b/Duplicati/Library/RestAPI/Database/ServerSettings.cs index dea19e8ec..c1fc38b30 100644 --- a/Duplicati/Library/RestAPI/Database/ServerSettings.cs +++ b/Duplicati/Library/RestAPI/Database/ServerSettings.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -29,6 +29,8 @@ using System.Text; using System.Security.Cryptography.X509Certificates; using Duplicati.Library.Utility; using Duplicati.Library.AutoUpdater; +using Microsoft.Extensions.DependencyInjection; +using Duplicati.WebserverCore.Abstractions; #nullable enable @@ -142,12 +144,13 @@ namespace Duplicati.Server.Database Value = n.Value }, Database.Connection.SERVER_SETTINGS_ID); - if (FIXMEGlobal.IsServerStarted) + var provider = databaseConnection.ServiceProvider; + if (provider != null) { - FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); + provider?.GetRequiredService()?.IncrementLastDataUpdateId(); + provider?.GetRequiredService()?.SignalNewEvent(); // If throttle options were changed, update now - FIXMEGlobal.WorkerThreadsManager.UpdateThrottleSpeeds(UploadSpeedLimit, DownloadSpeedLimit); + provider?.GetRequiredService()?.GetCurrentTask()?.UpdateThrottleSpeeds(UploadSpeedLimit, DownloadSpeedLimit); } // In case the usage reporter is enabled or disabled, refresh now @@ -602,7 +605,7 @@ namespace Duplicati.Server.Database lock (databaseConnection.m_lock) settings[CONST.UPDATE_CHECK_INTERVAL] = value; SaveSettings(); - FIXMEGlobal.UpdatePoller.Reschedule(); + databaseConnection?.ServiceProvider?.GetRequiredService()?.Reschedule(); } } diff --git a/Duplicati/Library/RestAPI/FIXMEGlobal.cs b/Duplicati/Library/RestAPI/FIXMEGlobal.cs index 9d3d66839..5a977f0c2 100644 --- a/Duplicati/Library/RestAPI/FIXMEGlobal.cs +++ b/Duplicati/Library/RestAPI/FIXMEGlobal.cs @@ -21,8 +21,6 @@ using Duplicati.Server; using System; -using Duplicati.Library.RestAPI.Abstractions; -using Duplicati.Library.Utility; using Duplicati.WebserverCore.Abstractions; using Microsoft.Extensions.DependencyInjection; using Duplicati.Library.Interface; @@ -66,15 +64,6 @@ namespace Duplicati.Library.RestAPI /// public static bool IsServerStarted => Provider != null; - /// - /// This is the working thread - /// - public static WorkerThread WorkThread => - Provider.GetRequiredService().WorkerThread; - - public static IWorkerThreadsManager WorkerThreadsManager => - Provider.GetRequiredService(); - public static Action StartOrStopUsageReporter; /// @@ -85,7 +74,7 @@ namespace Duplicati.Library.RestAPI /// /// This is the scheduling thread /// - public static IScheduler Scheduler => Provider.GetRequiredService(); + public static ISchedulerService Scheduler => Provider.GetRequiredService(); /// /// The log redirect handler diff --git a/Duplicati/Library/RestAPI/Runner.cs b/Duplicati/Library/RestAPI/Runner.cs index aa73ceabc..3480d1795 100644 --- a/Duplicati/Library/RestAPI/Runner.cs +++ b/Duplicati/Library/RestAPI/Runner.cs @@ -19,6 +19,8 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +#nullable enable + using System; using System.Linq; using System.Collections.Generic; @@ -26,51 +28,48 @@ using Duplicati.Library.Interface; using Duplicati.Server.Serialization; using Duplicati.Library.RestAPI; using Duplicati.Library.Utility; +using System.Threading.Tasks; namespace Duplicati.Server { public static class Runner { - public interface IRunnerData : Duplicati.Server.Serialization.Interface.IQueuedTask + public interface IRunnerData : Serialization.Interface.IQueuedTask { - Duplicati.Server.Serialization.Interface.IBackup Backup { get; } - IDictionary ExtraOptions { get; } - string[] FilterStrings { get; } - string[] ExtraArguments { get; } + Serialization.Interface.IBackup? Backup { get; } + IDictionary? ExtraOptions { get; } + string[]? FilterStrings { get; } + string[]? ExtraArguments { get; } int PageSize { get; } int PageOffset { get; } - void Stop(); - void Abort(); - void Pause(bool alsoTransfers); - void Resume(); - void UpdateThrottleSpeed(string uploadSpeed, string downloadSpeed); - void SetController(Duplicati.Library.Main.Controller controller); - DateTime? TaskStarted { get; set; } - DateTime? TaskFinished { get; set; } + void SetController(Library.Main.Controller? controller); } private class RunnerData : IRunnerData { private static long RunnerTaskID = 1; - public Duplicati.Server.Serialization.DuplicatiOperation Operation { get; internal set; } - public Duplicati.Server.Serialization.Interface.IBackup Backup { get; internal set; } - public IDictionary ExtraOptions { get; internal set; } - public string[] FilterStrings { get; internal set; } + public Func? OnStarting { get; set; } + public Func? OnFinished { get; set; } - public string BackupID { get { return Backup.ID; } } + public DuplicatiOperation Operation { get; internal set; } + public Serialization.Interface.IBackup? Backup { get; internal set; } + public IDictionary? ExtraOptions { get; internal set; } + public string[]? FilterStrings { get; internal set; } + + public string? BackupID { get { return Backup?.ID; } } public long TaskID { get { return m_taskID; } } - public string[] ExtraArguments { get; internal set; } + public string[]? ExtraArguments { get; internal set; } public int PageSize { get; internal set; } = 0; public int PageOffset { get; internal set; } = 0; public DateTime? TaskStarted { get; set; } public DateTime? TaskFinished { get; set; } - internal Duplicati.Library.Main.Controller Controller { get; set; } + internal Library.Main.Controller? Controller { get; set; } - public void SetController(Duplicati.Library.Main.Controller controller) + public void SetController(Library.Main.Controller? controller) { Controller = controller; } @@ -98,7 +97,7 @@ namespace Duplicati.Server public long OriginalUploadSpeed { get; set; } public long OriginalDownloadSpeed { get; set; } - public void UpdateThrottleSpeed(string uploadSpeed, string downloadSpeed) + public void UpdateThrottleSpeeds(string? uploadSpeed, string? downloadSpeed) { var controller = this.Controller; if (controller == null) @@ -113,14 +112,14 @@ namespace Duplicati.Server try { if (!string.IsNullOrWhiteSpace(uploadSpeed)) - server_upload_throttle = Duplicati.Library.Utility.Sizeparser.ParseSize(uploadSpeed, "kb"); + server_upload_throttle = Sizeparser.ParseSize(uploadSpeed, "kb"); } catch { } try { if (!string.IsNullOrWhiteSpace(downloadSpeed)) - server_download_throttle = Duplicati.Library.Utility.Sizeparser.ParseSize(downloadSpeed, "kb"); + server_download_throttle = Sizeparser.ParseSize(downloadSpeed, "kb"); } catch { } @@ -142,6 +141,9 @@ namespace Duplicati.Server { m_taskID = System.Threading.Interlocked.Increment(ref RunnerTaskID); } + + public Task Execute() + => Task.Run(() => Runner.Run(this, true)); } private class CustomRunnerTask : RunnerData @@ -164,13 +166,13 @@ namespace Duplicati.Server return new CustomRunnerTask(runner); } - public static IRunnerData CreateTask(Duplicati.Server.Serialization.DuplicatiOperation operation, Duplicati.Server.Serialization.Interface.IBackup backup, IDictionary extraOptions = null, string[] filterStrings = null, string[] extraArguments = null, int pageSize = 0, int pageOffset = 0) + public static IRunnerData CreateTask(DuplicatiOperation operation, Serialization.Interface.IBackup backup, IDictionary? extraOptions = null, string[]? filterStrings = null, string[]? extraArguments = null, int pageSize = 0, int pageOffset = 0) { return new RunnerData() { Operation = operation, Backup = backup, - ExtraOptions = extraOptions, + ExtraOptions = extraOptions ?? new Dictionary(), FilterStrings = filterStrings, ExtraArguments = extraArguments, PageSize = pageSize, @@ -178,15 +180,15 @@ namespace Duplicati.Server }; } - public static IRunnerData CreateListTask(Duplicati.Server.Serialization.Interface.IBackup backup, string[] filters, bool onlyPrefix, bool allVersions, bool folderContents, DateTime time) + public static IRunnerData CreateListTask(Serialization.Interface.IBackup backup, string[]? filters, bool onlyPrefix, bool allVersions, bool folderContents, DateTime time) { - var dict = new Dictionary(); + var dict = new Dictionary(); if (onlyPrefix) dict["list-prefix-only"] = "true"; if (allVersions) dict["all-versions"] = "true"; if (time.Ticks > 0) - dict["time"] = Duplicati.Library.Utility.Utility.SerializeDateTime(time.ToUniversalTime()); + dict["time"] = Utility.SerializeDateTime(time.ToUniversalTime()); if (folderContents) dict["list-folder-contents"] = "true"; @@ -197,19 +199,19 @@ namespace Duplicati.Server filters); } - public static IRunnerData CreateListFilesetsTask(Duplicati.Server.Serialization.Interface.IBackup backup, Dictionary extraOptions = null) + public static IRunnerData CreateListFilesetsTask(Serialization.Interface.IBackup backup, Dictionary? extraOptions = null) { return CreateTask( DuplicatiOperation.ListFilesets, backup, - extraOptions ?? new Dictionary()); + extraOptions ?? new Dictionary()); } - public static IRunnerData CreateListFolderContents(Duplicati.Server.Serialization.Interface.IBackup backup, string[] folders, DateTime time, int pageSize, int pageOffset) + public static IRunnerData CreateListFolderContents(Serialization.Interface.IBackup backup, string[] folders, DateTime time, int pageSize, int pageOffset) { - var dict = new Dictionary(); + var dict = new Dictionary(); if (time.Ticks > 0) - dict["time"] = Duplicati.Library.Utility.Utility.SerializeDateTime(time.ToUniversalTime()); + dict["time"] = Utility.SerializeDateTime(time.ToUniversalTime()); return CreateTask( DuplicatiOperation.ListFolderContents, @@ -220,9 +222,9 @@ namespace Duplicati.Server pageOffset: pageOffset); } - public static IRunnerData ListFileVersionsTask(Duplicati.Server.Serialization.Interface.IBackup backup, string[] filepaths, int pageSize, int pageOffset) + public static IRunnerData ListFileVersionsTask(Serialization.Interface.IBackup backup, string[] filepaths, int pageSize, int pageOffset) { - var dict = new Dictionary(); + var dict = new Dictionary(); return CreateTask( DuplicatiOperation.ListFileVersions, backup, @@ -232,11 +234,11 @@ namespace Duplicati.Server pageOffset: pageOffset); } - public static IRunnerData CreateSearchEntriesTask(Duplicati.Server.Serialization.Interface.IBackup backup, string[] filters, string[] folders, DateTime time, int pageSize, int pageOffset) + public static IRunnerData CreateSearchEntriesTask(Serialization.Interface.IBackup backup, string[] filters, string[] folders, DateTime time, int pageSize, int pageOffset) { - var dict = new Dictionary(); + var dict = new Dictionary(); if (time.Ticks > 0) - dict["time"] = Duplicati.Library.Utility.Utility.SerializeDateTime(time.ToUniversalTime()); + dict["time"] = Utility.SerializeDateTime(time.ToUniversalTime()); return CreateTask( DuplicatiOperation.SearchEntries, @@ -249,17 +251,17 @@ namespace Duplicati.Server } - public static IRunnerData CreateRestoreTask(Duplicati.Server.Serialization.Interface.IBackup backup, string[] filters, - DateTime time, string restoreTarget, bool overwrite, bool restore_permissions, - bool skip_metadata, string passphrase) + public static IRunnerData CreateRestoreTask(Serialization.Interface.IBackup backup, string[]? filters, + DateTime time, string? restoreTarget, bool overwrite, bool restore_permissions, + bool skip_metadata, string? passphrase) { - var dict = new Dictionary + var dict = new Dictionary { - ["time"] = Library.Utility.Utility.SerializeDateTime(time.ToUniversalTime()), - ["overwrite"] = overwrite ? Boolean.TrueString : Boolean.FalseString, - ["restore-permissions"] = restore_permissions ? Boolean.TrueString : Boolean.FalseString, - ["skip-metadata"] = skip_metadata ? Boolean.TrueString : Boolean.FalseString, - ["allow-passphrase-change"] = Boolean.TrueString + ["time"] = Utility.SerializeDateTime(time.ToUniversalTime()), + ["overwrite"] = overwrite ? bool.TrueString : bool.FalseString, + ["restore-permissions"] = restore_permissions ? bool.TrueString : bool.FalseString, + ["skip-metadata"] = skip_metadata ? bool.TrueString : bool.FalseString, + ["allow-passphrase-change"] = bool.TrueString }; if (!string.IsNullOrWhiteSpace(restoreTarget)) dict["restore-path"] = SpecialFolders.ExpandEnvironmentVariables(restoreTarget); @@ -272,26 +274,26 @@ namespace Duplicati.Server dict, filters); } - private class MessageSink : Duplicati.Library.Main.IMessageSink + private class MessageSink : Library.Main.IMessageSink { - private class ProgressState : Server.Serialization.Interface.IProgressEventData + private class ProgressState : Serialization.Interface.IProgressEventData { - private readonly string m_backupID; + private readonly string? m_backupID; private readonly long m_taskID; - internal Duplicati.Library.Main.BackendActionType m_backendAction; - internal string m_backendPath; + internal Library.Main.BackendActionType m_backendAction; + internal string? m_backendPath; internal long m_backendFileSize; internal long m_backendFileProgress; internal long m_backendSpeed; internal bool m_backendIsBlocking; - internal string m_currentFilename; + internal string? m_currentFilename; internal long m_currentFilesize; internal long m_currentFileoffset; internal bool m_currentFilecomplete; - internal Duplicati.Library.Main.OperationPhase m_phase; + internal Library.Main.OperationPhase m_phase; internal float m_overallProgress; internal long m_processedFileCount; internal long m_processedFileSize; @@ -299,7 +301,7 @@ namespace Duplicati.Server internal long m_totalFileSize; internal bool m_stillCounting; - public ProgressState(long taskId, string backupId) + public ProgressState(long taskId, string? backupId) { m_backupID = backupId; m_taskID = taskId; @@ -311,15 +313,15 @@ namespace Duplicati.Server } #region IProgressEventData implementation - public string BackupID { get { return m_backupID; } } + public string? BackupID { get { return m_backupID; } } public long TaskID { get { return m_taskID; } } public string BackendAction { get { return m_backendAction.ToString(); } } - public string BackendPath { get { return m_backendPath; } } + public string? BackendPath { get { return m_backendPath; } } public long BackendFileSize { get { return m_backendFileSize; } } public long BackendFileProgress { get { return m_backendFileProgress; } } public long BackendSpeed { get { return m_backendSpeed; } } public bool BackendIsBlocking { get { return m_backendIsBlocking; } } - public string CurrentFilename { get { return m_currentFilename; } } + public string? CurrentFilename { get { return m_currentFilename; } } public long CurrentFilesize { get { return m_currentFilesize; } } public long CurrentFileoffset { get { return m_currentFileoffset; } } public bool CurrentFilecomplete { get { return m_currentFilecomplete; } } @@ -334,16 +336,16 @@ namespace Duplicati.Server } private readonly ProgressState m_state; - private Duplicati.Library.Main.IBackendProgress m_backendProgress; - private Duplicati.Library.Main.IOperationProgress m_operationProgress; + private Library.Main.IBackendProgress? m_backendProgress; + private Library.Main.IOperationProgress? m_operationProgress; private readonly object m_lock = new object(); - public MessageSink(long taskId, string backupId) + public MessageSink(long taskId, string? backupId) { m_state = new ProgressState(taskId, backupId); } - public Server.Serialization.Interface.IProgressEventData Copy() + public Serialization.Interface.IProgressEventData Copy() { lock (m_lock) { @@ -395,6 +397,8 @@ namespace Duplicati.Server public static string GetCommandLine(IRunnerData data) { var backup = data.Backup; + if (backup == null) + throw new ArgumentNullException(nameof(backup)); var options = ApplyOptions(backup, GetCommonOptions()); if (data.ExtraOptions != null) @@ -416,22 +420,22 @@ namespace Duplicati.Server ); var cmd = new System.Text.StringBuilder(); - cmd.Append(Library.Utility.Utility.WrapAsCommandLine(new string[] { exe, "backup", backup.TargetURL }, false)); + cmd.Append(Utility.WrapAsCommandLine([exe, "backup", backup.TargetURL], false)); cmd.Append(" "); - cmd.Append(Library.Utility.Utility.WrapAsCommandLine(sources, true)); + cmd.Append(Utility.WrapAsCommandLine(sources, true)); // TODO: We should check each option to see if it is a path, and allow expansion on that foreach (var opt in options) - cmd.AppendFormat(" --{0}={1}", opt.Key, Library.Utility.Utility.WrapCommandLineElement(opt.Value, false)); + cmd.AppendFormat(" --{0}={1}", opt.Key, Utility.WrapCommandLineElement(opt.Value, false)); if (cf != null) foreach (var f in cf) - cmd.AppendFormat(" --{0}={1}", f.Include ? "include" : "exclude", Library.Utility.Utility.WrapCommandLineElement(f.Expression, true)); + cmd.AppendFormat(" --{0}={1}", f.Include ? "include" : "exclude", Utility.WrapCommandLineElement(f.Expression, true)); if (bf != null) foreach (var f in bf) - cmd.AppendFormat(" --{0}={1}", f.Include ? "include" : "exclude", Library.Utility.Utility.WrapCommandLineElement(f.Expression, true)); + cmd.AppendFormat(" --{0}={1}", f.Include ? "include" : "exclude", Utility.WrapCommandLineElement(f.Expression, true)); return cmd.ToString(); } @@ -439,6 +443,8 @@ namespace Duplicati.Server public static string[] GetCommandLineParts(IRunnerData data) { var backup = data.Backup; + if (backup == null) + throw new ArgumentNullException(nameof(backup)); var options = ApplyOptions(backup, GetCommonOptions()); if (data.ExtraOptions != null) @@ -474,7 +480,7 @@ namespace Duplicati.Server return parts.ToArray(); } - public static Duplicati.Library.Interface.IBasicResults Run(IRunnerData data, bool fromQueue) + public static IBasicResults? Run(IRunnerData data, bool fromQueue) { data.TaskStarted = DateTime.Now; if (data is CustomRunnerTask task) @@ -500,12 +506,11 @@ namespace Duplicati.Server } var backup = data.Backup; - if (backup.Metadata == null) - { - backup.Metadata = new Dictionary(); - } + if (backup == null) + throw new ArgumentNullException(nameof(backup)); - Duplicati.Library.Utility.TempFolder tempfolder = null; + backup.Metadata ??= new Dictionary(); + TempFolder? tempfolder = null; try { @@ -552,16 +557,17 @@ namespace Duplicati.Server catch { } ((RunnerData)data).Controller = controller; - data.UpdateThrottleSpeed(FIXMEGlobal.DataConnection.ApplicationSettings.UploadSpeedLimit, FIXMEGlobal.DataConnection.ApplicationSettings.DownloadSpeedLimit); + var appSettings = FIXMEGlobal.DataConnection.ApplicationSettings; + data.UpdateThrottleSpeeds(appSettings.UploadSpeedLimit, appSettings.DownloadSpeedLimit); // Pass on the provider, will be replaced if configured in the backup controller.SetSecretProvider(FIXMEGlobal.SecretProvider); if (backup.Metadata.ContainsKey("LastCompactFinished")) - controller.LastCompact = Library.Utility.Utility.DeserializeDateTime(backup.Metadata["LastCompactFinished"]); + controller.LastCompact = Utility.DeserializeDateTime(backup.Metadata["LastCompactFinished"]); if (backup.Metadata.ContainsKey("LastVacuumFinished")) - controller.LastVacuum = Library.Utility.Utility.DeserializeDateTime(backup.Metadata["LastVacuumFinished"]); + controller.LastVacuum = Utility.DeserializeDateTime(backup.Metadata["LastVacuumFinished"]); switch (data.Operation) { @@ -622,12 +628,12 @@ namespace Duplicati.Server } case DuplicatiOperation.CreateReport: { - using (var tf = new Duplicati.Library.Utility.TempFile()) + using (var tf = new TempFile()) { var r = controller.CreateLogDatabase(tf); var tempid = FIXMEGlobal.DataConnection.RegisterTempFile("create-bug-report", r.TargetPath, DateTime.Now.AddDays(3)); - if (string.Equals(tf, r.TargetPath, Library.Utility.Utility.ClientFilenameStringComparison)) + if (string.Equals(tf, r.TargetPath, Utility.ClientFilenameStringComparison)) tf.Protected = true; FIXMEGlobal.DataConnection.RegisterNotification( @@ -656,16 +662,18 @@ namespace Duplicati.Server case DuplicatiOperation.Delete: { - if (Library.Utility.Utility.ParseBoolOption(data.ExtraOptions.AsReadOnly(), "delete-remote-files")) - controller.DeleteAllRemoteFiles(); - - if (Library.Utility.Utility.ParseBoolOption(data.ExtraOptions.AsReadOnly(), "delete-local-db")) + if (data.ExtraOptions != null) { - string dbpath; - options.TryGetValue("dbpath", out dbpath); + if (Utility.ParseBoolOption(data.ExtraOptions.AsReadOnly(), "delete-remote-files")) + controller.DeleteAllRemoteFiles(); - if (!string.IsNullOrWhiteSpace(dbpath) && System.IO.File.Exists(dbpath)) - System.IO.File.Delete(dbpath); + if (Utility.ParseBoolOption(data.ExtraOptions.AsReadOnly(), "delete-local-db")) + { + options.TryGetValue("dbpath", out var dbpath); + + if (!string.IsNullOrWhiteSpace(dbpath) && System.IO.File.Exists(dbpath)) + System.IO.File.Delete(dbpath); + } } FIXMEGlobal.DataConnection.DeleteBackup(backup); FIXMEGlobal.Scheduler.Reschedule(); @@ -706,8 +714,9 @@ namespace Duplicati.Server } catch (Exception ex) { - FIXMEGlobal.DataConnection.LogError(data.Backup.ID, string.Format("Failed while executing {0} \"{1}\" (id: {2})", data.Operation, data.Backup.Name, data.Backup.ID), ex); - UpdateMetadataError(data.Backup, ex); + FIXMEGlobal.DataConnection.LogError(data.Backup?.ID, string.Format("Failed while executing {0} \"{1}\" (id: {2})", data.Operation, data.Backup?.Name, data.Backup?.ID), ex); + if (data.Backup != null) + UpdateMetadataError(data.Backup, ex); Library.UsageReporter.Reporter.Report(ex); if (!fromQueue) @@ -722,23 +731,26 @@ namespace Duplicati.Server } } - private static Duplicati.Library.Utility.TempFolder StoreTaskConfigAndGetTempFolder(IRunnerData data, Dictionary options) + private static TempFolder? StoreTaskConfigAndGetTempFolder(IRunnerData data, Dictionary options) { + if (data.Backup == null) + throw new ArgumentNullException(nameof(data.Backup)); + var all_tasks = string.Equals(options["store-task-config"], "all", StringComparison.OrdinalIgnoreCase) || string.Equals(options["store-task-config"], "*", StringComparison.OrdinalIgnoreCase); - var this_task = Duplicati.Library.Utility.Utility.ParseBool(options["store-task-config"], false); + var this_task = Utility.ParseBool(options["store-task-config"], false); options.Remove("store-task-config"); - Duplicati.Library.Utility.TempFolder tempfolder = null; + TempFolder? tempfolder = null; if (all_tasks || this_task) { - tempfolder = new Duplicati.Library.Utility.TempFolder(); + tempfolder = new TempFolder(); var temppath = System.IO.Path.Combine(tempfolder, "task-setup.json"); - using (var tempfile = Duplicati.Library.Utility.TempFile.WrapExistingFile(temppath)) + using (var tempfile = TempFile.WrapExistingFile(temppath)) { - object taskdata = null; + object? taskdata = null; if (all_tasks) - taskdata = FIXMEGlobal.DataConnection.Backups.Where(x => !x.IsTemporary).Select(x => FIXMEGlobal.DataConnection.PrepareBackupForExport(FIXMEGlobal.DataConnection.GetBackup(x.ID))); + taskdata = FIXMEGlobal.DataConnection.Backups.Where(x => !x.IsTemporary).Select(x => FIXMEGlobal.DataConnection.PrepareBackupForExport(FIXMEGlobal.DataConnection.GetBackup(x.ID)!)); else taskdata = new[] { FIXMEGlobal.DataConnection.PrepareBackupForExport(data.Backup) }; @@ -748,7 +760,7 @@ namespace Duplicati.Server tempfile.Protected = true; - options.TryGetValue("control-files", out string controlfiles); + options.TryGetValue("control-files", out var controlfiles); if (string.IsNullOrWhiteSpace(controlfiles)) controlfiles = tempfile; @@ -769,7 +781,7 @@ namespace Duplicati.Server if (!backup.IsTemporary) FIXMEGlobal.DataConnection.SetMetadata(backup.Metadata, long.Parse(backup.ID), null); - string messageid = null; + string? messageid = null; if (ex is UserInformationException exception) messageid = exception.HelpID; @@ -973,23 +985,20 @@ namespace Duplicati.Server return true; } - private static void DisableModule(string module, Dictionary options) + private static void DisableModule(string module, Dictionary options) { - string disabledModules; - string enabledModules; - - if (options.TryGetValue("enable-module", out enabledModules)) + if (options.TryGetValue("enable-module", out var enabledModules)) { var emods = (enabledModules ?? "").Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries); options["enable-module"] = string.Join(",", emods.Where(x => module.Equals(x, StringComparison.OrdinalIgnoreCase))); } - options.TryGetValue("disable-module", out disabledModules); + options.TryGetValue("disable-module", out var disabledModules); var mods = (disabledModules ?? "").Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries); options["disable-module"] = string.Join(",", mods.Union(new string[] { module }).Distinct(StringComparer.OrdinalIgnoreCase)); } - internal static Dictionary ApplyOptions(Duplicati.Server.Serialization.Interface.IBackup backup, Dictionary options) + internal static Dictionary ApplyOptions(Serialization.Interface.IBackup backup, Dictionary options) { options["backup-name"] = backup.Name; options["dbpath"] = backup.DBPath; @@ -1023,7 +1032,7 @@ namespace Duplicati.Server return options; } - private static Library.Utility.IFilter ApplyFilter(Serialization.Interface.IBackup backup, Library.Utility.IFilter filter) + private static IFilter? ApplyFilter(Serialization.Interface.IBackup backup, IFilter? filter) { var f2 = backup.Filters; if (f2 != null && f2.Length > 0) @@ -1035,24 +1044,24 @@ namespace Duplicati.Server ? SpecialFolders.ExpandEnvironmentVariablesRegexp(n.Expression) : SpecialFolders.ExpandEnvironmentVariables(n.Expression) orderby n.Order - select (Library.Utility.IFilter)(new Library.Utility.FilterExpression(exp, n.Include))) - .Aggregate((a, b) => Library.Utility.FilterExpression.Combine(a, b)); + select (IFilter)new FilterExpression(exp, n.Include)) + .Aggregate((a, b) => FilterExpression.Combine(a, b)); - filter = Library.Utility.FilterExpression.Combine(filter, nf); + filter = FilterExpression.Combine(filter, nf); } return filter; } - public static Dictionary GetCommonOptions() + public static Dictionary GetCommonOptions() { return (from n in FIXMEGlobal.DataConnection.Settings where TestIfOptionApplies() - select n).ToDictionary(k => k.Name.StartsWith("--", StringComparison.Ordinal) ? k.Name.Substring(2) : k.Name, k => k.Value); + select n).ToDictionary(k => k.Name.StartsWith("--", StringComparison.Ordinal) ? k.Name.Substring(2) : k.Name, k => (string?)k.Value); } - private static Duplicati.Library.Utility.IFilter GetCommonFilter() + private static IFilter? GetCommonFilter() { var filters = FIXMEGlobal.DataConnection.Filters; if (filters == null || filters.Length == 0) @@ -1062,8 +1071,8 @@ namespace Duplicati.Server (from n in filters orderby n.Order let exp = Environment.ExpandEnvironmentVariables(n.Expression) - select (Duplicati.Library.Utility.IFilter)(new Duplicati.Library.Utility.FilterExpression(exp, n.Include))) - .Aggregate((a, b) => Duplicati.Library.Utility.FilterExpression.Combine(a, b)); + select (IFilter)new FilterExpression(exp, n.Include)) + .Aggregate((a, b) => FilterExpression.Combine(a, b)); } } } diff --git a/Duplicati/Library/RestAPI/Scheduler.cs b/Duplicati/Library/RestAPI/Scheduler.cs index fc744e41c..388a850b7 100644 --- a/Duplicati/Library/RestAPI/Scheduler.cs +++ b/Duplicati/Library/RestAPI/Scheduler.cs @@ -19,6 +19,8 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +#nullable enable + using Duplicati.Server.Serialization.Interface; using System; @@ -28,6 +30,8 @@ using System.Linq; using System.Threading; using Duplicati.Library.Utility; using Duplicati.Library.RestAPI; +using System.Threading.Tasks; +using Duplicati.WebserverCore.Abstractions; // TODO: Rewrite this class. // It should just signal what new backups to run, and not mix with the worker thread. @@ -39,7 +43,7 @@ namespace Duplicati.Server /// public class Scheduler { - private static readonly string LOGTAG = Duplicati.Library.Logging.Log.LogTagFromType(); + private static readonly string LOGTAG = Library.Logging.Log.LogTagFromType(); /// /// The thread that runs the scheduler @@ -51,11 +55,6 @@ namespace Duplicati.Server /// private volatile bool m_terminate; - /// - /// The worker thread that is invoked to do work - /// - private WorkerThread m_worker; - /// /// The wait event /// @@ -67,9 +66,9 @@ namespace Duplicati.Server private readonly object m_lock = new object(); /// - /// An event that is raised when the schedule changes + /// The queue runner service /// - public event EventHandler NewSchedule; + private readonly IQueueRunnerService m_queueRunnerService; /// /// The currently scheduled items @@ -79,41 +78,24 @@ namespace Duplicati.Server /// /// List of update tasks, used to set the timestamp on the schedule once completed /// - private Dictionary> m_updateTasks; + private Dictionary> m_updateTasks; /// /// Constructs a new scheduler /// - public Scheduler() + public Scheduler(IQueueRunnerService queueRunnerService) { - } - - /// - /// Initializes scheduler - /// - /// The worker thread - public void Init(WorkerThread worker) - { - m_worker = worker; + m_queueRunnerService = queueRunnerService; m_thread = new Thread(new ThreadStart(Runner)); - m_worker.CompletedWork += OnCompleted; - m_worker.StartingWork += OnStartingWork; - m_schedule = new KeyValuePair[0]; + m_schedule = []; m_terminate = false; m_event = new AutoResetEvent(false); - m_updateTasks = new Dictionary>(); + m_updateTasks = new Dictionary>(); m_thread.IsBackground = true; m_thread.Name = "TaskScheduler"; m_thread.Start(); } - public IList> GetSchedulerQueueIds() - { - return (from n in WorkerQueue - where n.Backup != null - select new Tuple(n.TaskID, n.Backup.ID)).ToList(); - } - public IList> GetProposedSchedule() { return ( @@ -147,14 +129,6 @@ namespace Duplicati.Server } } - /// - /// A snapshot copy of the current worker queue, that is items that are scheduled, but waiting for execution - /// - public List WorkerQueue - { - get { return m_worker?.CurrentTasks?.Where(t => t != null)?.ToList() ?? []; } - } - /// /// Terminates the thread. Any items still in queue will be removed /// @@ -233,9 +207,9 @@ namespace Duplicati.Server return res; } - private void OnCompleted(WorkerThread worker, Runner.IRunnerData task) + private Task OnCompleted(Runner.IRunnerData task) { - Tuple t = null; + Tuple? t = null; lock (m_lock) { if (task != null && m_updateTasks.TryGetValue(task, out t)) @@ -248,24 +222,26 @@ namespace Duplicati.Server t.Item1.LastRun = t.Item3; FIXMEGlobal.DataConnection.AddOrUpdateSchedule(t.Item1); } + + return Task.CompletedTask; } - private void OnStartingWork(WorkerThread worker, Runner.IRunnerData task) + private Task OnStartingWork(Runner.IRunnerData task) { if (task is null) - { - return; - } + return Task.CompletedTask; lock (m_lock) { - if (m_updateTasks.TryGetValue(task, out Tuple scheduleInfo)) + if (m_updateTasks.TryGetValue(task, out var scheduleInfo)) { // Item2 is the scheduled start time (Time in the Schedule table). // Item3 is the actual start time (LastRun in the Schedule table). m_updateTasks[task] = Tuple.Create(scheduleInfo.Item1, scheduleInfo.Item2, DateTime.UtcNow); } } + + return Task.CompletedTask; } /// @@ -328,33 +304,33 @@ namespace Duplicati.Server .Select(x => x.ToString())) { //See if it is already queued - var tmplst = from n in m_worker.CurrentTasks - where n.Operation == Duplicati.Server.Serialization.DuplicatiOperation.Backup - select n.Backup; - var tastTemp = m_worker.CurrentTask; + var tmplst = from n in m_queueRunnerService.GetCurrentTasks() + where n.Operation == Serialization.DuplicatiOperation.Backup + select n.BackupID; + var tastTemp = m_queueRunnerService.GetCurrentTask(); if (tastTemp != null && tastTemp.Operation == - Duplicati.Server.Serialization.DuplicatiOperation.Backup) - tmplst = tmplst.Union(new[] { tastTemp.Backup }); + Serialization.DuplicatiOperation.Backup) + tmplst = tmplst.Union(new[] { tastTemp.BackupID }); //If it is not already in queue, put it there - if (!tmplst.Any(x => x.ID == id)) + if (!tmplst.Any(x => x == id)) { var entry = FIXMEGlobal.DataConnection.GetBackup(id); if (entry != null) { - Dictionary options = Duplicati.Server.Runner.GetCommonOptions(); - Duplicati.Server.Runner.ApplyOptions(entry, options); - if ((new Duplicati.Library.Main.Options(options)).DisableOnBattery && - (Duplicati.Library.Utility.Power.PowerSupply.GetSource() == - Duplicati.Library.Utility.Power.PowerSupply.Source.Battery)) + var options = Server.Runner.GetCommonOptions(); + Server.Runner.ApplyOptions(entry, options); + if (new Library.Main.Options(options).DisableOnBattery && + (Library.Utility.Power.PowerSupply.GetSource() == + Library.Utility.Power.PowerSupply.Source.Battery)) { - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, + Library.Logging.Log.WriteInformationMessage(LOGTAG, "BackupDisabledOnBattery", "Scheduled backup disabled while on battery power."); } else { - Dictionary taskOptions = null; + Dictionary? taskOptions = null; try { var nextRun = GetNextValidTime(start, @@ -362,15 +338,17 @@ namespace Duplicati.Server Math.Max(DateTime.UtcNow.AddSeconds(1).Ticks, start.AddSeconds(1).Ticks), DateTimeKind.Utc), sc.Repeat, sc.AllowedDays, timeZoneInfo); - taskOptions = new Dictionary() + taskOptions = new Dictionary() { { "next-scheduled-run", Utility.SerializeDateTime(nextRun.ToUniversalTime()) } }; } catch { } - jobsToRun.Add(Server.Runner.CreateTask( - Serialization.DuplicatiOperation.Backup, entry, taskOptions)); + var job = Server.Runner.CreateTask(Serialization.DuplicatiOperation.Backup, entry, taskOptions); + job.OnStarting = () => OnStartingWork(job); + job.OnFinished = (_) => OnCompleted(job); + jobsToRun.Add(job); } } } @@ -391,7 +369,7 @@ namespace Duplicati.Server continue; } - Server.Runner.IRunnerData lastJob = jobsToRun.LastOrDefault(); + var lastJob = jobsToRun.LastOrDefault(); if (lastJob != null) { lock (m_lock) @@ -403,7 +381,7 @@ namespace Duplicati.Server } foreach (var job in jobsToRun) - m_worker.AddTask(job); + m_queueRunnerService.AddTask(job); if (start < DateTime.UtcNow) { @@ -428,11 +406,6 @@ namespace Duplicati.Server foreach (var c in (from n in scheduled where !existing.ContainsKey(n.Key) select n.Key).ToArray()) scheduled.Remove(c); - //Raise event if needed - // TODO: This triggers a new data event and a reconnect with long-poll - if (NewSchedule != null) - NewSchedule(this, null); - int waittime = 0; //Figure out a sensible amount of time to sleep the thread diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index c83243532..7f54fb5a4 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -1397,7 +1397,7 @@ namespace Duplicati.Library.Utility /// The wrapped commandline element. /// The argument to wrap. /// A flag indicating if environment variables are allowed to be expanded - public static string WrapCommandLineElement(string arg, bool allowEnvExpansion) + public static string WrapCommandLineElement(string? arg, bool allowEnvExpansion) { if (string.IsNullOrWhiteSpace(arg)) return arg; diff --git a/Duplicati/Library/Utility/WorkerThread.cs b/Duplicati/Library/Utility/WorkerThread.cs deleted file mode 100644 index 285aac182..000000000 --- a/Duplicati/Library/Utility/WorkerThread.cs +++ /dev/null @@ -1,401 +0,0 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -using System; -using System.Collections.Generic; -using System.Text; -using System.Threading; - - -// TODO: Delete this class. -// It is essentially a queue that is processed by a worker thread, and can be implemented using a BlockingCollection or similar. - -namespace Duplicati.Library.Utility -{ - /// - /// Class to encapsulate a thread that runs a list of queued operations - /// - /// The type to operate on - public class WorkerThread where Tx : class - { - /// - /// Locking object for shared data - /// - private readonly object m_lock = new object(); - /// - /// The wait event - /// - private readonly AutoResetEvent m_event; - /// - /// The internal list of tasks to perform - /// - private Queue m_tasks; - /// - /// A flag used to terminate the thread - /// - private volatile bool m_terminate; - /// - /// The coordinating thread - /// - private Thread m_thread; - - /// - /// A value indicating if the coordinating thread is running - /// - private volatile bool m_active; - - /// - /// The current task being processed - /// - private Tx m_currentTask; - /// - /// A callback that performs the actual work on the item - /// - private readonly Action m_delegate; - - /// - /// An event that is raised when the runner state changes - /// - public event Action, RunState> WorkerStateChanged; - - /// - /// Event that occurs when a new operation is being processed - /// - public event Action, Tx> StartingWork; - /// - /// Event that occurs when an operation has completed - /// - public event Action, Tx> CompletedWork; - /// - /// Event that occurs when an error is detected - /// - public event Action, Tx, Exception> OnError; - /// - /// An event that occurs when a new task is added to the queue or an existing one is removed - /// - public event Action> WorkQueueChanged; - - /// - /// The internal state - /// - private volatile RunState m_state; - - /// - /// The states the scheduler can take - /// - public enum RunState - { - /// - /// The program is running as normal - /// - Run, - /// - /// The program is suspended by the user - /// - Paused - } - - /// - /// Constructs a new WorkerThread - /// - /// The callback that performs the work - public WorkerThread(Action item, bool paused) - { - m_delegate = item; - m_event = new AutoResetEvent(paused); - m_terminate = false; - m_tasks = new Queue(); - m_state = paused ? WorkerThread.RunState.Paused : WorkerThread.RunState.Run; - - m_thread = new Thread(new ThreadStart(Runner)); - m_thread.IsBackground = true; - m_thread.Name = "WorkerThread<" + typeof(Tx).Name + ">"; - m_thread.Start(); - } - - /// - /// Gets a copy of the current queue - /// - public List CurrentTasks - { - get - { - lock (m_lock) - return new List(m_tasks); - } - - } - - /// - /// Gets a value indicating if the worker is running - /// - public bool Active - { - get { return m_active; } - } - - /// - /// Adds a task to the queue - /// - /// The task to add - public void AddTask(Tx task) - { - lock (m_lock) - { - m_tasks.Enqueue(task); - m_event.Set(); - } - - if (WorkQueueChanged != null) - WorkQueueChanged(this); - } - - /// - /// An overloaded AddTask method that allows a task to skip to the front of a queue - /// It does this by creating a new queue, adding the new task first, and then adding - /// all the old tasks to the new queue. It's cleaner to use a linked list, - /// but the performance difference is negligible on such a small queue. - /// - /// Task. - /// If set to true skip queue. - public void AddTask(Tx task, bool skipQueue) - { - if (!skipQueue) - { - // Fall back to default AddTask method - AddTask(task); - return; - } - - lock (m_lock) - { - Queue newQueue = new Queue(); - newQueue.Enqueue(task); - while (m_tasks.Count > 0) - { - Tx n = m_tasks.Dequeue(); - newQueue.Enqueue(n); - } - m_tasks = newQueue; - m_event.Set(); - } - - if (WorkQueueChanged != null) - WorkQueueChanged(this); - } - - - /// - /// Removes a task from the queue, does not remove the task if it is currently running - /// - /// The task to remove - public void RemoveTask(Tx task) - { - lock (m_lock) - { - Queue tmp = new Queue(); - while (m_tasks.Count > 0) - { - Tx n = m_tasks.Dequeue(); - if (n != task) - tmp.Enqueue(n); - } - - m_tasks = tmp; - } - - if (WorkQueueChanged != null) - WorkQueueChanged(this); - } - - /// - /// This will clear the pending queue - /// True if the current running thread should be aborted - /// - public void ClearQueue(bool abortThread) - { - lock (m_lock) - m_tasks.Clear(); - - if (abortThread) - { - try - { - m_thread.Interrupt(); - m_thread.Join(500); - } - catch - { - } - - m_thread = new Thread(new ThreadStart(Runner)); - m_thread.Start(); - } - } - - /// - /// Gets a reference to the currently executing task. - /// BEWARE: This is not protected by a mutex, DO NOT MODIFY IT!!!! - /// - public Tx CurrentTask - { - get - { - return m_currentTask; - } - } - - /// - /// Terminates the thread. Any items still in queue will be removed - /// - /// True if the call should block until the thread has exited, false otherwise - public void Terminate(bool wait) - { - m_terminate = true; - m_event.Set(); - - if (wait) - m_thread.Join(); - } - - /// - /// This is the thread entry point - /// - private void Runner() - { - while (!m_terminate) - { - m_currentTask = null; - - lock (m_lock) - if (m_state == WorkerThread.RunState.Run && m_tasks.Count > 0) - m_currentTask = m_tasks.Dequeue(); - - if (m_currentTask == null && !m_terminate) - { - if (m_state == WorkerThread.RunState.Run) - m_event.WaitOne(); //Sleep until signaled - else - { - if (WorkerStateChanged != null) - WorkerStateChanged(this, m_state); - - //Sleep for brief periods, until signaled - while (!m_terminate && m_state != WorkerThread.RunState.Run) - m_event.WaitOne(1000 * 60 * 5, false); - - //If we were not terminated, we are now ready to run - if (!m_terminate) - { - m_state = WorkerThread.RunState.Run; - if (WorkerStateChanged != null) - WorkerStateChanged(this, m_state); - } - } - } - - if (m_terminate) - return; - - if (m_currentTask == null && m_state == WorkerThread.RunState.Run) - lock (m_lock) - if (m_tasks.Count > 0) - m_currentTask = m_tasks.Dequeue(); - - if (m_currentTask == null) - continue; - - if (StartingWork != null) - StartingWork(this, m_currentTask); - - try - { - m_active = true; - m_delegate(m_currentTask); - } - catch (Exception ex) - { - //TODO: Here where Thread.ResetAbort() was called we shall integrate the CancelationToken pattern. - if (OnError != null) - try { OnError(this, m_currentTask, ex); } - catch - { - // ignored - } - } - finally - { - //TODO: Here where Thread.ResetAbort() was called we shall integrate the CancelationToken pattern. - m_active = false; - } - - var task = m_currentTask; - m_currentTask = null; - - if (CompletedWork != null) - try { CompletedWork(this, task); } - catch (Exception ex) - { - try { OnError(this, task, ex); } - catch - { - // ignored - } - } - } - } - - /// - /// Gets the current run state - /// - public RunState State { get { return m_state; } } - - /// - /// Instructs Duplicati to run scheduled backups - /// - public void Resume() - { - m_state = RunState.Run; - m_event.Set(); - } - - /// - /// Instructs Duplicati to pause scheduled backups - /// - public void Pause() - { - m_state = RunState.Paused; - m_event.Set(); - } - - /// - /// Waits the specified number of milliseconds for the thread to terminate - /// - /// The number of milliseconds to wait - /// True if the thread is terminated, false if a timeout occured - public bool Join(int millisecondTimeout) - { - if (m_thread != null) - return m_thread.Join(millisecondTimeout); - return true; - } - } -} \ No newline at end of file diff --git a/Duplicati/Server/Duplicati.Server.Serialization/Interface/IQueuedTask.cs b/Duplicati/Server/Duplicati.Server.Serialization/Interface/IQueuedTask.cs index 385287292..cb95b681a 100644 --- a/Duplicati/Server/Duplicati.Server.Serialization/Interface/IQueuedTask.cs +++ b/Duplicati/Server/Duplicati.Server.Serialization/Interface/IQueuedTask.cs @@ -17,16 +17,75 @@ // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -namespace Duplicati.Server.Serialization.Interface -{ - public interface IQueuedTask - { - long TaskID { get; } - string BackupID { get; } - Duplicati.Server.Serialization.DuplicatiOperation Operation { get; } - - } -} - +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.Threading.Tasks; + +namespace Duplicati.Server.Serialization.Interface; + +/// +/// Represents a queued task. +/// +public interface IQueuedTask +{ + /// + /// The task ID. + /// + long TaskID { get; } + /// + /// The backup ID, if applicable. + /// + string? BackupID { get; } + /// + /// The operation type of the task. + /// + DuplicatiOperation Operation { get; } + /// + /// Callback to be executed when the task is starting. + /// + Func? OnStarting { get; set; } + /// + /// Callback to be executed when the task is finished. + /// If the task completes successfully, the exception parameter will be null. + /// + Func? OnFinished { get; set; } + + /// + /// That action that performs the task. + /// + Task Execute(); + /// + /// Updates the throttle speeds for the task. + /// + /// The upload speed to set. + /// The download speed to set. + void UpdateThrottleSpeeds(string? uploadSpeed, string? downloadSpeed); + /// + /// The time when the task was starting to execute. + /// + DateTime? TaskStarted { get; set; } + /// + /// The time when the task was finished executing. + /// + DateTime? TaskFinished { get; set; } + /// + /// Stops the task. + /// + void Stop(); + /// + /// Aborts the task. + /// + void Abort(); + /// + /// Pauses the task. + /// + /// If true, also pauses transfers. + void Pause(bool alsoTransfers); + /// + /// Resumes the task. + /// + void Resume(); +} diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index 55c734049..933cca135 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -102,7 +102,7 @@ namespace Duplicati.Server /// /// This is the scheduling thread /// - public static IScheduler Scheduler { get => FIXMEGlobal.Scheduler; } + public static ISchedulerService Scheduler { get => FIXMEGlobal.Scheduler; } /// /// The thread running the ping-pong handler @@ -243,6 +243,7 @@ namespace Duplicati.Server var crashed = false; var terminated = false; + IQueueRunnerService queueRunner = null; try { DataConnection = GetDatabaseConnection(commandlineOptions, silentConsole); @@ -266,13 +267,14 @@ namespace Duplicati.Server DuplicatiWebserver = StartWebServer(commandlineOptions, DataConnection).Await(); + queueRunner = DuplicatiWebserver.Provider.GetRequiredService(); + DataConnection.SetServiceProvider(DuplicatiWebserver.Provider); + UpdatePoller.Init(Library.Utility.Utility.ParseBoolOption(commandlineOptions, DISABLE_UPDATE_CHECK_OPTION)); SetPurgeTempFilesTimer(commandlineOptions); - LiveControl.StateChanged = LiveControl_StateChanged; - - SetWorkerThread(); + LiveControl.StateChanged = (e) => { LiveControl_StateChanged(queueRunner, DataConnection, StatusEventNotifyer, e); }; if (Library.Utility.Utility.ParseBoolOption(commandlineOptions, PING_PONG_KEEPALIVE_OPTION)) { @@ -363,7 +365,7 @@ namespace Duplicati.Server () => { if (ShutdownModernWebserver != null) ShutdownModernWebserver(); }, () => UpdatePoller?.Terminate(), () => Scheduler?.Terminate(true), - () => FIXMEGlobal.WorkThread?.Terminate(true), + () => queueRunner?.Terminate(true), () => ApplicationInstance?.Dispose(), () => PurgeTempFilesTimer?.Dispose(), () => Library.UsageReporter.Reporter.ShutDown(), @@ -432,36 +434,6 @@ namespace Duplicati.Server return server; } - private static void SetWorkerThread() - { - FIXMEGlobal.WorkerThreadsManager.Spawn(x => { Runner.Run(x, true); }); - FIXMEGlobal.WorkThread.StartingWork += (worker, task) => - { - SignalNewEvent(null, null); - task.TaskStarted = DateTime.Now; - }; - FIXMEGlobal.WorkThread.CompletedWork += (worker, task) => - { - SignalNewEvent(null, null); - FIXMEGlobal.Provider.GetRequiredService()?.AddTaskResult(new CachedTaskResult(task.TaskID, task.BackupID, task.TaskStarted, task.TaskFinished ?? DateTime.Now, null)); - }; - FIXMEGlobal.WorkThread.WorkQueueChanged += (worker) => { SignalNewEvent(null, null); }; - FIXMEGlobal.Scheduler.SubScribeToNewSchedule(() => SignalNewEvent(null, null)); - FIXMEGlobal.WorkThread.OnError += (worker, task, exception) => - { - DataConnection.LogError(task?.BackupID, "Error in worker", exception); - FIXMEGlobal.Provider.GetRequiredService()?.AddTaskResult(new CachedTaskResult(task.TaskID, task.BackupID, task.TaskStarted, task.TaskFinished ?? DateTime.Now, exception)); - }; - - var lastScheduleId = FIXMEGlobal.NotificationUpdateService.LastDataUpdateId; - StatusEventNotifyer.NewEvent += (sender, e) => - { - if (lastScheduleId == FIXMEGlobal.NotificationUpdateService.LastDataUpdateId) return; - lastScheduleId = FIXMEGlobal.NotificationUpdateService.LastDataUpdateId; - Scheduler.Reschedule(); - }; - } - private static void SetPurgeTempFilesTimer(Dictionary commandlineOptions) { var lastPurge = new DateTime(0); @@ -881,23 +853,22 @@ namespace Duplicati.Server /// This event handler updates the trayicon menu with the current state of the runner. /// /// - private static void LiveControl_StateChanged(LiveControls.LiveControlEvent e) + private static void LiveControl_StateChanged(IQueueRunnerService queueRunnerService, Connection connection, EventPollNotify eventPollNotify, LiveControls.LiveControlEvent e) { - var worker = FIXMEGlobal.WorkThread; - var appSettings = FIXMEGlobal.DataConnection.ApplicationSettings; + var appSettings = connection.ApplicationSettings; switch (e.State) { case LiveControls.LiveControlState.Paused: { - worker.Pause(); - worker.CurrentTask?.Pause(e.TransfersPaused); + queueRunnerService.Pause(); + queueRunnerService.GetCurrentTask()?.Pause(e.TransfersPaused); appSettings.PausedUntil = e.WaitTimeExpiration; break; } case LiveControls.LiveControlState.Running: { - worker.Resume(); - worker.CurrentTask?.Resume(); + queueRunnerService.Resume(); + queueRunnerService.GetCurrentTask()?.Resume(); appSettings.PausedUntil = null; break; } @@ -906,7 +877,7 @@ namespace Duplicati.Server break; } - StatusEventNotifyer.SignalNewEvent(); + eventPollNotify.SignalNewEvent(); } diff --git a/Duplicati/WebserverCore/Dto/ServerStatusDto.cs b/Duplicati/WebserverCore/Dto/ServerStatusDto.cs index 8426d0675..97b09011a 100644 --- a/Duplicati/WebserverCore/Dto/ServerStatusDto.cs +++ b/Duplicati/WebserverCore/Dto/ServerStatusDto.cs @@ -30,7 +30,7 @@ public sealed record ServerStatusDto /// /// Gets or sets the active task. /// - public required Tuple? ActiveTask { get; init; } + public required Tuple? ActiveTask { get; init; } /// /// Gets or sets the state of the program. @@ -40,7 +40,7 @@ public sealed record ServerStatusDto /// /// Gets the IDs of the tasks in the scheduler queue. /// - public required IList> SchedulerQueueIds { get; init; } = []; + public required IList> SchedulerQueueIds { get; init; } = []; /// /// Gets or sets the proposed schedule. diff --git a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs index 038a1a429..acad0cfac 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs @@ -21,7 +21,6 @@ using System.Text.Json; using System.Text.Json.Serialization; using Duplicati.Library.Interface; -using Duplicati.Library.RestAPI.Abstractions; using Duplicati.Server; using Duplicati.Server.Database; using Duplicati.Server.Serialization; @@ -86,8 +85,8 @@ public class BackupGet : IEndpointV1 => ExecuteGetIsdbUsedElsewhere(GetBackup(connection, id))) .RequireAuthorization(); - group.MapGet("/backup/{id}/isactive", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id) - => ExecuteGetIsActive(workerThreadsManager, GetBackup(connection, id))) + group.MapGet("/backup/{id}/isactive", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id) + => ExecuteGetIsActive(queueRunnerService, GetBackup(connection, id))) .RequireAuthorization(); } @@ -152,7 +151,7 @@ public class BackupGet : IEndpointV1 if (!allVersions) time = Library.Utility.Timeparser.ParseTimeInterval(timestring, DateTime.Now); - var r = Runner.Run(Runner.CreateListTask(backup, [filter], prefixOnly, allVersions, folderContents, time), false) as Duplicati.Library.Interface.IListResults; + var r = Runner.Run(Runner.CreateListTask(backup, filter == null ? null : [filter], prefixOnly, allVersions, folderContents, time), false) as Duplicati.Library.Interface.IListResults; if (r == null) throw new ServerErrorException("No result from list operation"); @@ -205,7 +204,7 @@ public class BackupGet : IEndpointV1 private static IEnumerable ExecuteGetFilesets(IBackup bk, bool includeMetadata, bool fromRemoteOnly) { - var extra = new Dictionary + var extra = new Dictionary { ["list-sets-only"] = "true" }; @@ -304,17 +303,14 @@ public class BackupGet : IEndpointV1 private static Dto.IsDbUsedElsewhereDto ExecuteGetIsdbUsedElsewhere(IBackup bk) => new Dto.IsDbUsedElsewhereDto(Library.Main.CLIDatabaseLocator.IsDatabasePathInUse(bk.DBPath)); - private static Dto.IsBackupActiveDto ExecuteGetIsActive(IWorkerThreadsManager workerThreadsManager, IBackup bk) + private static Dto.IsBackupActiveDto ExecuteGetIsActive(IQueueRunnerService queueRunnerService, IBackup bk) { - if (workerThreadsManager.WorkerThread == null) - throw new InvalidOperationException("Worker thread not available"); - - var t = workerThreadsManager.WorkerThread.CurrentTask; - var bt = t?.Backup; - if (bt != null && bk.ID == bt.ID) + var t = queueRunnerService.GetCurrentTask(); + var bt = t?.BackupID; + if (bt != null && bk.ID == bt) return new Dto.IsBackupActiveDto("OK", true); - if (workerThreadsManager.WorkerThread.CurrentTasks.Any(x => x?.Backup == null || x.Backup.ID == bk.ID)) + if (queueRunnerService.GetCurrentTasks().Any(x => x?.BackupID == null || x.BackupID == bk.ID)) return new Dto.IsBackupActiveDto("OK", true); return new Dto.IsBackupActiveDto("OK", false); diff --git a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupPost.cs b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupPost.cs index 8b58ff846..3b40ff06d 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupPost.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupPost.cs @@ -18,7 +18,6 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using Duplicati.Library.RestAPI.Abstractions; using Duplicati.Server; using Duplicati.Server.Database; using Duplicati.Server.Serialization; @@ -45,44 +44,44 @@ public class BackupPost : IEndpointV1 => UpdateDatabasePath(connection, GetBackup(connection, id), input.path, false)) .RequireAuthorization(); - group.MapPost("/backup/{id}/restore", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id, [FromBody] Dto.RestoreInputDto input) - => ExecuteRestore(GetBackup(connection, id), workerThreadsManager, input)) + group.MapPost("/backup/{id}/restore", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id, [FromBody] Dto.RestoreInputDto input) + => ExecuteRestore(GetBackup(connection, id), queueRunnerService, input)) .RequireAuthorization(); - group.MapPost("/backup/{id}/createreport", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id) - => ExecuteCreateReport(GetBackup(connection, id), workerThreadsManager)) + group.MapPost("/backup/{id}/createreport", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id) + => ExecuteCreateReport(GetBackup(connection, id), queueRunnerService)) .RequireAuthorization(); - group.MapPost("/backup/{id}/repair", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id, Dto.RepairInputDto? input) - => ExecuteRepair(GetBackup(connection, id), workerThreadsManager, input)) + group.MapPost("/backup/{id}/repair", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id, Dto.RepairInputDto? input) + => ExecuteRepair(GetBackup(connection, id), queueRunnerService, input)) .RequireAuthorization(); - group.MapPost("/backup/{id}/repairupdate", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id, Dto.RepairInputDto? input) - => ExecuteRepairUpdate(GetBackup(connection, id), workerThreadsManager, input)) + group.MapPost("/backup/{id}/repairupdate", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id, Dto.RepairInputDto? input) + => ExecuteRepairUpdate(GetBackup(connection, id), queueRunnerService, input)) .RequireAuthorization(); - group.MapPost("/backup/{id}/vacuum", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id) - => ExecuteVacuum(GetBackup(connection, id), workerThreadsManager)) + group.MapPost("/backup/{id}/vacuum", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id) + => ExecuteVacuum(GetBackup(connection, id), queueRunnerService)) .RequireAuthorization(); - group.MapPost("/backup/{id}/verify", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id) - => ExecuteVerify(GetBackup(connection, id), workerThreadsManager)) + group.MapPost("/backup/{id}/verify", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id) + => ExecuteVerify(GetBackup(connection, id), queueRunnerService)) .RequireAuthorization(); - group.MapPost("/backup/{id}/compact", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id) - => ExecuteCompact(GetBackup(connection, id), workerThreadsManager)) + group.MapPost("/backup/{id}/compact", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id) + => ExecuteCompact(GetBackup(connection, id), queueRunnerService)) .RequireAuthorization(); - group.MapPost("/backup/{id}/start", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id) - => ExecuteRunBackup(GetBackup(connection, id), workerThreadsManager)) + group.MapPost("/backup/{id}/start", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id) + => ExecuteRunBackup(GetBackup(connection, id), queueRunnerService)) .RequireAuthorization(); - group.MapPost("/backup/{id}/run", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id) - => ExecuteRunBackup(GetBackup(connection, id), workerThreadsManager)) + group.MapPost("/backup/{id}/run", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id) + => ExecuteRunBackup(GetBackup(connection, id), queueRunnerService)) .RequireAuthorization(); - group.MapPost("/backup/{id}/report-remote-size", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id) - => ExecuteReportRemoteSize(GetBackup(connection, id), workerThreadsManager)) + group.MapPost("/backup/{id}/report-remote-size", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id) + => ExecuteReportRemoteSize(GetBackup(connection, id), queueRunnerService)) .RequireAuthorization(); group.MapPost("/backup/{id}/copytotemp", ([FromServices] Connection connection, [FromRoute] string id) @@ -114,8 +113,8 @@ public class BackupPost : IEndpointV1 - private static Dto.TaskStartedDto ExecuteRestore(IBackup backup, IWorkerThreadsManager workerThreadsManager, Dto.RestoreInputDto input) - => new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateRestoreTask( + private static Dto.TaskStartedDto ExecuteRestore(IBackup backup, IQueueRunnerService queueRunnerService, Dto.RestoreInputDto input) + => new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateRestoreTask( backup, input.paths ?? [], Library.Utility.Timeparser.ParseTimeInterval(input.time, DateTime.Now), @@ -125,31 +124,31 @@ public class BackupPost : IEndpointV1 input.skip_metadata ?? false, string.IsNullOrWhiteSpace(input.passphrase) ? null : input.passphrase))); - private static Dto.TaskStartedDto ExecuteCreateReport(IBackup backup, IWorkerThreadsManager workerThreadsManager) - => new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateTask(DuplicatiOperation.CreateReport, backup))); + private static Dto.TaskStartedDto ExecuteCreateReport(IBackup backup, IQueueRunnerService queueRunnerService) + => new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateTask(DuplicatiOperation.CreateReport, backup))); - private static Dto.TaskStartedDto ExecuteReportRemoteSize(IBackup backup, IWorkerThreadsManager workerThreadsManager) - => new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateTask(DuplicatiOperation.ListRemote, backup))); + private static Dto.TaskStartedDto ExecuteReportRemoteSize(IBackup backup, IQueueRunnerService queueRunnerService) + => new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateTask(DuplicatiOperation.ListRemote, backup))); - private static Dto.TaskStartedDto ExecuteRepair(IBackup backup, IWorkerThreadsManager workerThreadsManager, Dto.RepairInputDto? input) - => DoRepair(backup, false, workerThreadsManager, input); + private static Dto.TaskStartedDto ExecuteRepair(IBackup backup, IQueueRunnerService queueRunnerService, Dto.RepairInputDto? input) + => DoRepair(backup, false, queueRunnerService, input); - private static Dto.TaskStartedDto ExecuteRepairUpdate(IBackup backup, IWorkerThreadsManager workerThreadsManager, Dto.RepairInputDto? input) - => DoRepair(backup, true, workerThreadsManager, input); + private static Dto.TaskStartedDto ExecuteRepairUpdate(IBackup backup, IQueueRunnerService queueRunnerService, Dto.RepairInputDto? input) + => DoRepair(backup, true, queueRunnerService, input); - private static Dto.TaskStartedDto ExecuteVacuum(IBackup backup, IWorkerThreadsManager workerThreadsManager) - => new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateTask(DuplicatiOperation.Vacuum, backup))); + private static Dto.TaskStartedDto ExecuteVacuum(IBackup backup, IQueueRunnerService queueRunnerService) + => new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateTask(DuplicatiOperation.Vacuum, backup))); - private static Dto.TaskStartedDto ExecuteVerify(IBackup backup, IWorkerThreadsManager workerThreadsManager) - => new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateTask(DuplicatiOperation.Verify, backup))); + private static Dto.TaskStartedDto ExecuteVerify(IBackup backup, IQueueRunnerService queueRunnerService) + => new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateTask(DuplicatiOperation.Verify, backup))); - private static Dto.TaskStartedDto ExecuteCompact(IBackup backup, IWorkerThreadsManager workerThreadsManager) - => new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateTask(DuplicatiOperation.Compact, backup))); + private static Dto.TaskStartedDto ExecuteCompact(IBackup backup, IQueueRunnerService queueRunnerService) + => new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateTask(DuplicatiOperation.Compact, backup))); - private static Dto.TaskStartedDto DoRepair(IBackup backup, bool repairUpdate, IWorkerThreadsManager workerThreadsManager, Dto.RepairInputDto? input) + private static Dto.TaskStartedDto DoRepair(IBackup backup, bool repairUpdate, IQueueRunnerService queueRunnerService, Dto.RepairInputDto? input) { // These are all props on the input object - var extra = new Dictionary(); + var extra = new Dictionary(); if (input != null) { if (input.only_paths.HasValue) @@ -168,23 +167,23 @@ public class BackupPost : IEndpointV1 var filters = input?.paths ?? []; - return new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateTask(repairUpdate ? DuplicatiOperation.RepairUpdate : DuplicatiOperation.Repair, backup, extra, filters))); + return new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateTask(repairUpdate ? DuplicatiOperation.RepairUpdate : DuplicatiOperation.Repair, backup, extra, filters))); } - private static Dto.TaskStartedDto ExecuteRunBackup(IBackup backup, IWorkerThreadsManager workerThreadsManager) + private static Dto.TaskStartedDto ExecuteRunBackup(IBackup backup, IQueueRunnerService queueRunnerService) { - var t = workerThreadsManager.WorkerThread?.CurrentTask; - var bt = t?.Backup; + var t = queueRunnerService.GetCurrentTask(); + var bt = t?.BackupID; // Already running - if (bt != null && backup.ID == bt.ID) + if (bt != null && backup.ID == bt) return new Dto.TaskStartedDto("OK", t!.TaskID); - t = workerThreadsManager.WorkerThread?.CurrentTasks.FirstOrDefault(x => x?.Backup != null && x.Backup.ID == backup.ID); + t = queueRunnerService.GetCurrentTasks().FirstOrDefault(x => x.BackupID == backup.ID); if (t != null) return new Dto.TaskStartedDto("OK", t.TaskID); - return new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateTask(DuplicatiOperation.Backup, backup), true)); + return new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateTask(DuplicatiOperation.Backup, backup), true)); } private class WrappedBackup : Server.Database.Backup diff --git a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupPutDelete.cs b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupPutDelete.cs index 8bb4ce4b4..754cab235 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupPutDelete.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupPutDelete.cs @@ -18,8 +18,6 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System.Text.Json; -using Duplicati.Library.RestAPI.Abstractions; using Duplicati.Server; using Duplicati.Server.Database; using Duplicati.Server.Serialization; @@ -38,9 +36,9 @@ public class BackupPutDelete : IEndpointV1 => ExecutePut(GetBackup(connection, id), connection, input)) .RequireAuthorization(); - group.MapDelete("/backup/{id}", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromServices] LiveControls liveControls, [FromServices] IHttpContextAccessor httpContextAccessor, [FromRoute] string id, [FromQuery(Name = "delete-remote-files")] bool? delete_remote_files, [FromQuery(Name = "delete-local-db")] bool? delete_local_db, [FromQuery] bool? force) => + group.MapDelete("/backup/{id}", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromServices] LiveControls liveControls, [FromServices] IHttpContextAccessor httpContextAccessor, [FromRoute] string id, [FromQuery(Name = "delete-remote-files")] bool? delete_remote_files, [FromQuery(Name = "delete-local-db")] bool? delete_local_db, [FromQuery] bool? force) => { - var res = ExecuteDelete(GetBackup(connection, id), workerThreadsManager, liveControls, delete_remote_files ?? false, delete_local_db, force ?? false); + var res = ExecuteDelete(GetBackup(connection, id), queueRunnerService, liveControls, delete_remote_files ?? false, delete_local_db, force ?? false); if (res.Status != "OK" && httpContextAccessor.HttpContext != null) httpContextAccessor.HttpContext.Response.StatusCode = 500; return res; @@ -132,67 +130,56 @@ public class BackupPutDelete : IEndpointV1 } } - private static Dto.DeleteBackupOutputDto ExecuteDelete(IBackup backup, IWorkerThreadsManager workerThreadsManager, LiveControls liveControls, bool delete_remote_files, bool? delete_local_db, bool force) + private static Dto.DeleteBackupOutputDto ExecuteDelete(IBackup backup, IQueueRunnerService queueRunnerService, LiveControls liveControls, bool delete_remote_files, bool? delete_local_db, bool force) { - if (workerThreadsManager.WorkerThread!.Active) + try { - try + var nt = queueRunnerService.GetCurrentTask(); + if (backup.ID == nt?.BackupID) { - //TODO: It's not safe to access the values like this, - //because the runner thread might interfere - var nt = workerThreadsManager.WorkerThread.CurrentTask; - if (backup.Equals(nt?.Backup)) + if (!force) + return new Dto.DeleteBackupOutputDto("failed", "backup-in-progress", nt?.TaskID); + + + bool hasPaused = liveControls.State != LiveControls.LiveControlState.Paused; + if (hasPaused) + liveControls.Pause(true); + nt.Abort(); + + for (int i = 0; i < 10; i++) { - if (!force) - return new Dto.DeleteBackupOutputDto("failed", "backup-in-progress", nt?.TaskID); - - - bool hasPaused = liveControls.State != LiveControls.LiveControlState.Paused; - if (hasPaused) - liveControls.Pause(true); - nt.Abort(); - - for (int i = 0; i < 10; i++) - if (workerThreadsManager.WorkerThread.Active) - { - var t = workerThreadsManager.WorkerThread.CurrentTask; - if (backup.Equals(t == null ? null : t.Backup)) - Thread.Sleep(1000); - else - break; - } - else - break; - - if (workerThreadsManager.WorkerThread.Active) - { - var t = workerThreadsManager.WorkerThread.CurrentTask; - if (backup.Equals(t == null ? null : t.Backup)) - { - if (hasPaused) - liveControls.Resume(); - - return new Dto.DeleteBackupOutputDto("failed", "backup-unstoppable", t?.TaskID); - } - } + var tt = queueRunnerService.GetCurrentTask(); + if (backup.ID == tt?.BackupID) + Thread.Sleep(1000); + else + break; + } + var t = queueRunnerService.GetCurrentTask(); + if (backup.ID == t?.BackupID) + { if (hasPaused) liveControls.Resume(); + + return new Dto.DeleteBackupOutputDto("failed", "backup-unstoppable", t?.TaskID); } - } - catch (Exception ex) - { - return new Dto.DeleteBackupOutputDto("error", ex.Message, null); + + if (hasPaused) + liveControls.Resume(); } } + catch (Exception ex) + { + return new Dto.DeleteBackupOutputDto("error", ex.Message, null); + } - var extra = new Dictionary(); + var extra = new Dictionary(); if (delete_local_db.HasValue) extra["delete-local-db"] = delete_local_db.Value.ToString(); if (delete_remote_files) extra["delete-remote-files"] = "true"; - return new Dto.DeleteBackupOutputDto("OK", null, workerThreadsManager.AddTask(Runner.CreateTask(DuplicatiOperation.Delete, backup, extra))); + return new Dto.DeleteBackupOutputDto("OK", null, queueRunnerService.AddTask(Runner.CreateTask(DuplicatiOperation.Delete, backup, extra))); } } diff --git a/Duplicati/WebserverCore/Endpoints/V1/Tasks.cs b/Duplicati/WebserverCore/Endpoints/V1/Tasks.cs index a1a72cca2..1da85989d 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Tasks.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Tasks.cs @@ -34,35 +34,35 @@ public class Tasks : IEndpointV1 } public static void Map(RouteGroupBuilder group) { - group.MapGet("/tasks", Execute).RequireAuthorization(); - group.MapGet("/task/{taskid}", ([FromRoute] long taskId, [FromServices] ITaskCacheService taskCacheService) => ExecuteGet(taskCacheService, taskId)).RequireAuthorization(); - group.MapPost("/task/{taskid}/stop", ([FromRoute] long taskId) => ExecutePost(taskId, TaskStopState.Stop)).RequireAuthorization(); - group.MapPost("/task/{taskid}/abort", ([FromRoute] long taskId) => ExecutePost(taskId, TaskStopState.Abort)).RequireAuthorization(); + group.MapGet("/tasks", ([FromServices] IQueueRunnerService queueRunnerService) => Execute(queueRunnerService)).RequireAuthorization(); + group.MapGet("/task/{taskid}", ([FromRoute] long taskId, [FromServices] IQueueRunnerService queueRunnerService) => ExecuteGet(queueRunnerService, taskId)).RequireAuthorization(); + group.MapPost("/task/{taskid}/stop", ([FromRoute] long taskId, [FromServices] IQueueRunnerService queueRunnerService) => ExecutePost(queueRunnerService, taskId, TaskStopState.Stop)).RequireAuthorization(); + group.MapPost("/task/{taskid}/abort", ([FromRoute] long taskId, [FromServices] IQueueRunnerService queueRunnerService) => ExecutePost(queueRunnerService, taskId, TaskStopState.Abort)).RequireAuthorization(); } - private static IEnumerable Execute() + private static IEnumerable Execute(IQueueRunnerService queueRunnerService) { - var cur = FIXMEGlobal.WorkThread.CurrentTask; - var n = FIXMEGlobal.WorkThread.CurrentTasks; + var cur = queueRunnerService.GetCurrentTask(); + var n = queueRunnerService.GetCurrentTasks(); if (cur != null) n.Insert(0, cur); - return n; + return n.OfType(); } - private static Dto.GetTaskStateDto ExecuteGet(ITaskCacheService taskCacheService, long taskid) + private static Dto.GetTaskStateDto ExecuteGet(IQueueRunnerService queueRunnerService, long taskid) { - var task = FIXMEGlobal.WorkThread.CurrentTask; - var tasks = FIXMEGlobal.WorkThread.CurrentTasks; + var task = queueRunnerService.GetCurrentTask(); + var tasks = queueRunnerService.GetCurrentTasks(); if (task != null && task.TaskID == taskid) return new Dto.GetTaskStateDto("Running", taskid, task.TaskStarted, task.TaskFinished); if (tasks.FirstOrDefault(x => x.TaskID == taskid) == null) { - var res = taskCacheService.GetCachedTaskResults(taskid); + var res = queueRunnerService.GetCachedTaskResults(taskid); if (res == null) throw new NotFoundException("No such task found"); @@ -79,10 +79,10 @@ public class Tasks : IEndpointV1 return new Dto.GetTaskStateDto("Waiting", taskid, null, null); } - private static void ExecutePost(long taskid, TaskStopState stopState) + private static void ExecutePost(IQueueRunnerService queueRunnerService, long taskid, TaskStopState stopState) { - var task = FIXMEGlobal.WorkThread.CurrentTask; - var tasks = FIXMEGlobal.WorkThread.CurrentTasks; + var task = queueRunnerService.GetCurrentTask(); + var tasks = queueRunnerService.GetCurrentTasks(); if (task != null) tasks.Insert(0, task); diff --git a/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs b/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs index 45264c9fe..ae2a52593 100644 --- a/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs +++ b/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs @@ -20,7 +20,6 @@ // DEALINGS IN THE SOFTWARE. using Duplicati.Library.IO; using Duplicati.Library.RestAPI; -using Duplicati.Library.RestAPI.Abstractions; using Duplicati.Server; using Duplicati.Server.Database; using Duplicati.Server.Serialization; @@ -44,7 +43,6 @@ public static class ServiceCollectionsExtensions .AddSingleton(Serializer.JsonSettings) .AddSingleton() .AddSingleton() - .AddSingleton() .AddSingleton(connection); @@ -57,8 +55,7 @@ public static class ServiceCollectionsExtensions .AddTransient() .AddTransient() .AddSingleton() - .AddSingleton() - .AddSingleton() + .AddSingleton() .AddSingleton() .AddTransient() .AddSingleton() @@ -69,7 +66,7 @@ public static class ServiceCollectionsExtensions .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton(); + .AddSingleton(); return services; } diff --git a/Duplicati/WebserverCore/Services/CommandlineRunService.cs b/Duplicati/WebserverCore/Services/CommandlineRunService.cs index 81bff62b4..3d9747ab1 100644 --- a/Duplicati/WebserverCore/Services/CommandlineRunService.cs +++ b/Duplicati/WebserverCore/Services/CommandlineRunService.cs @@ -20,13 +20,12 @@ // DEALINGS IN THE SOFTWARE. using System.Text; using Duplicati.Library.RestAPI; -using Duplicati.Library.RestAPI.Abstractions; using Duplicati.Server; using Duplicati.WebserverCore.Abstractions; namespace Duplicati.WebserverCore.Services; -public class CommandlineRunService(IWorkerThreadsManager workerThreadsManager) : ICommandlineRunService +public class CommandlineRunService(IQueueRunnerService queueRunnerService) : ICommandlineRunService { private static readonly string LOGTAG = Library.Logging.Log.LogTagFromType(); @@ -180,7 +179,7 @@ public class CommandlineRunService(IWorkerThreadsManager workerThreadsManager) : } }); - workerThreadsManager.AddTask(k.Task); + queueRunnerService.AddTask(k.Task); return k.ID; } diff --git a/Duplicati/WebserverCore/Services/QueueRunnerService.cs b/Duplicati/WebserverCore/Services/QueueRunnerService.cs new file mode 100644 index 000000000..9335bcc93 --- /dev/null +++ b/Duplicati/WebserverCore/Services/QueueRunnerService.cs @@ -0,0 +1,208 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using Duplicati.Library.Utility; +using Duplicati.Server; +using Duplicati.Server.Database; +using Duplicati.Server.Serialization.Interface; +using Duplicati.WebserverCore.Abstractions; + +namespace Duplicati.WebserverCore.Services; + +/// +/// Simple queue that will run the given task +/// +public class QueueRunnerService(Connection connection, EventPollNotify eventPollNotify) : IQueueRunnerService +{ + private readonly object _lock = new(); + /// + /// A thread-safe dictionary to store cached task results. + /// + private readonly Dictionary _taskCache = new(); + + /// + /// The maximum number of completed task results to keep in memory + /// + private static readonly int MAX_TASK_RESULT_CACHE_SIZE = 100; + + private readonly List _tasks = new(); + private Task? _currentTask; + private IQueuedTask? _activeTask; + private bool _isPaused; + private bool _isTerminated; + + public long AddTask(IQueuedTask task) + => AddTask(task, false); + + public long AddTask(IQueuedTask task, bool skipQueue) + { + lock (_lock) + if (skipQueue) + _tasks.Insert(0, task); + else + _tasks.Add(task); + + eventPollNotify.SignalNewEvent(); + StartNextTask(); + return task.TaskID; + } + + public bool GetIsActive() + => _activeTask != null; + + public IQueuedTask? GetCurrentTask() + => _activeTask; + + public List GetCurrentTasks() + { + lock (_lock) + return [.. _tasks]; + } + + public void Pause() + { + lock (_lock) + _isPaused = true; + } + + public void Resume() + { + lock (_lock) + _isPaused = false; + + StartNextTask(); + } + + public void Terminate(bool wait) + { + _isTerminated = true; + if (wait) + { + var task = _currentTask; + if (task != null) + task.Await(); + } + } + + private void StartNextTask() + { + lock (_lock) + { + if (_isTerminated || _isPaused || (_currentTask != null && !_currentTask.IsCompleted)) + return; + + // Clean up completed tasks + if (_currentTask != null && _currentTask.IsCompleted) + { + _currentTask = null; + _activeTask = null; + } + + if (_tasks.Count == 0) + return; + + _activeTask = _tasks[0]; + _tasks.RemoveAt(0); + var task = _activeTask; + _currentTask = Task.Run(() => RunTask(task), CancellationToken.None); + } + } + + private async Task RunTask(IQueuedTask task) + { + var completed = false; + try + { + eventPollNotify.SignalNewEvent(); + task.TaskStarted = DateTime.UtcNow; + if (task.OnStarting != null) + await task.OnStarting().ConfigureAwait(false); + + await task.Execute(); + + // If the task is completed, don't call OnFinished again + completed = true; + AddTaskResult(new CachedTaskResult(task.TaskID, task.BackupID, task.TaskStarted, task.TaskFinished ?? DateTime.Now, null)); + if (task.OnFinished != null) + await task.OnFinished(null).ConfigureAwait(false); + } + catch (Exception ex) + { + connection.LogError(task.BackupID, "Error in worker", ex); + if (!completed) + { + AddTaskResult(new CachedTaskResult(task.TaskID, task.BackupID, task.TaskStarted, task.TaskFinished ?? DateTime.Now, ex)); + if (task.OnFinished != null) + await task.OnFinished(ex).ConfigureAwait(false); + } + } + finally + { + task.TaskFinished = DateTime.UtcNow; + _currentTask = null; + _activeTask = null; + eventPollNotify.SignalNewEvent(); + StartNextTask(); + } + } + + public IList> GetQueueWithIds() + { + return (from n in GetCurrentTasks() + where n.BackupID != null + select new Tuple(n.TaskID, n.BackupID)).ToList(); + } + + + /// + public CachedTaskResult? GetCachedTaskResults(long taskID) + { + lock (_lock) + { + _taskCache.TryGetValue(taskID, out var result); + return result; + } + } + + private void AddTaskResult(CachedTaskResult taskResult) + { + lock (_lock) + { + // If the task result is already in the cache, remove it + if (_taskCache.TryGetValue(taskResult.TaskID, out var existingResult)) + { + // If the stored task result has an exception, do not overwrite it + if (existingResult.Exception != null) + return; + } + + // Add/update the new task result in the cache + _taskCache[taskResult.TaskID] = taskResult; + + // If the cache size exceeds the maximum, remove the oldest entry + while (_taskCache.Count >= MAX_TASK_RESULT_CACHE_SIZE) + { + var oldestTaskID = _taskCache.Keys.Min(); + _taskCache.Remove(oldestTaskID); + } + } + } +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Services/SchedulerService.cs b/Duplicati/WebserverCore/Services/SchedulerService.cs index 15296edf2..b327c83ba 100644 --- a/Duplicati/WebserverCore/Services/SchedulerService.cs +++ b/Duplicati/WebserverCore/Services/SchedulerService.cs @@ -19,31 +19,32 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using Duplicati.Library.Utility; +using Duplicati.Library.RestAPI; using Duplicati.Server; using Duplicati.Server.Serialization.Interface; using Duplicati.WebserverCore.Abstractions; namespace WebserverCore.Services; -public class SchedulerService : IScheduler +public class SchedulerService : ISchedulerService { - private readonly Duplicati.Server.Scheduler scheduler; - public SchedulerService(Duplicati.Server.Scheduler scheduler) + private readonly Scheduler scheduler; + public SchedulerService(EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IQueueRunnerService queueRunnerService) { - this.scheduler = scheduler; + this.scheduler = new Scheduler(queueRunnerService); + var lastScheduleId = notificationUpdateService.LastDataUpdateId; + eventPollNotify.NewEvent += (sender, e) => + { + if (lastScheduleId != notificationUpdateService.LastDataUpdateId) + { + lastScheduleId = notificationUpdateService.LastDataUpdateId; + Reschedule(); + } + }; } public List> Schedule => scheduler.Schedule; - public List WorkerQueue => scheduler.WorkerQueue; - - public void SubScribeToNewSchedule(Action handler) - => scheduler.NewSchedule += (_, _) => handler(); - - public IList> GetSchedulerQueueIds() - => scheduler.GetSchedulerQueueIds(); - public IList> GetProposedSchedule() => scheduler.GetProposedSchedule(); @@ -52,7 +53,4 @@ public class SchedulerService : IScheduler public void Terminate(bool wait) => scheduler.Terminate(wait); - - public void Init(WorkerThread worker) - => scheduler.Init(worker); } diff --git a/Duplicati/WebserverCore/Services/StatusService.cs b/Duplicati/WebserverCore/Services/StatusService.cs index 7c3845b8b..336ea261e 100644 --- a/Duplicati/WebserverCore/Services/StatusService.cs +++ b/Duplicati/WebserverCore/Services/StatusService.cs @@ -19,7 +19,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.RestAPI; -using Duplicati.Library.RestAPI.Abstractions; using Duplicati.Server; using Duplicati.Server.Serialization; using Duplicati.WebserverCore.Abstractions; @@ -31,22 +30,24 @@ public class StatusService( LiveControls liveControls, UpdatePollThread updatePollThread, IUpdateService updateService, - IWorkerThreadsManager workerThreadsManager, + IQueueRunnerService queueRunnerService, ISettingsService settingsService, - IScheduler scheduler, + ISchedulerService scheduler, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService) : IStatusService { public ServerStatusDto GetStatus() { + var task = queueRunnerService.GetCurrentTask(); + var status = new ServerStatusDto { UpdatedVersion = GetUpdatedVersion(), UpdaterState = updatePollThread.ThreadState, UpdateDownloadProgress = updatePollThread.DownloadProgess, - ActiveTask = workerThreadsManager.CurrentTask, - SchedulerQueueIds = scheduler.GetSchedulerQueueIds(), + ActiveTask = task == null ? null : new Tuple(task.TaskID, task.BackupID), + SchedulerQueueIds = queueRunnerService.GetQueueWithIds(), ProposedSchedule = scheduler.GetProposedSchedule(), LastEventID = eventPollNotify.EventNo, LastDataUpdateID = notificationUpdateService.LastDataUpdateId, @@ -72,7 +73,8 @@ public class StatusService( private SuggestedStatusIcon MapStateToIcon() { - if (workerThreadsManager.CurrentTask == null) + var task = queueRunnerService.GetCurrentTask(); + if (task == null) { if (liveControls.State == LiveControls.LiveControlState.Paused) return SuggestedStatusIcon.Paused; diff --git a/Duplicati/WebserverCore/Services/TaskCacheResults.cs b/Duplicati/WebserverCore/Services/TaskCacheResults.cs deleted file mode 100644 index 0f8d171a3..000000000 --- a/Duplicati/WebserverCore/Services/TaskCacheResults.cs +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. -using Duplicati.WebserverCore.Abstractions; - -namespace Duplicati.WebserverCore.Services; - -/// -/// Service for caching task results. -/// -public class TaskCacheService : ITaskCacheService -{ - /// - /// A thread-safe dictionary to store cached task results. - /// - private readonly Dictionary _taskCache = new(); - - /// - /// The maximum number of completed task results to keep in memory - /// - private static readonly int MAX_TASK_RESULT_CACHE_SIZE = 100; - - /// - public CachedTaskResult? GetCachedTaskResults(long taskID) - { - lock (_taskCache) - { - _taskCache.TryGetValue(taskID, out var result); - return result; - } - } - - /// - public void AddTaskResult(CachedTaskResult taskResult) - { - lock (_taskCache) - { - // If the task result is already in the cache, remove it - if (_taskCache.TryGetValue(taskResult.TaskID, out var existingResult)) - { - // If the stored task result has an exception, do not overwrite it - if (existingResult.Exception != null) - return; - } - - // Add/update the new task result in the cache - _taskCache[taskResult.TaskID] = taskResult; - - // If the cache size exceeds the maximum, remove the oldest entry - while (_taskCache.Count >= MAX_TASK_RESULT_CACHE_SIZE) - { - var oldestTaskID = _taskCache.Keys.Min(); - _taskCache.Remove(oldestTaskID); - } - } - } - -} diff --git a/Duplicati/WebserverCore/Services/WorkerThreadsManager.cs b/Duplicati/WebserverCore/Services/WorkerThreadsManager.cs deleted file mode 100644 index 829837f4f..000000000 --- a/Duplicati/WebserverCore/Services/WorkerThreadsManager.cs +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. -#nullable enable -using Duplicati.Library.IO; -using Duplicati.Library.RestAPI; -using Duplicati.Library.RestAPI.Abstractions; -using Duplicati.Library.Utility; -using Duplicati.Server; -using Duplicati.WebserverCore.Abstractions; - -namespace Duplicati.WebserverCore.Services; - -public class WorkerThreadsManager(ILiveControls liveControls, IScheduler scheduler) : IWorkerThreadsManager -{ - public WorkerThread? WorkerThread { get; private set; } - - public void Spawn(Action item) - { - WorkerThread = new WorkerThread(item, liveControls.IsPaused); - scheduler.Init(WorkerThread); - } - - public Tuple? CurrentTask - { - get - { - var t = WorkerThread?.CurrentTask; - return t == null ? null : new Tuple(t.TaskID, t.Backup.ID); - } - } - - public void UpdateThrottleSpeeds(string? uploadSpeed, string? downloadSpeed) - { - WorkerThread?.CurrentTask?.UpdateThrottleSpeed(uploadSpeed, downloadSpeed); - } - - public long AddTask(Runner.IRunnerData data, bool skipQueue = false) - { - WorkerThread!.AddTask(data, skipQueue); - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); - return data.TaskID; - } -} \ No newline at end of file From 16acb282c968891f7f7f4c64ed355cbf8b25a1e8 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 3 May 2025 15:54:23 +0200 Subject: [PATCH 06/24] More FIXMEGlobal cleanup --- Duplicati/Library/RestAPI/FIXMEGlobal.cs | 19 +- Duplicati/Library/RestAPI/Runner.cs | 184 ++++++++++-------- Duplicati/Library/RestAPI/Scheduler.cs | 31 +-- .../Interface/IQueuedTask.cs | 4 - Duplicati/Server/Program.cs | 42 ++-- .../Endpoints/V1/Backup/BackupGet.cs | 32 +-- .../Endpoints/V1/Backup/BackupPutDelete.cs | 8 +- .../Endpoints/V1/RemoteOperation.cs | 17 +- .../WebserverCore/Endpoints/V1/Updates.cs | 11 +- .../WebserverCore/Endpoints/V1/WebModules.cs | 9 +- .../Endpoints/V2/BackupListing.cs | 34 ++-- .../Endpoints/V2/DestinationVerify.cs | 14 +- .../Services/QueueRunnerService.cs | 2 +- .../Services/SchedulerService.cs | 5 +- 14 files changed, 208 insertions(+), 204 deletions(-) diff --git a/Duplicati/Library/RestAPI/FIXMEGlobal.cs b/Duplicati/Library/RestAPI/FIXMEGlobal.cs index 5a977f0c2..6b2491e70 100644 --- a/Duplicati/Library/RestAPI/FIXMEGlobal.cs +++ b/Duplicati/Library/RestAPI/FIXMEGlobal.cs @@ -48,12 +48,7 @@ namespace Duplicati.Library.RestAPI /// /// A delegate method for creating a copy of the current progress state /// - public static Func GenerateProgressState; - - /// - /// The status event signaler, used to control long polling of status updates - /// - public static EventPollNotify StatusEventNotifyer => Provider.GetRequiredService(); + public static Func GenerateProgressState; /// /// For keeping and incrementing last last events Ids of db save and last notification @@ -71,28 +66,16 @@ namespace Duplicati.Library.RestAPI /// public static string DataFolder; - /// - /// This is the scheduling thread - /// - public static ISchedulerService Scheduler => Provider.GetRequiredService(); - /// /// The log redirect handler /// public static readonly LogWriteHandler LogHandler = new LogWriteHandler(); - /// - /// The update poll thread. - /// - public static UpdatePollThread UpdatePoller => Provider.GetRequiredService(); - - /// /// Used to check the origin of the web server (e.g. Tray icon or a stand alone Server) /// public static string Origin = "Server"; - /// /// The application exit event /// diff --git a/Duplicati/Library/RestAPI/Runner.cs b/Duplicati/Library/RestAPI/Runner.cs index 3480d1795..841454dcb 100644 --- a/Duplicati/Library/RestAPI/Runner.cs +++ b/Duplicati/Library/RestAPI/Runner.cs @@ -29,6 +29,8 @@ using Duplicati.Server.Serialization; using Duplicati.Library.RestAPI; using Duplicati.Library.Utility; using System.Threading.Tasks; +using Duplicati.Server.Database; +using Duplicati.Server.Serialization.Interface; namespace Duplicati.Server { @@ -52,6 +54,12 @@ namespace Duplicati.Server public Func? OnStarting { get; set; } public Func? OnFinished { get; set; } + /// + /// Callback to be executed when the task is finished. + /// Only supported for the delete operation for now. + /// + internal Action? AfterTaskFinished { get; set; } + public DuplicatiOperation Operation { get; internal set; } public Serialization.Interface.IBackup? Backup { get; internal set; } public IDictionary? ExtraOptions { get; internal set; } @@ -141,9 +149,6 @@ namespace Duplicati.Server { m_taskID = System.Threading.Interlocked.Increment(ref RunnerTaskID); } - - public Task Execute() - => Task.Run(() => Runner.Run(this, true)); } private class CustomRunnerTask : RunnerData @@ -166,7 +171,7 @@ namespace Duplicati.Server return new CustomRunnerTask(runner); } - public static IRunnerData CreateTask(DuplicatiOperation operation, Serialization.Interface.IBackup backup, IDictionary? extraOptions = null, string[]? filterStrings = null, string[]? extraArguments = null, int pageSize = 0, int pageOffset = 0) + public static IRunnerData CreateTask(DuplicatiOperation operation, IBackup backup, IDictionary? extraOptions = null, string[]? filterStrings = null, string[]? extraArguments = null, int pageSize = 0, int pageOffset = 0) { return new RunnerData() { @@ -180,7 +185,18 @@ namespace Duplicati.Server }; } - public static IRunnerData CreateListTask(Serialization.Interface.IBackup backup, string[]? filters, bool onlyPrefix, bool allVersions, bool folderContents, DateTime time) + public static IRunnerData CreateDeleteTask(IBackup backup, IDictionary extraOptions, Action? afterTaskFinished = null) + { + return new RunnerData() + { + Operation = DuplicatiOperation.Delete, + Backup = backup, + ExtraOptions = extraOptions, + AfterTaskFinished = afterTaskFinished + }; + } + + public static IRunnerData CreateListTask(IBackup backup, string[]? filters, bool onlyPrefix, bool allVersions, bool folderContents, DateTime time) { var dict = new Dictionary(); if (onlyPrefix) @@ -199,7 +215,7 @@ namespace Duplicati.Server filters); } - public static IRunnerData CreateListFilesetsTask(Serialization.Interface.IBackup backup, Dictionary? extraOptions = null) + public static IRunnerData CreateListFilesetsTask(IBackup backup, Dictionary? extraOptions = null) { return CreateTask( DuplicatiOperation.ListFilesets, @@ -207,7 +223,7 @@ namespace Duplicati.Server extraOptions ?? new Dictionary()); } - public static IRunnerData CreateListFolderContents(Serialization.Interface.IBackup backup, string[] folders, DateTime time, int pageSize, int pageOffset) + public static IRunnerData CreateListFolderContents(IBackup backup, string[]? folders, DateTime time, int pageSize, int pageOffset) { var dict = new Dictionary(); if (time.Ticks > 0) @@ -222,7 +238,7 @@ namespace Duplicati.Server pageOffset: pageOffset); } - public static IRunnerData ListFileVersionsTask(Serialization.Interface.IBackup backup, string[] filepaths, int pageSize, int pageOffset) + public static IRunnerData ListFileVersionsTask(IBackup backup, string[]? filepaths, int pageSize, int pageOffset) { var dict = new Dictionary(); return CreateTask( @@ -234,7 +250,7 @@ namespace Duplicati.Server pageOffset: pageOffset); } - public static IRunnerData CreateSearchEntriesTask(Serialization.Interface.IBackup backup, string[] filters, string[] folders, DateTime time, int pageSize, int pageOffset) + public static IRunnerData CreateSearchEntriesTask(IBackup backup, string[]? filters, string[]? folders, DateTime time, int pageSize, int pageOffset) { var dict = new Dictionary(); if (time.Ticks > 0) @@ -251,7 +267,7 @@ namespace Duplicati.Server } - public static IRunnerData CreateRestoreTask(Serialization.Interface.IBackup backup, string[]? filters, + public static IRunnerData CreateRestoreTask(IBackup backup, string[]? filters, DateTime time, string? restoreTarget, bool overwrite, bool restore_permissions, bool skip_metadata, string? passphrase) { @@ -276,7 +292,7 @@ namespace Duplicati.Server } private class MessageSink : Library.Main.IMessageSink { - private class ProgressState : Serialization.Interface.IProgressEventData + private class ProgressState : IProgressEventData { private readonly string? m_backupID; private readonly long m_taskID; @@ -394,18 +410,18 @@ namespace Duplicati.Server #endregion } - public static string GetCommandLine(IRunnerData data) + public static string GetCommandLine(Connection databaseConnection, IRunnerData data) { var backup = data.Backup; if (backup == null) throw new ArgumentNullException(nameof(backup)); - var options = ApplyOptions(backup, GetCommonOptions()); + var options = ApplyOptions(databaseConnection, backup, GetCommonOptions(databaseConnection)); if (data.ExtraOptions != null) foreach (var k in data.ExtraOptions) options[k.Key] = k.Value; - var cf = FIXMEGlobal.DataConnection.Filters; + var cf = databaseConnection.Filters; var bf = backup.Filters; var sources = @@ -440,18 +456,18 @@ namespace Duplicati.Server return cmd.ToString(); } - public static string[] GetCommandLineParts(IRunnerData data) + public static string[] GetCommandLineParts(Connection databaseConnection, IRunnerData data) { var backup = data.Backup; if (backup == null) throw new ArgumentNullException(nameof(backup)); - var options = ApplyOptions(backup, GetCommonOptions()); + var options = ApplyOptions(databaseConnection, backup, GetCommonOptions(databaseConnection)); if (data.ExtraOptions != null) foreach (var k in data.ExtraOptions) options[k.Key] = k.Value; - var cf = FIXMEGlobal.DataConnection.Filters; + var cf = databaseConnection.Filters; var bf = backup.Filters; var sources = @@ -480,7 +496,15 @@ namespace Duplicati.Server return parts.ToArray(); } - public static IBasicResults? Run(IRunnerData data, bool fromQueue) + public static IBasicResults? Run(Connection databaseConnection, EventPollNotify eventPollNotify, IQueuedTask data, bool fromQueue) + { + if (data is IRunnerData runnerData) + return Run(databaseConnection, eventPollNotify, runnerData, fromQueue); + + throw new ArgumentException("Invalid task type", nameof(data)); + } + + private static IBasicResults? Run(Connection databaseConnection, EventPollNotify eventPollNotify, IRunnerData data, bool fromQueue) { data.TaskStarted = DateTime.Now; if (data is CustomRunnerTask task) @@ -489,13 +513,13 @@ namespace Duplicati.Server { var sink = new MessageSink(task.TaskID, null); FIXMEGlobal.GenerateProgressState = sink.Copy; - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); + eventPollNotify.SignalNewEvent(); task.Run(sink); } catch (Exception ex) { - FIXMEGlobal.DataConnection.LogError(string.Empty, "Failed while executing custom task", ex); + databaseConnection.LogError(string.Empty, "Failed while executing custom task", ex); } finally { @@ -518,10 +542,10 @@ namespace Duplicati.Server if (fromQueue) { FIXMEGlobal.GenerateProgressState = () => sink.Copy(); - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); + eventPollNotify.SignalNewEvent(); } - var options = ApplyOptions(backup, GetCommonOptions()); + var options = ApplyOptions(databaseConnection, backup, GetCommonOptions(databaseConnection)); if (data.ExtraOptions != null) foreach (var k in data.ExtraOptions) options[k.Key] = k.Value; @@ -529,7 +553,7 @@ namespace Duplicati.Server // Pack in the system or task config for easy restore if (data.Operation == DuplicatiOperation.Backup && options.ContainsKey("store-task-config")) { - tempfolder = StoreTaskConfigAndGetTempFolder(data, options); + tempfolder = StoreTaskConfigAndGetTempFolder(databaseConnection, data, options); } // Attach a log scope that tags all messages to relay the TaskID and BackupID @@ -557,7 +581,7 @@ namespace Duplicati.Server catch { } ((RunnerData)data).Controller = controller; - var appSettings = FIXMEGlobal.DataConnection.ApplicationSettings; + var appSettings = databaseConnection.ApplicationSettings; data.UpdateThrottleSpeeds(appSettings.UploadSpeedLimit, appSettings.DownloadSpeedLimit); // Pass on the provider, will be replaced if configured in the backup @@ -573,7 +597,7 @@ namespace Duplicati.Server { case DuplicatiOperation.Backup: { - var filter = ApplyFilter(backup, GetCommonFilter()); + var filter = ApplyFilter(backup, GetCommonFilter(databaseConnection)); var sources = (from n in backup.Sources let p = SpecialFolders.ExpandEnvironmentVariables(n) @@ -581,62 +605,62 @@ namespace Duplicati.Server select p).ToArray(); var r = controller.Backup(sources, filter); - UpdateMetadata(backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, backup, r); return r; } case DuplicatiOperation.List: { var r = controller.List(data.FilterStrings, null); - UpdateMetadata(backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, backup, r); return r; } case DuplicatiOperation.Repair: { var r = controller.Repair(data.FilterStrings == null ? null : new Library.Utility.FilterExpression(data.FilterStrings)); - UpdateMetadata(backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, backup, r); return r; } case DuplicatiOperation.RepairUpdate: { var r = controller.UpdateDatabaseWithVersions(); - UpdateMetadata(backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, backup, r); return r; } case DuplicatiOperation.Remove: { var r = controller.Delete(); - UpdateMetadata(backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, backup, r); return r; } case DuplicatiOperation.Restore: { var r = controller.Restore(data.FilterStrings); - UpdateMetadata(backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, backup, r); return r; } case DuplicatiOperation.Verify: { var r = controller.Test(); - UpdateMetadata(backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, backup, r); return r; } case DuplicatiOperation.Compact: { var r = controller.Compact(); - UpdateMetadata(backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, backup, r); return r; } case DuplicatiOperation.CreateReport: { - using (var tf = new TempFile()) + using (var tf = new Library.Utility.TempFile()) { var r = controller.CreateLogDatabase(tf); - var tempid = FIXMEGlobal.DataConnection.RegisterTempFile("create-bug-report", r.TargetPath, DateTime.Now.AddDays(3)); + var tempid = databaseConnection.RegisterTempFile("create-bug-report", r.TargetPath, DateTime.Now.AddDays(3)); if (string.Equals(tf, r.TargetPath, Utility.ClientFilenameStringComparison)) tf.Protected = true; - FIXMEGlobal.DataConnection.RegisterNotification( + databaseConnection.RegisterNotification( NotificationType.Information, "Bugreport ready", "Bugreport is ready for download", @@ -656,7 +680,7 @@ namespace Duplicati.Server case DuplicatiOperation.ListRemote: { var r = controller.ListRemote(); - UpdateMetadata(backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, backup, r); return r; } @@ -675,21 +699,21 @@ namespace Duplicati.Server System.IO.File.Delete(dbpath); } } - FIXMEGlobal.DataConnection.DeleteBackup(backup); - FIXMEGlobal.Scheduler.Reschedule(); + databaseConnection.DeleteBackup(backup); + (data as RunnerData)?.AfterTaskFinished?.Invoke(data); return null; } case DuplicatiOperation.Vacuum: { var r = controller.Vacuum(); - UpdateMetadata(backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, backup, r); return r; } case DuplicatiOperation.ListFilesets: { var r = controller.ListFilesets(); - UpdateMetadata(backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, backup, r); return r; } case DuplicatiOperation.ListFolderContents: @@ -714,9 +738,9 @@ namespace Duplicati.Server } catch (Exception ex) { - FIXMEGlobal.DataConnection.LogError(data.Backup?.ID, string.Format("Failed while executing {0} \"{1}\" (id: {2})", data.Operation, data.Backup?.Name, data.Backup?.ID), ex); + databaseConnection.LogError(data.Backup?.ID, string.Format("Failed while executing {0} \"{1}\" (id: {2})", data.Operation, data.Backup?.Name, data.Backup?.ID), ex); if (data.Backup != null) - UpdateMetadataError(data.Backup, ex); + UpdateMetadataError(databaseConnection, data.Backup, ex); Library.UsageReporter.Reporter.Report(ex); if (!fromQueue) @@ -731,7 +755,7 @@ namespace Duplicati.Server } } - private static TempFolder? StoreTaskConfigAndGetTempFolder(IRunnerData data, Dictionary options) + private static TempFolder? StoreTaskConfigAndGetTempFolder(Connection databaseConnection, IRunnerData data, Dictionary options) { if (data.Backup == null) throw new ArgumentNullException(nameof(data.Backup)); @@ -746,13 +770,13 @@ namespace Duplicati.Server { tempfolder = new TempFolder(); var temppath = System.IO.Path.Combine(tempfolder, "task-setup.json"); - using (var tempfile = TempFile.WrapExistingFile(temppath)) + using (var tempfile = Library.Utility.TempFile.WrapExistingFile(temppath)) { object? taskdata = null; if (all_tasks) - taskdata = FIXMEGlobal.DataConnection.Backups.Where(x => !x.IsTemporary).Select(x => FIXMEGlobal.DataConnection.PrepareBackupForExport(FIXMEGlobal.DataConnection.GetBackup(x.ID)!)); + taskdata = databaseConnection.Backups.Where(x => !x.IsTemporary).Select(x => databaseConnection.PrepareBackupForExport(databaseConnection.GetBackup(x.ID)!)); else - taskdata = new[] { FIXMEGlobal.DataConnection.PrepareBackupForExport(data.Backup) }; + taskdata = new[] { databaseConnection.PrepareBackupForExport(data.Backup) }; using (var fs = System.IO.File.OpenWrite(tempfile)) using (var sw = new System.IO.StreamWriter(fs, System.Text.Encoding.UTF8)) @@ -773,20 +797,20 @@ namespace Duplicati.Server return tempfolder; } - private static void UpdateMetadataError(Duplicati.Server.Serialization.Interface.IBackup backup, Exception ex) + private static void UpdateMetadataError(Connection databaseConnection, Serialization.Interface.IBackup backup, Exception ex) { - backup.Metadata["LastErrorDate"] = Library.Utility.Utility.SerializeDateTime(DateTime.UtcNow); + backup.Metadata["LastErrorDate"] = Utility.SerializeDateTime(DateTime.UtcNow); backup.Metadata["LastErrorMessage"] = ex.Message; if (!backup.IsTemporary) - FIXMEGlobal.DataConnection.SetMetadata(backup.Metadata, long.Parse(backup.ID), null); + databaseConnection.SetMetadata(backup.Metadata, long.Parse(backup.ID), null); string? messageid = null; if (ex is UserInformationException exception) messageid = exception.HelpID; FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); - FIXMEGlobal.DataConnection.RegisterNotification( + databaseConnection.RegisterNotification( NotificationType.Error, backup.IsTemporary ? "Error" : string.Format("Error while running {0}", backup.Name), @@ -804,31 +828,31 @@ namespace Duplicati.Server ); } - private static void UpdateMetadataLastCompact(Duplicati.Server.Serialization.Interface.IBackup backup, Duplicati.Library.Interface.ICompactResults r) + private static void UpdateMetadataLastCompact(IBackup backup, ICompactResults r) { if (r != null) { backup.Metadata["LastCompactDuration"] = r.Duration.ToString(); - backup.Metadata["LastCompactStarted"] = Library.Utility.Utility.SerializeDateTime(r.BeginTime.ToUniversalTime()); - backup.Metadata["LastCompactFinished"] = Library.Utility.Utility.SerializeDateTime(r.EndTime.ToUniversalTime()); + backup.Metadata["LastCompactStarted"] = Utility.SerializeDateTime(r.BeginTime.ToUniversalTime()); + backup.Metadata["LastCompactFinished"] = Utility.SerializeDateTime(r.EndTime.ToUniversalTime()); } } - private static void UpdateMetadataLastVacuum(Duplicati.Server.Serialization.Interface.IBackup backup, Duplicati.Library.Interface.IVacuumResults r) + private static void UpdateMetadataLastVacuum(IBackup backup, IVacuumResults r) { if (r != null) { backup.Metadata["LastVacuumDuration"] = r.Duration.ToString(); - backup.Metadata["LastVacuumStarted"] = Library.Utility.Utility.SerializeDateTime(r.BeginTime.ToUniversalTime()); - backup.Metadata["LastVacuumFinished"] = Library.Utility.Utility.SerializeDateTime(r.EndTime.ToUniversalTime()); + backup.Metadata["LastVacuumStarted"] = Utility.SerializeDateTime(r.BeginTime.ToUniversalTime()); + backup.Metadata["LastVacuumFinished"] = Utility.SerializeDateTime(r.EndTime.ToUniversalTime()); } } - private static void UpdateMetadata(Duplicati.Server.Serialization.Interface.IBackup backup, Duplicati.Library.Interface.IParsedBackendStatistics r) + private static void UpdateMetadataStatistics(IBackup backup, IParsedBackendStatistics r) { if (r != null) { - backup.Metadata["LastBackupDate"] = Library.Utility.Utility.SerializeDateTime(r.LastBackupDate.ToUniversalTime()); + backup.Metadata["LastBackupDate"] = Utility.SerializeDateTime(r.LastBackupDate.ToUniversalTime()); backup.Metadata["BackupListCount"] = r.BackupListCount.ToString(); backup.Metadata["TotalQuotaSpace"] = r.TotalQuotaSpace.ToString(); backup.Metadata["FreeQuotaSpace"] = r.FreeQuotaSpace.ToString(); @@ -837,28 +861,28 @@ namespace Duplicati.Server backup.Metadata["TargetFilesSize"] = r.KnownFileSize.ToString(); backup.Metadata["TargetFilesCount"] = r.KnownFileCount.ToString(); backup.Metadata["TargetFilesetsCount"] = r.KnownFilesets.ToString(); - backup.Metadata["TargetSizeString"] = Duplicati.Library.Utility.Utility.FormatSizeString(r.KnownFileSize); + backup.Metadata["TargetSizeString"] = Utility.FormatSizeString(r.KnownFileSize); } } - private static void UpdateMetadata(Duplicati.Server.Serialization.Interface.IBackup backup, Duplicati.Library.Interface.IBasicResults result) + private static void UpdateMetadataBase(Connection databaseConnection, EventPollNotify eventPollNotify, Serialization.Interface.IBackup backup, IBasicResults result) { if (result is IRestoreResults r1) { backup.Metadata["LastRestoreDuration"] = r1.Duration.ToString(); - backup.Metadata["LastRestoreStarted"] = Library.Utility.Utility.SerializeDateTime(result.BeginTime.ToUniversalTime()); - backup.Metadata["LastRestoreFinished"] = Library.Utility.Utility.SerializeDateTime(result.EndTime.ToUniversalTime()); + backup.Metadata["LastRestoreStarted"] = Utility.SerializeDateTime(result.BeginTime.ToUniversalTime()); + backup.Metadata["LastRestoreFinished"] = Utility.SerializeDateTime(result.EndTime.ToUniversalTime()); } if (result is IParsedBackendStatistics r2 && !result.Interrupted) { - UpdateMetadata(backup, r2); + UpdateMetadataStatistics(backup, r2); } if (result is IBackendStatsticsReporter r3 && !result.Interrupted) { if (r3.BackendStatistics is IParsedBackendStatistics statistics) - UpdateMetadata(backup, statistics); + UpdateMetadataStatistics(backup, statistics); } if (result is ICompactResults r4 && !result.Interrupted) @@ -881,8 +905,8 @@ namespace Duplicati.Server backup.Metadata["SourceFilesSize"] = r.SizeOfExaminedFiles.ToString(); backup.Metadata["SourceFilesCount"] = r.ExaminedFiles.ToString(); backup.Metadata["SourceSizeString"] = Duplicati.Library.Utility.Utility.FormatSizeString(r.SizeOfExaminedFiles); - backup.Metadata["LastBackupStarted"] = Library.Utility.Utility.SerializeDateTime(r.BeginTime.ToUniversalTime()); - backup.Metadata["LastBackupFinished"] = Library.Utility.Utility.SerializeDateTime(r.EndTime.ToUniversalTime()); + backup.Metadata["LastBackupStarted"] = Utility.SerializeDateTime(r.BeginTime.ToUniversalTime()); + backup.Metadata["LastBackupFinished"] = Utility.SerializeDateTime(r.EndTime.ToUniversalTime()); backup.Metadata["LastBackupDuration"] = r.Duration.ToString(); if (r.CompactResults != null) @@ -912,7 +936,7 @@ namespace Duplicati.Server titleType = "Warning"; } - FIXMEGlobal.DataConnection.RegisterNotification( + databaseConnection.RegisterNotification( r.FilesWithError == 0 && !r.Errors.Any() ? NotificationType.Warning : NotificationType.Error, backup.IsTemporary ? "Warning" : $"{titleType} while running {backup.Name}", message, @@ -958,7 +982,7 @@ namespace Duplicati.Server else if (result.ParsedResult == ParsedResultType.Error && result.Errors.Count() == 1) message = $"Error: {result.Errors.Single()}"; - FIXMEGlobal.DataConnection.RegisterNotification( + databaseConnection.RegisterNotification( type, title, message, @@ -973,10 +997,10 @@ namespace Duplicati.Server } if (!backup.IsTemporary) - FIXMEGlobal.DataConnection.SetMetadata(backup.Metadata, long.Parse(backup.ID), null); + databaseConnection.SetMetadata(backup.Metadata, long.Parse(backup.ID), null); FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); - FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); + eventPollNotify.SignalNewEvent(); } private static bool TestIfOptionApplies() @@ -998,7 +1022,7 @@ namespace Duplicati.Server options["disable-module"] = string.Join(",", mods.Union(new string[] { module }).Distinct(StringComparer.OrdinalIgnoreCase)); } - internal static Dictionary ApplyOptions(Serialization.Interface.IBackup backup, Dictionary options) + internal static Dictionary ApplyOptions(Connection databaseConnection, Serialization.Interface.IBackup backup, Dictionary options) { options["backup-name"] = backup.Name; options["dbpath"] = backup.DBPath; @@ -1018,7 +1042,7 @@ namespace Duplicati.Server DisableModule("console-password-input", options); // Patch in additional report urls - var additionalReportUrl = FIXMEGlobal.DataConnection.ApplicationSettings.AdditionalReportUrl; + var additionalReportUrl = databaseConnection.ApplicationSettings.AdditionalReportUrl; if (!string.IsNullOrWhiteSpace(additionalReportUrl)) { options["send-http-json-urls"] = string.Join(";", @@ -1032,7 +1056,7 @@ namespace Duplicati.Server return options; } - private static IFilter? ApplyFilter(Serialization.Interface.IBackup backup, IFilter? filter) + private static Library.Utility.IFilter? ApplyFilter(IBackup backup, Library.Utility.IFilter? filter) { var f2 = backup.Filters; if (f2 != null && f2.Length > 0) @@ -1044,7 +1068,7 @@ namespace Duplicati.Server ? SpecialFolders.ExpandEnvironmentVariablesRegexp(n.Expression) : SpecialFolders.ExpandEnvironmentVariables(n.Expression) orderby n.Order - select (IFilter)new FilterExpression(exp, n.Include)) + select (Library.Utility.IFilter)new FilterExpression(exp, n.Include)) .Aggregate((a, b) => FilterExpression.Combine(a, b)); filter = FilterExpression.Combine(filter, nf); @@ -1053,17 +1077,17 @@ namespace Duplicati.Server return filter; } - public static Dictionary GetCommonOptions() + public static Dictionary GetCommonOptions(Connection databaseConnection) { return - (from n in FIXMEGlobal.DataConnection.Settings + (from n in databaseConnection.Settings where TestIfOptionApplies() select n).ToDictionary(k => k.Name.StartsWith("--", StringComparison.Ordinal) ? k.Name.Substring(2) : k.Name, k => (string?)k.Value); } - private static IFilter? GetCommonFilter() + private static Library.Utility.IFilter? GetCommonFilter(Connection databaseConnection) { - var filters = FIXMEGlobal.DataConnection.Filters; + var filters = databaseConnection.Filters; if (filters == null || filters.Length == 0) return null; @@ -1071,7 +1095,7 @@ namespace Duplicati.Server (from n in filters orderby n.Order let exp = Environment.ExpandEnvironmentVariables(n.Expression) - select (IFilter)new FilterExpression(exp, n.Include)) + select (Library.Utility.IFilter)new FilterExpression(exp, n.Include)) .Aggregate((a, b) => FilterExpression.Combine(a, b)); } } diff --git a/Duplicati/Library/RestAPI/Scheduler.cs b/Duplicati/Library/RestAPI/Scheduler.cs index 388a850b7..bc1f175cc 100644 --- a/Duplicati/Library/RestAPI/Scheduler.cs +++ b/Duplicati/Library/RestAPI/Scheduler.cs @@ -29,9 +29,9 @@ using System.Text; using System.Linq; using System.Threading; using Duplicati.Library.Utility; -using Duplicati.Library.RestAPI; using System.Threading.Tasks; using Duplicati.WebserverCore.Abstractions; +using Duplicati.Server.Database; // TODO: Rewrite this class. // It should just signal what new backups to run, and not mix with the worker thread. @@ -69,6 +69,10 @@ namespace Duplicati.Server /// The queue runner service /// private readonly IQueueRunnerService m_queueRunnerService; + /// + /// The data connection + /// + private readonly Connection m_dataConnection; /// /// The currently scheduled items @@ -83,8 +87,11 @@ namespace Duplicati.Server /// /// Constructs a new scheduler /// - public Scheduler(IQueueRunnerService queueRunnerService) + /// The data connection + /// The queue runner service + public Scheduler(Connection connection, IQueueRunnerService queueRunnerService) { + m_dataConnection = connection; m_queueRunnerService = queueRunnerService; m_thread = new Thread(new ThreadStart(Runner)); m_schedule = []; @@ -99,7 +106,7 @@ namespace Duplicati.Server public IList> GetProposedSchedule() { return ( - from n in FIXMEGlobal.Scheduler.Schedule + from n in this.Schedule let backupid = (from t in n.Value.Tags where t != null && t.StartsWith("ID=", StringComparison.Ordinal) select t.Substring("ID=".Length)).FirstOrDefault() @@ -220,7 +227,7 @@ namespace Duplicati.Server { t.Item1.Time = t.Item2; t.Item1.LastRun = t.Item3; - FIXMEGlobal.DataConnection.AddOrUpdateSchedule(t.Item1); + m_dataConnection.AddOrUpdateSchedule(t.Item1); } return Task.CompletedTask; @@ -256,8 +263,8 @@ namespace Duplicati.Server // to avoid frequent db lookups //Determine schedule list - var timeZoneInfo = FIXMEGlobal.DataConnection.ApplicationSettings.Timezone; - var lst = FIXMEGlobal.DataConnection.Schedules; + var timeZoneInfo = m_dataConnection.ApplicationSettings.Timezone; + var lst = m_dataConnection.Schedules; foreach (var sc in lst) { if (!string.IsNullOrEmpty(sc.Repeat)) @@ -291,7 +298,7 @@ namespace Duplicati.Server } catch (Exception ex) { - FIXMEGlobal.DataConnection.LogError(sc.ID.ToString(), "Scheduler failed to find next date", + m_dataConnection.LogError(sc.ID.ToString(), "Scheduler failed to find next date", ex); } @@ -300,7 +307,7 @@ namespace Duplicati.Server { var jobsToRun = new List(); //TODO: Cache this to avoid frequent lookups - foreach (var id in FIXMEGlobal.DataConnection.GetBackupIDsForTags(sc.Tags).Distinct() + foreach (var id in m_dataConnection.GetBackupIDsForTags(sc.Tags).Distinct() .Select(x => x.ToString())) { //See if it is already queued @@ -315,11 +322,11 @@ namespace Duplicati.Server //If it is not already in queue, put it there if (!tmplst.Any(x => x == id)) { - var entry = FIXMEGlobal.DataConnection.GetBackup(id); + var entry = m_dataConnection.GetBackup(id); if (entry != null) { - var options = Server.Runner.GetCommonOptions(); - Server.Runner.ApplyOptions(entry, options); + var options = Server.Runner.GetCommonOptions(m_dataConnection); + Server.Runner.ApplyOptions(m_dataConnection, entry, options); if (new Library.Main.Options(options).DisableOnBattery && (Library.Utility.Power.PowerSupply.GetSource() == Library.Utility.Power.PowerSupply.Source.Battery)) @@ -364,7 +371,7 @@ namespace Duplicati.Server } catch (Exception ex) { - FIXMEGlobal.DataConnection.LogError(sc.ID.ToString(), + m_dataConnection.LogError(sc.ID.ToString(), "Scheduler failed to find next date", ex); continue; } diff --git a/Duplicati/Server/Duplicati.Server.Serialization/Interface/IQueuedTask.cs b/Duplicati/Server/Duplicati.Server.Serialization/Interface/IQueuedTask.cs index cb95b681a..ccaa14a35 100644 --- a/Duplicati/Server/Duplicati.Server.Serialization/Interface/IQueuedTask.cs +++ b/Duplicati/Server/Duplicati.Server.Serialization/Interface/IQueuedTask.cs @@ -53,10 +53,6 @@ public interface IQueuedTask /// Func? OnFinished { get; set; } - /// - /// That action that performs the task. - /// - Task Execute(); /// /// Updates the throttle speeds for the task. /// diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index 933cca135..794ceb955 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -99,11 +99,6 @@ namespace Duplicati.Server /// public static Database.Connection DataConnection { get => FIXMEGlobal.DataConnection; set => FIXMEGlobal.DataConnection = value; } - /// - /// This is the scheduling thread - /// - public static ISchedulerService Scheduler { get => FIXMEGlobal.Scheduler; } - /// /// The thread running the ping-pong handler /// @@ -137,20 +132,10 @@ namespace Duplicati.Server DuplicatiWebserver.Stop().GetAwaiter().GetResult(); } - /// - /// The update poll thread. - /// - public static UpdatePollThread UpdatePoller => FIXMEGlobal.UpdatePoller; - /// /// An event that is set once the server is ready to respond to requests /// - public static readonly System.Threading.ManualResetEvent ServerStartedEvent = new System.Threading.ManualResetEvent(false); - - /// - /// The status event signaler, used to control long polling of status updates - /// - public static EventPollNotify StatusEventNotifyer => FIXMEGlobal.Provider.GetRequiredService(); + public static readonly ManualResetEvent ServerStartedEvent = new ManualResetEvent(false); /// /// A delegate method for creating a copy of the current progress state @@ -244,6 +229,9 @@ namespace Duplicati.Server var crashed = false; var terminated = false; IQueueRunnerService queueRunner = null; + UpdatePollThread updatePollThread = null; + EventPollNotify eventPollNotify = null; + ISchedulerService scheduler = null; try { DataConnection = GetDatabaseConnection(commandlineOptions, silentConsole); @@ -267,14 +255,17 @@ namespace Duplicati.Server DuplicatiWebserver = StartWebServer(commandlineOptions, DataConnection).Await(); - queueRunner = DuplicatiWebserver.Provider.GetRequiredService(); DataConnection.SetServiceProvider(DuplicatiWebserver.Provider); + queueRunner = DuplicatiWebserver.Provider.GetRequiredService(); + updatePollThread = DuplicatiWebserver.Provider.GetRequiredService(); + eventPollNotify = DuplicatiWebserver.Provider.GetRequiredService(); + scheduler = DuplicatiWebserver.Provider.GetRequiredService(); - UpdatePoller.Init(Library.Utility.Utility.ParseBoolOption(commandlineOptions, DISABLE_UPDATE_CHECK_OPTION)); + updatePollThread.Init(Library.Utility.Utility.ParseBoolOption(commandlineOptions, DISABLE_UPDATE_CHECK_OPTION)); SetPurgeTempFilesTimer(commandlineOptions); - LiveControl.StateChanged = (e) => { LiveControl_StateChanged(queueRunner, DataConnection, StatusEventNotifyer, e); }; + LiveControl.StateChanged = (e) => { LiveControl_StateChanged(queueRunner, DataConnection, eventPollNotify, e); }; if (Library.Utility.Utility.ParseBoolOption(commandlineOptions, PING_PONG_KEEPALIVE_OPTION)) { @@ -361,10 +352,10 @@ namespace Duplicati.Server Log.WriteInformationMessage(LOGTAG, "ServerStopping", Strings.Program.ServerStopping); var steps = new Action[] { - () => StatusEventNotifyer.SignalNewEvent(), - () => { if (ShutdownModernWebserver != null) ShutdownModernWebserver(); }, - () => UpdatePoller?.Terminate(), - () => Scheduler?.Terminate(true), + () => eventPollNotify?.SignalNewEvent(), + () => ShutdownModernWebserver(), + () => updatePollThread?.Terminate(), + () => scheduler?.Terminate(true), () => queueRunner?.Terminate(true), () => ApplicationInstance?.Dispose(), () => PurgeTempFilesTimer?.Dispose(), @@ -844,11 +835,6 @@ namespace Duplicati.Server Library.UsageReporter.Reporter.SetReportLevel(reportLevel, disableUsageReporter); } - private static void SignalNewEvent(object sender, EventArgs e) - { - StatusEventNotifyer.SignalNewEvent(); - } - /// /// This event handler updates the trayicon menu with the current state of the runner. /// diff --git a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs index acad0cfac..da2e33802 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs @@ -41,8 +41,8 @@ public class BackupGet : IEndpointV1 => ExecuteGet(connection, GetBackup(connection, id))) .RequireAuthorization(); - group.MapGet("/backup/{id}/files", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery] string? filter, [FromQuery] string? time, [FromQuery(Name = "all-versions")] bool? allVersions, [FromQuery(Name = "prefix-only")] bool? prefixOnly, [FromQuery(Name = "folder-contents")] bool? folderContents) - => ExecuteGetFiles(GetBackup(connection, id), filter, time, allVersions ?? false, prefixOnly ?? false, folderContents ?? false, new Dictionary())) + group.MapGet("/backup/{id}/files", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromRoute] string id, [FromQuery] string? filter, [FromQuery] string? time, [FromQuery(Name = "all-versions")] bool? allVersions, [FromQuery(Name = "prefix-only")] bool? prefixOnly, [FromQuery(Name = "folder-contents")] bool? folderContents) + => ExecuteGetFiles(connection, eventPollNotify, GetBackup(connection, id), filter, time, allVersions ?? false, prefixOnly ?? false, folderContents ?? false, new Dictionary())) .RequireAuthorization(); group.MapGet("/backup/{id}/log", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery] long? offset, [FromQuery] long? pagesize) @@ -53,16 +53,16 @@ public class BackupGet : IEndpointV1 => ExecuteGetRemotelog(connection, GetBackup(connection, id), offset, pagesize ?? 100)) .RequireAuthorization(); - group.MapGet("/backup/{id}/filesets", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery(Name = "include-metadata")] bool? includeMetadata, [FromQuery(Name = "from-remote-only")] bool? fromRemoteOnly) - => ExecuteGetFilesets(GetBackup(connection, id), includeMetadata ?? false, fromRemoteOnly ?? false)) + group.MapGet("/backup/{id}/filesets", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromRoute] string id, [FromQuery(Name = "include-metadata")] bool? includeMetadata, [FromQuery(Name = "from-remote-only")] bool? fromRemoteOnly) + => ExecuteGetFilesets(connection, eventPollNotify, GetBackup(connection, id), includeMetadata ?? false, fromRemoteOnly ?? false)) .RequireAuthorization(); group.MapGet("/backup/{id}/export-argsonly", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery(Name = "export-passwords")] bool? exportPasswords, [FromQuery] string? passphrase) - => ExecuteGetExportArgsOnly(GetBackup(connection, id), exportPasswords ?? false)) + => ExecuteGetExportArgsOnly(connection, GetBackup(connection, id), exportPasswords ?? false)) .RequireAuthorization(); group.MapGet("/backup/{id}/export-cmdline", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery(Name = "export-passwords")] bool? exportPasswords, [FromQuery] string? passphrase) - => ExecuteGetExportCmdline(GetBackup(connection, id), exportPasswords ?? false)) + => ExecuteGetExportCmdline(connection, GetBackup(connection, id), exportPasswords ?? false)) .RequireAuthorization(); group.MapGet("/backup/{id}/export", ([FromServices] Connection connection, [FromServices] IHttpContextAccessor httpContextAccessor, [FromServices] IJWTTokenProvider jWTTokenProvider, [FromRoute] string id, [FromQuery(Name = "export-passwords")] bool? exportPasswords, [FromQuery] string? passphrase, [FromQuery] string token, CancellationToken ct) => @@ -142,7 +142,7 @@ public class BackupGet : IEndpointV1 ); } - private static Dictionary SearchFiles(IBackup backup, string? filter, string? timestring, bool allVersions, bool prefixOnly, bool folderContents, Dictionary extraValues) + private static Dictionary SearchFiles(Connection connection, EventPollNotify eventPollNotify, IBackup backup, string? filter, string? timestring, bool allVersions, bool prefixOnly, bool folderContents, Dictionary extraValues) { if (string.IsNullOrWhiteSpace(timestring) && !allVersions) throw new BadRequestException("Invalid or missing time"); @@ -151,7 +151,7 @@ public class BackupGet : IEndpointV1 if (!allVersions) time = Library.Utility.Timeparser.ParseTimeInterval(timestring, DateTime.Now); - var r = Runner.Run(Runner.CreateListTask(backup, filter == null ? null : [filter], prefixOnly, allVersions, folderContents, time), false) as Duplicati.Library.Interface.IListResults; + var r = Runner.Run(connection, eventPollNotify, Runner.CreateListTask(backup, filter == null ? null : [filter], prefixOnly, allVersions, folderContents, time), false) as IListResults; if (r == null) throw new ServerErrorException("No result from list operation"); @@ -170,8 +170,8 @@ public class BackupGet : IEndpointV1 return result; } - private static Dictionary ExecuteGetFiles(IBackup bk, string? filter, string? timestring, bool allVersions, bool prefixOnly, bool folderContents, Dictionary extraValues) - => SearchFiles(bk, filter, timestring, allVersions, prefixOnly, folderContents, extraValues); + private static Dictionary ExecuteGetFiles(Connection connection, EventPollNotify eventPollNotify, IBackup bk, string? filter, string? timestring, bool allVersions, bool prefixOnly, bool folderContents, Dictionary extraValues) + => SearchFiles(connection, eventPollNotify, bk, filter, timestring, allVersions, prefixOnly, folderContents, extraValues); private static List> ExecuteGetLog(Connection connection, IBackup bk, long? offset, long pagesize) { @@ -202,7 +202,7 @@ public class BackupGet : IEndpointV1 } } - private static IEnumerable ExecuteGetFilesets(IBackup bk, bool includeMetadata, bool fromRemoteOnly) + private static IEnumerable ExecuteGetFilesets(Connection connection, EventPollNotify eventPollNotify, IBackup bk, bool includeMetadata, bool fromRemoteOnly) { var extra = new Dictionary { @@ -218,7 +218,7 @@ public class BackupGet : IEndpointV1 try { - var r = Runner.Run(Runner.CreateTask(DuplicatiOperation.List, bk, extra), false) as IListResults; + var r = Runner.Run(connection, eventPollNotify, Runner.CreateTask(DuplicatiOperation.List, bk, extra), false) as IListResults; if (r == null) throw new ServerErrorException("No result from list operation"); @@ -239,20 +239,20 @@ public class BackupGet : IEndpointV1 backup.SanitizeTargetUrl(); } - private static Dto.ExportCommandlineDto ExecuteGetExportCmdline(IBackup backup, bool exportPasswords) + private static Dto.ExportCommandlineDto ExecuteGetExportCmdline(Connection connection, IBackup backup, bool exportPasswords) { if (!exportPasswords) RemovePasswords(backup); - return new Dto.ExportCommandlineDto(Runner.GetCommandLine(Runner.CreateTask(DuplicatiOperation.Backup, backup))); + return new Dto.ExportCommandlineDto(Runner.GetCommandLine(connection, Runner.CreateTask(DuplicatiOperation.Backup, backup))); } - private static Dto.ExportArgsOnlyDto ExecuteGetExportArgsOnly(IBackup backup, bool exportPasswords) + private static Dto.ExportArgsOnlyDto ExecuteGetExportArgsOnly(Connection connection, IBackup backup, bool exportPasswords) { if (!exportPasswords) RemovePasswords(backup); - var parts = Runner.GetCommandLineParts(Runner.CreateTask(DuplicatiOperation.Backup, backup)); + var parts = Runner.GetCommandLineParts(connection, Runner.CreateTask(DuplicatiOperation.Backup, backup)); return new Dto.ExportArgsOnlyDto( parts.First(), parts.Skip(1).Where(x => !x.StartsWith("--", StringComparison.Ordinal)), diff --git a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupPutDelete.cs b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupPutDelete.cs index 754cab235..77e763e42 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupPutDelete.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupPutDelete.cs @@ -36,9 +36,9 @@ public class BackupPutDelete : IEndpointV1 => ExecutePut(GetBackup(connection, id), connection, input)) .RequireAuthorization(); - group.MapDelete("/backup/{id}", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromServices] LiveControls liveControls, [FromServices] IHttpContextAccessor httpContextAccessor, [FromRoute] string id, [FromQuery(Name = "delete-remote-files")] bool? delete_remote_files, [FromQuery(Name = "delete-local-db")] bool? delete_local_db, [FromQuery] bool? force) => + group.MapDelete("/backup/{id}", ([FromServices] Connection connection, [FromServices] ISchedulerService schedulerService, [FromServices] IQueueRunnerService queueRunnerService, [FromServices] LiveControls liveControls, [FromServices] IHttpContextAccessor httpContextAccessor, [FromRoute] string id, [FromQuery(Name = "delete-remote-files")] bool? delete_remote_files, [FromQuery(Name = "delete-local-db")] bool? delete_local_db, [FromQuery] bool? force) => { - var res = ExecuteDelete(GetBackup(connection, id), queueRunnerService, liveControls, delete_remote_files ?? false, delete_local_db, force ?? false); + var res = ExecuteDelete(GetBackup(connection, id), schedulerService, queueRunnerService, liveControls, delete_remote_files ?? false, delete_local_db, force ?? false); if (res.Status != "OK" && httpContextAccessor.HttpContext != null) httpContextAccessor.HttpContext.Response.StatusCode = 500; return res; @@ -130,7 +130,7 @@ public class BackupPutDelete : IEndpointV1 } } - private static Dto.DeleteBackupOutputDto ExecuteDelete(IBackup backup, IQueueRunnerService queueRunnerService, LiveControls liveControls, bool delete_remote_files, bool? delete_local_db, bool force) + private static Dto.DeleteBackupOutputDto ExecuteDelete(IBackup backup, ISchedulerService schedulerService, IQueueRunnerService queueRunnerService, LiveControls liveControls, bool delete_remote_files, bool? delete_local_db, bool force) { try { @@ -179,7 +179,7 @@ public class BackupPutDelete : IEndpointV1 if (delete_remote_files) extra["delete-remote-files"] = "true"; - return new Dto.DeleteBackupOutputDto("OK", null, queueRunnerService.AddTask(Runner.CreateTask(DuplicatiOperation.Delete, backup, extra))); + return new Dto.DeleteBackupOutputDto("OK", null, queueRunnerService.AddTask(Runner.CreateDeleteTask(backup, extra, (_) => schedulerService.Reschedule()))); } } diff --git a/Duplicati/WebserverCore/Endpoints/V1/RemoteOperation.cs b/Duplicati/WebserverCore/Endpoints/V1/RemoteOperation.cs index 8771de22d..3abd2d4ef 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/RemoteOperation.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/RemoteOperation.cs @@ -22,6 +22,7 @@ using Duplicati.Library.Interface; using Duplicati.Library.Main; using Duplicati.Library.RestAPI; using Duplicati.Server; +using Duplicati.Server.Database; using Duplicati.WebserverCore.Abstractions; using Duplicati.WebserverCore.Exceptions; using Microsoft.AspNetCore.Mvc; @@ -38,8 +39,8 @@ namespace Duplicati.WebserverCore.Endpoints.V1 => ExecuteDbPath(input.path)) .RequireAuthorization(); - group.MapPost("/remoteoperation/test", ([FromQuery] bool? autocreate, [FromBody] RemoteOperationInput input, CancellationToken cancelToken) - => ExecuteTest(input.path, autocreate ?? false, cancelToken)) + group.MapPost("/remoteoperation/test", ([FromServices] Connection connection, [FromQuery] bool? autocreate, [FromBody] RemoteOperationInput input, CancellationToken cancelToken) + => ExecuteTest(connection, input.path, autocreate ?? false, cancelToken)) .RequireAuthorization(); group.MapPost("/remoteoperation/create", ([FromBody] RemoteOperationInput input, CancellationToken cancelToken) @@ -53,11 +54,11 @@ namespace Duplicati.WebserverCore.Endpoints.V1 return new Dto.GetDbPathDto(!string.IsNullOrWhiteSpace(path), path); } - private static Dictionary ParseUrlOptions(Library.Utility.Uri uri) + private static Dictionary ParseUrlOptions(Connection connection, Library.Utility.Uri uri) { var qp = uri.QueryParameters; - var opts = Runner.GetCommonOptions(); + var opts = Runner.GetCommonOptions(connection); foreach (var k in qp.Keys.Cast()) opts[k] = qp[k]; @@ -87,10 +88,10 @@ namespace Duplicati.WebserverCore.Endpoints.V1 } } - private static async Task GetBackend(string url, CancellationToken cancelToken) + private static async Task GetBackend(Connection connection, string url, CancellationToken cancelToken) { var uri = new Library.Utility.Uri(url); - var opts = ParseUrlOptions(uri); + var opts = ParseUrlOptions(connection, uri); var tmp = new[] { uri }; await SecretProviderHelper.ApplySecretProviderAsync([], tmp, opts, Library.Utility.TempFolder.SystemTempPath, FIXMEGlobal.SecretProvider, cancelToken); @@ -101,13 +102,13 @@ namespace Duplicati.WebserverCore.Endpoints.V1 return new TupleDisposeWrapper(backend, modules); } - private static async Task ExecuteTest(string url, bool autoCreate, CancellationToken cancelToken) + private static async Task ExecuteTest(Connection connection, string url, bool autoCreate, CancellationToken cancelToken) { TupleDisposeWrapper? wrapper = null; try { - wrapper = await GetBackend(url, cancelToken); + wrapper = await GetBackend(connection, url, cancelToken); using (var b = wrapper.Backend) { diff --git a/Duplicati/WebserverCore/Endpoints/V1/Updates.cs b/Duplicati/WebserverCore/Endpoints/V1/Updates.cs index 86e252dbf..029f60750 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Updates.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Updates.cs @@ -18,8 +18,9 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using Duplicati.Library.RestAPI; +using Duplicati.Server; using Duplicati.WebserverCore.Abstractions; +using Microsoft.AspNetCore.Mvc; namespace Duplicati.WebserverCore.Endpoints.V1; @@ -27,9 +28,11 @@ public class Updates : IEndpointV1 { public static void Map(RouteGroupBuilder group) { - group.MapPost("/updates/check", Execute).RequireAuthorization(); + group.MapPost("/updates/check", + ([FromServices] UpdatePollThread updatePollThread) => Execute(updatePollThread)) + .RequireAuthorization(); } - private static void Execute() - => FIXMEGlobal.UpdatePoller.CheckNow(); + private static void Execute(UpdatePollThread updatePollThread) + => updatePollThread.CheckNow(); } \ No newline at end of file diff --git a/Duplicati/WebserverCore/Endpoints/V1/WebModules.cs b/Duplicati/WebserverCore/Endpoints/V1/WebModules.cs index 7c35c171f..20421f853 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/WebModules.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/WebModules.cs @@ -22,6 +22,7 @@ using Duplicati.Library.Interface; using Duplicati.Library.Main; using Duplicati.Library.RestAPI; using Duplicati.Server; +using Duplicati.Server.Database; using Duplicati.WebserverCore.Abstractions; using Duplicati.WebserverCore.Exceptions; using Microsoft.AspNetCore.Mvc; @@ -33,18 +34,20 @@ public record WebModules : IEndpointV1 public static void Map(RouteGroupBuilder group) { group.MapGet("/webmodules", ExecuteGet).RequireAuthorization(); - group.MapPost("/webmodule/{modulekey}", ([FromRoute] string modulekey, [FromBody] Dictionary options, CancellationToken cancellationToken) => ExecutePost(modulekey, options, cancellationToken)).RequireAuthorization(); + group.MapPost("/webmodule/{modulekey}", ([FromServices] Connection connection, [FromRoute] string modulekey, [FromBody] Dictionary options, CancellationToken cancellationToken) + => ExecutePost(connection, modulekey, options, cancellationToken)) + .RequireAuthorization(); } private static IEnumerable ExecuteGet() => Library.DynamicLoader.WebLoader.Modules; - private static async Task ExecutePost(string modulekey, Dictionary inputOptions, CancellationToken cancellationToken) + private static async Task ExecutePost(Connection connection, string modulekey, Dictionary inputOptions, CancellationToken cancellationToken) { var m = Library.DynamicLoader.WebLoader.Modules.FirstOrDefault(x => x.Key.Equals(modulekey, StringComparison.OrdinalIgnoreCase)) ?? throw new NotFoundException("No such module found"); - var options = Runner.GetCommonOptions(); + var options = Runner.GetCommonOptions(connection); foreach (var k in inputOptions.Keys) options[k] = inputOptions[k]; diff --git a/Duplicati/WebserverCore/Endpoints/V2/BackupListing.cs b/Duplicati/WebserverCore/Endpoints/V2/BackupListing.cs index d013f73d5..5e301489e 100644 --- a/Duplicati/WebserverCore/Endpoints/V2/BackupListing.cs +++ b/Duplicati/WebserverCore/Endpoints/V2/BackupListing.cs @@ -33,33 +33,33 @@ public class BackupListing : IEndpointV2 { public static void Map(RouteGroupBuilder group) { - group.MapPost("/backup/list-filesets", ([FromServices] Connection connection, [FromBody] Dto.V2.ListFilesetsRequestDto input) - => ExecuteGetFilesets(GetBackup(connection, input.BackupId))) + group.MapPost("/backup/list-filesets", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromBody] Dto.V2.ListFilesetsRequestDto input) + => ExecuteGetFilesets(connection, eventPollNotify, GetBackup(connection, input.BackupId))) .RequireAuthorization(); - group.MapPost("/backup/list-folder", ([FromServices] Connection connection, [FromBody] Dto.V2.ListFolderContentRequestDto input) - => ExecuteListFolder(GetBackup(connection, input.BackupId), input)) + group.MapPost("/backup/list-folder", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromBody] Dto.V2.ListFolderContentRequestDto input) + => ExecuteListFolder(connection, eventPollNotify, GetBackup(connection, input.BackupId), input)) .RequireAuthorization(); - group.MapPost("/backup/list-versions", ([FromServices] Connection connection, [FromBody] Dto.V2.ListFileVersionsRequestDto input) - => ExecuteListVersions(GetBackup(connection, input.BackupId), input)) + group.MapPost("/backup/list-versions", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromBody] Dto.V2.ListFileVersionsRequestDto input) + => ExecuteListVersions(connection, eventPollNotify, GetBackup(connection, input.BackupId), input)) .RequireAuthorization(); - group.MapPost("/backup/search", ([FromServices] Connection connection, [FromBody] Dto.V2.SearchEntriesRequestDto input) - => ExecuteSearch(GetBackup(connection, input.BackupId), input)) + group.MapPost("/backup/search", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromBody] Dto.V2.SearchEntriesRequestDto input) + => ExecuteSearch(connection, eventPollNotify, GetBackup(connection, input.BackupId), input)) .RequireAuthorization(); } private static IBackup GetBackup(Connection connection, string id) => connection.GetBackup(id) ?? throw new NotFoundException("Backup not found"); - private static Dto.V2.ListFolderContentResponseDto ExecuteListFolder(IBackup bk, Dto.V2.ListFolderContentRequestDto input) + private static Dto.V2.ListFolderContentResponseDto ExecuteListFolder(Connection connection, EventPollNotify eventPollNotify, IBackup bk, Dto.V2.ListFolderContentRequestDto input) { var time = string.IsNullOrWhiteSpace(input.Time) ? new DateTime(0) : Library.Utility.Timeparser.ParseTimeInterval(input.Time, DateTime.Now); - var r = Runner.Run(Runner.CreateListFolderContents(bk, input.Paths, time, input.PageSize ?? 1000, input.Page ?? 0), false) as IListFolderResults; + var r = Runner.Run(connection, eventPollNotify, Runner.CreateListFolderContents(bk, input.Paths, time, input.PageSize ?? 1000, input.Page ?? 0), false) as IListFolderResults; if (r == null) throw new ServerErrorException("No result from list operation"); @@ -78,16 +78,16 @@ public class BackupListing : IEndpointV2 r.Entries.TotalCount); } - private static Dto.V2.ListFilesetsResponseDto ExecuteGetFilesets(IBackup bk) + private static Dto.V2.ListFilesetsResponseDto ExecuteGetFilesets(Connection connection, EventPollNotify eventPollNotify, IBackup bk) { - var extra = new Dictionary(); + var extra = new Dictionary(); // Retries will hang the http request extra["number-of-retries"] = "0"; try { - var r = Runner.Run(Runner.CreateListFilesetsTask(bk, extra), false) as IListFilesetResults; + var r = Runner.Run(connection, eventPollNotify, Runner.CreateListFilesetsTask(bk, extra), false) as IListFilesetResults; if (r == null) throw new ServerErrorException("No result from list operation"); @@ -112,9 +112,9 @@ public class BackupListing : IEndpointV2 } } - private static Dto.V2.ListFileVersionsOutputDto ExecuteListVersions(IBackup bk, Dto.V2.ListFileVersionsRequestDto input) + private static Dto.V2.ListFileVersionsOutputDto ExecuteListVersions(Connection connection, EventPollNotify eventPollNotify, IBackup bk, Dto.V2.ListFileVersionsRequestDto input) { - var r = Runner.Run(Runner.ListFileVersionsTask(bk, input.Paths, input.PageSize ?? 1000, input.Page ?? 0), false) as IListFileVersionsResults; + var r = Runner.Run(connection, eventPollNotify, Runner.ListFileVersionsTask(bk, input.Paths, input.PageSize ?? 1000, input.Page ?? 0), false) as IListFileVersionsResults; if (r == null) throw new ServerErrorException("No result from list operation"); @@ -135,13 +135,13 @@ public class BackupListing : IEndpointV2 r.FileVersions.TotalCount); } - private static Dto.V2.SearchEntriesResponseDto ExecuteSearch(IBackup bk, Dto.V2.SearchEntriesRequestDto input) + private static Dto.V2.SearchEntriesResponseDto ExecuteSearch(Connection connection, EventPollNotify eventPollNotify, IBackup bk, Dto.V2.SearchEntriesRequestDto input) { var time = string.IsNullOrWhiteSpace(input.Time) ? new DateTime(0) : Library.Utility.Timeparser.ParseTimeInterval(input.Time, DateTime.Now); - var r = Runner.Run(Runner.CreateSearchEntriesTask(bk, input.Filters, input.Paths, time, input.PageSize ?? 1000, input.Page ?? 0), false) as ISearchFilesResults; + var r = Runner.Run(connection, eventPollNotify, Runner.CreateSearchEntriesTask(bk, input.Filters, input.Paths, time, input.PageSize ?? 1000, input.Page ?? 0), false) as ISearchFilesResults; if (r == null) throw new ServerErrorException("No result from list operation"); diff --git a/Duplicati/WebserverCore/Endpoints/V2/DestinationVerify.cs b/Duplicati/WebserverCore/Endpoints/V2/DestinationVerify.cs index 6e63db05b..c44725bf7 100644 --- a/Duplicati/WebserverCore/Endpoints/V2/DestinationVerify.cs +++ b/Duplicati/WebserverCore/Endpoints/V2/DestinationVerify.cs @@ -36,7 +36,7 @@ public class DestinationVerify : IEndpointV2 public static void Map(RouteGroupBuilder group) { group.MapPost("/destination/test", ([FromServices] Connection connection, [FromBody] Dto.V2.DestinationTestRequestDto input, CancellationToken cancelToken) - => ExecuteTest(input, cancelToken)) + => ExecuteTest(connection, input, cancelToken)) .RequireAuthorization(); } @@ -51,11 +51,11 @@ public class DestinationVerify : IEndpointV2 } } - private static Dictionary ParseUrlOptions(Library.Utility.Uri uri) + private static Dictionary ParseUrlOptions(Connection connection, Library.Utility.Uri uri) { var qp = uri.QueryParameters; - var opts = Runner.GetCommonOptions(); + var opts = Runner.GetCommonOptions(connection); foreach (var k in qp.Keys.Cast()) opts[k] = qp[k]; @@ -75,10 +75,10 @@ public class DestinationVerify : IEndpointV2 return modules; } - private static async Task GetBackend(string url, CancellationToken cancelToken) + private static async Task GetBackend(Connection connection, string url, CancellationToken cancelToken) { var uri = new Library.Utility.Uri(url); - var opts = ParseUrlOptions(uri); + var opts = ParseUrlOptions(connection, uri); var tmp = new[] { uri }; await SecretProviderHelper.ApplySecretProviderAsync([], tmp, opts, Library.Utility.TempFolder.SystemTempPath, FIXMEGlobal.SecretProvider, cancelToken); @@ -90,13 +90,13 @@ public class DestinationVerify : IEndpointV2 } - private static async Task ExecuteTest(DestinationTestRequestDto input, CancellationToken cancelToken) + private static async Task ExecuteTest(Connection connection, DestinationTestRequestDto input, CancellationToken cancelToken) { TupleDisposeWrapper? wrapper = null; try { - wrapper = await GetBackend(input.DestinationUrl, cancelToken); + wrapper = await GetBackend(connection, input.DestinationUrl, cancelToken); using (var b = wrapper.Backend) { diff --git a/Duplicati/WebserverCore/Services/QueueRunnerService.cs b/Duplicati/WebserverCore/Services/QueueRunnerService.cs index 9335bcc93..4313c0814 100644 --- a/Duplicati/WebserverCore/Services/QueueRunnerService.cs +++ b/Duplicati/WebserverCore/Services/QueueRunnerService.cs @@ -136,7 +136,7 @@ public class QueueRunnerService(Connection connection, EventPollNotify eventPoll if (task.OnStarting != null) await task.OnStarting().ConfigureAwait(false); - await task.Execute(); + Runner.Run(connection, eventPollNotify, task, false); // If the task is completed, don't call OnFinished again completed = true; diff --git a/Duplicati/WebserverCore/Services/SchedulerService.cs b/Duplicati/WebserverCore/Services/SchedulerService.cs index b327c83ba..381fe38d9 100644 --- a/Duplicati/WebserverCore/Services/SchedulerService.cs +++ b/Duplicati/WebserverCore/Services/SchedulerService.cs @@ -21,6 +21,7 @@ using Duplicati.Library.RestAPI; using Duplicati.Server; +using Duplicati.Server.Database; using Duplicati.Server.Serialization.Interface; using Duplicati.WebserverCore.Abstractions; @@ -29,9 +30,9 @@ namespace WebserverCore.Services; public class SchedulerService : ISchedulerService { private readonly Scheduler scheduler; - public SchedulerService(EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IQueueRunnerService queueRunnerService) + public SchedulerService(Connection connection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IQueueRunnerService queueRunnerService) { - this.scheduler = new Scheduler(queueRunnerService); + this.scheduler = new Scheduler(connection, queueRunnerService); var lastScheduleId = notificationUpdateService.LastDataUpdateId; eventPollNotify.NewEvent += (sender, e) => { From d788f44ef3e1e1fd838c733b4b69a3bb8f4fa668 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 3 May 2025 16:03:37 +0200 Subject: [PATCH 07/24] One more removed property --- Duplicati/Library/RestAPI/FIXMEGlobal.cs | 4 -- Duplicati/Library/RestAPI/Runner.cs | 38 +++++++++---------- .../Endpoints/V1/Backup/BackupGet.cs | 21 +++++----- .../Endpoints/V2/BackupListing.cs | 33 ++++++++-------- .../Services/QueueRunnerService.cs | 5 ++- 5 files changed, 50 insertions(+), 51 deletions(-) diff --git a/Duplicati/Library/RestAPI/FIXMEGlobal.cs b/Duplicati/Library/RestAPI/FIXMEGlobal.cs index 6b2491e70..f91a6100b 100644 --- a/Duplicati/Library/RestAPI/FIXMEGlobal.cs +++ b/Duplicati/Library/RestAPI/FIXMEGlobal.cs @@ -50,10 +50,6 @@ namespace Duplicati.Library.RestAPI /// public static Func GenerateProgressState; - /// - /// For keeping and incrementing last last events Ids of db save and last notification - /// - public static INotificationUpdateService NotificationUpdateService => Provider.GetRequiredService(); /// /// Checks if the server has started and is listening for events /// diff --git a/Duplicati/Library/RestAPI/Runner.cs b/Duplicati/Library/RestAPI/Runner.cs index 841454dcb..ed99ee376 100644 --- a/Duplicati/Library/RestAPI/Runner.cs +++ b/Duplicati/Library/RestAPI/Runner.cs @@ -496,15 +496,15 @@ namespace Duplicati.Server return parts.ToArray(); } - public static IBasicResults? Run(Connection databaseConnection, EventPollNotify eventPollNotify, IQueuedTask data, bool fromQueue) + public static IBasicResults? Run(Connection databaseConnection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IQueuedTask data, bool fromQueue) { if (data is IRunnerData runnerData) - return Run(databaseConnection, eventPollNotify, runnerData, fromQueue); + return Run(databaseConnection, eventPollNotify, notificationUpdateService, runnerData, fromQueue); throw new ArgumentException("Invalid task type", nameof(data)); } - private static IBasicResults? Run(Connection databaseConnection, EventPollNotify eventPollNotify, IRunnerData data, bool fromQueue) + private static IBasicResults? Run(Connection databaseConnection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IRunnerData data, bool fromQueue) { data.TaskStarted = DateTime.Now; if (data is CustomRunnerTask task) @@ -605,49 +605,49 @@ namespace Duplicati.Server select p).ToArray(); var r = controller.Backup(sources, filter); - UpdateMetadataBase(databaseConnection, eventPollNotify, backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r); return r; } case DuplicatiOperation.List: { var r = controller.List(data.FilterStrings, null); - UpdateMetadataBase(databaseConnection, eventPollNotify, backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r); return r; } case DuplicatiOperation.Repair: { var r = controller.Repair(data.FilterStrings == null ? null : new Library.Utility.FilterExpression(data.FilterStrings)); - UpdateMetadataBase(databaseConnection, eventPollNotify, backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r); return r; } case DuplicatiOperation.RepairUpdate: { var r = controller.UpdateDatabaseWithVersions(); - UpdateMetadataBase(databaseConnection, eventPollNotify, backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r); return r; } case DuplicatiOperation.Remove: { var r = controller.Delete(); - UpdateMetadataBase(databaseConnection, eventPollNotify, backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r); return r; } case DuplicatiOperation.Restore: { var r = controller.Restore(data.FilterStrings); - UpdateMetadataBase(databaseConnection, eventPollNotify, backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r); return r; } case DuplicatiOperation.Verify: { var r = controller.Test(); - UpdateMetadataBase(databaseConnection, eventPollNotify, backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r); return r; } case DuplicatiOperation.Compact: { var r = controller.Compact(); - UpdateMetadataBase(databaseConnection, eventPollNotify, backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r); return r; } case DuplicatiOperation.CreateReport: @@ -680,7 +680,7 @@ namespace Duplicati.Server case DuplicatiOperation.ListRemote: { var r = controller.ListRemote(); - UpdateMetadataBase(databaseConnection, eventPollNotify, backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r); return r; } @@ -706,14 +706,14 @@ namespace Duplicati.Server case DuplicatiOperation.Vacuum: { var r = controller.Vacuum(); - UpdateMetadataBase(databaseConnection, eventPollNotify, backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r); return r; } case DuplicatiOperation.ListFilesets: { var r = controller.ListFilesets(); - UpdateMetadataBase(databaseConnection, eventPollNotify, backup, r); + UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r); return r; } case DuplicatiOperation.ListFolderContents: @@ -740,7 +740,7 @@ namespace Duplicati.Server { databaseConnection.LogError(data.Backup?.ID, string.Format("Failed while executing {0} \"{1}\" (id: {2})", data.Operation, data.Backup?.Name, data.Backup?.ID), ex); if (data.Backup != null) - UpdateMetadataError(databaseConnection, data.Backup, ex); + UpdateMetadataError(databaseConnection, notificationUpdateService, data.Backup, ex); Library.UsageReporter.Reporter.Report(ex); if (!fromQueue) @@ -797,7 +797,7 @@ namespace Duplicati.Server return tempfolder; } - private static void UpdateMetadataError(Connection databaseConnection, Serialization.Interface.IBackup backup, Exception ex) + private static void UpdateMetadataError(Connection databaseConnection, INotificationUpdateService notificationUpdateService, IBackup backup, Exception ex) { backup.Metadata["LastErrorDate"] = Utility.SerializeDateTime(DateTime.UtcNow); backup.Metadata["LastErrorMessage"] = ex.Message; @@ -809,7 +809,7 @@ namespace Duplicati.Server if (ex is UserInformationException exception) messageid = exception.HelpID; - FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); + notificationUpdateService.IncrementLastDataUpdateId(); databaseConnection.RegisterNotification( NotificationType.Error, backup.IsTemporary ? @@ -865,7 +865,7 @@ namespace Duplicati.Server } } - private static void UpdateMetadataBase(Connection databaseConnection, EventPollNotify eventPollNotify, Serialization.Interface.IBackup backup, IBasicResults result) + private static void UpdateMetadataBase(Connection databaseConnection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IBackup backup, IBasicResults result) { if (result is IRestoreResults r1) { @@ -999,7 +999,7 @@ namespace Duplicati.Server if (!backup.IsTemporary) databaseConnection.SetMetadata(backup.Metadata, long.Parse(backup.ID), null); - FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); + notificationUpdateService.IncrementLastDataUpdateId(); eventPollNotify.SignalNewEvent(); } diff --git a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs index da2e33802..045fd9a45 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs @@ -21,6 +21,7 @@ using System.Text.Json; using System.Text.Json.Serialization; using Duplicati.Library.Interface; +using Duplicati.Library.RestAPI; using Duplicati.Server; using Duplicati.Server.Database; using Duplicati.Server.Serialization; @@ -41,8 +42,8 @@ public class BackupGet : IEndpointV1 => ExecuteGet(connection, GetBackup(connection, id))) .RequireAuthorization(); - group.MapGet("/backup/{id}/files", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromRoute] string id, [FromQuery] string? filter, [FromQuery] string? time, [FromQuery(Name = "all-versions")] bool? allVersions, [FromQuery(Name = "prefix-only")] bool? prefixOnly, [FromQuery(Name = "folder-contents")] bool? folderContents) - => ExecuteGetFiles(connection, eventPollNotify, GetBackup(connection, id), filter, time, allVersions ?? false, prefixOnly ?? false, folderContents ?? false, new Dictionary())) + group.MapGet("/backup/{id}/files", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromServices] INotificationUpdateService notificationUpdateService, [FromRoute] string id, [FromQuery] string? filter, [FromQuery] string? time, [FromQuery(Name = "all-versions")] bool? allVersions, [FromQuery(Name = "prefix-only")] bool? prefixOnly, [FromQuery(Name = "folder-contents")] bool? folderContents) + => ExecuteGetFiles(connection, eventPollNotify, notificationUpdateService, GetBackup(connection, id), filter, time, allVersions ?? false, prefixOnly ?? false, folderContents ?? false, new Dictionary())) .RequireAuthorization(); group.MapGet("/backup/{id}/log", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery] long? offset, [FromQuery] long? pagesize) @@ -53,8 +54,8 @@ public class BackupGet : IEndpointV1 => ExecuteGetRemotelog(connection, GetBackup(connection, id), offset, pagesize ?? 100)) .RequireAuthorization(); - group.MapGet("/backup/{id}/filesets", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromRoute] string id, [FromQuery(Name = "include-metadata")] bool? includeMetadata, [FromQuery(Name = "from-remote-only")] bool? fromRemoteOnly) - => ExecuteGetFilesets(connection, eventPollNotify, GetBackup(connection, id), includeMetadata ?? false, fromRemoteOnly ?? false)) + group.MapGet("/backup/{id}/filesets", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromServices] INotificationUpdateService notificationUpdateService, [FromRoute] string id, [FromQuery(Name = "include-metadata")] bool? includeMetadata, [FromQuery(Name = "from-remote-only")] bool? fromRemoteOnly) + => ExecuteGetFilesets(connection, eventPollNotify, notificationUpdateService, GetBackup(connection, id), includeMetadata ?? false, fromRemoteOnly ?? false)) .RequireAuthorization(); group.MapGet("/backup/{id}/export-argsonly", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery(Name = "export-passwords")] bool? exportPasswords, [FromQuery] string? passphrase) @@ -142,7 +143,7 @@ public class BackupGet : IEndpointV1 ); } - private static Dictionary SearchFiles(Connection connection, EventPollNotify eventPollNotify, IBackup backup, string? filter, string? timestring, bool allVersions, bool prefixOnly, bool folderContents, Dictionary extraValues) + private static Dictionary SearchFiles(Connection connection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IBackup backup, string? filter, string? timestring, bool allVersions, bool prefixOnly, bool folderContents, Dictionary extraValues) { if (string.IsNullOrWhiteSpace(timestring) && !allVersions) throw new BadRequestException("Invalid or missing time"); @@ -151,7 +152,7 @@ public class BackupGet : IEndpointV1 if (!allVersions) time = Library.Utility.Timeparser.ParseTimeInterval(timestring, DateTime.Now); - var r = Runner.Run(connection, eventPollNotify, Runner.CreateListTask(backup, filter == null ? null : [filter], prefixOnly, allVersions, folderContents, time), false) as IListResults; + var r = Runner.Run(connection, eventPollNotify, notificationUpdateService, Runner.CreateListTask(backup, filter == null ? null : [filter], prefixOnly, allVersions, folderContents, time), false) as IListResults; if (r == null) throw new ServerErrorException("No result from list operation"); @@ -170,8 +171,8 @@ public class BackupGet : IEndpointV1 return result; } - private static Dictionary ExecuteGetFiles(Connection connection, EventPollNotify eventPollNotify, IBackup bk, string? filter, string? timestring, bool allVersions, bool prefixOnly, bool folderContents, Dictionary extraValues) - => SearchFiles(connection, eventPollNotify, bk, filter, timestring, allVersions, prefixOnly, folderContents, extraValues); + private static Dictionary ExecuteGetFiles(Connection connection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IBackup bk, string? filter, string? timestring, bool allVersions, bool prefixOnly, bool folderContents, Dictionary extraValues) + => SearchFiles(connection, eventPollNotify, notificationUpdateService, bk, filter, timestring, allVersions, prefixOnly, folderContents, extraValues); private static List> ExecuteGetLog(Connection connection, IBackup bk, long? offset, long pagesize) { @@ -202,7 +203,7 @@ public class BackupGet : IEndpointV1 } } - private static IEnumerable ExecuteGetFilesets(Connection connection, EventPollNotify eventPollNotify, IBackup bk, bool includeMetadata, bool fromRemoteOnly) + private static IEnumerable ExecuteGetFilesets(Connection connection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IBackup bk, bool includeMetadata, bool fromRemoteOnly) { var extra = new Dictionary { @@ -218,7 +219,7 @@ public class BackupGet : IEndpointV1 try { - var r = Runner.Run(connection, eventPollNotify, Runner.CreateTask(DuplicatiOperation.List, bk, extra), false) as IListResults; + var r = Runner.Run(connection, eventPollNotify, notificationUpdateService, Runner.CreateTask(DuplicatiOperation.List, bk, extra), false) as IListResults; if (r == null) throw new ServerErrorException("No result from list operation"); diff --git a/Duplicati/WebserverCore/Endpoints/V2/BackupListing.cs b/Duplicati/WebserverCore/Endpoints/V2/BackupListing.cs index 5e301489e..d94fa043e 100644 --- a/Duplicati/WebserverCore/Endpoints/V2/BackupListing.cs +++ b/Duplicati/WebserverCore/Endpoints/V2/BackupListing.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Interface; +using Duplicati.Library.RestAPI; using Duplicati.Server; using Duplicati.Server.Database; using Duplicati.Server.Serialization.Interface; @@ -33,33 +34,33 @@ public class BackupListing : IEndpointV2 { public static void Map(RouteGroupBuilder group) { - group.MapPost("/backup/list-filesets", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromBody] Dto.V2.ListFilesetsRequestDto input) - => ExecuteGetFilesets(connection, eventPollNotify, GetBackup(connection, input.BackupId))) + group.MapPost("/backup/list-filesets", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromServices] INotificationUpdateService notificationUpdateService, [FromBody] Dto.V2.ListFilesetsRequestDto input) + => ExecuteGetFilesets(connection, eventPollNotify, notificationUpdateService, GetBackup(connection, input.BackupId))) .RequireAuthorization(); - group.MapPost("/backup/list-folder", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromBody] Dto.V2.ListFolderContentRequestDto input) - => ExecuteListFolder(connection, eventPollNotify, GetBackup(connection, input.BackupId), input)) + group.MapPost("/backup/list-folder", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromServices] INotificationUpdateService notificationUpdateService, [FromBody] Dto.V2.ListFolderContentRequestDto input) + => ExecuteListFolder(connection, eventPollNotify, notificationUpdateService, GetBackup(connection, input.BackupId), input)) .RequireAuthorization(); - group.MapPost("/backup/list-versions", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromBody] Dto.V2.ListFileVersionsRequestDto input) - => ExecuteListVersions(connection, eventPollNotify, GetBackup(connection, input.BackupId), input)) + group.MapPost("/backup/list-versions", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromServices] INotificationUpdateService notificationUpdateService, [FromBody] Dto.V2.ListFileVersionsRequestDto input) + => ExecuteListVersions(connection, eventPollNotify, notificationUpdateService, GetBackup(connection, input.BackupId), input)) .RequireAuthorization(); - group.MapPost("/backup/search", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromBody] Dto.V2.SearchEntriesRequestDto input) - => ExecuteSearch(connection, eventPollNotify, GetBackup(connection, input.BackupId), input)) + group.MapPost("/backup/search", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromServices] INotificationUpdateService notificationUpdateService, [FromBody] Dto.V2.SearchEntriesRequestDto input) + => ExecuteSearch(connection, eventPollNotify, notificationUpdateService, GetBackup(connection, input.BackupId), input)) .RequireAuthorization(); } private static IBackup GetBackup(Connection connection, string id) => connection.GetBackup(id) ?? throw new NotFoundException("Backup not found"); - private static Dto.V2.ListFolderContentResponseDto ExecuteListFolder(Connection connection, EventPollNotify eventPollNotify, IBackup bk, Dto.V2.ListFolderContentRequestDto input) + private static Dto.V2.ListFolderContentResponseDto ExecuteListFolder(Connection connection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IBackup bk, Dto.V2.ListFolderContentRequestDto input) { var time = string.IsNullOrWhiteSpace(input.Time) ? new DateTime(0) : Library.Utility.Timeparser.ParseTimeInterval(input.Time, DateTime.Now); - var r = Runner.Run(connection, eventPollNotify, Runner.CreateListFolderContents(bk, input.Paths, time, input.PageSize ?? 1000, input.Page ?? 0), false) as IListFolderResults; + var r = Runner.Run(connection, eventPollNotify, notificationUpdateService, Runner.CreateListFolderContents(bk, input.Paths, time, input.PageSize ?? 1000, input.Page ?? 0), false) as IListFolderResults; if (r == null) throw new ServerErrorException("No result from list operation"); @@ -78,7 +79,7 @@ public class BackupListing : IEndpointV2 r.Entries.TotalCount); } - private static Dto.V2.ListFilesetsResponseDto ExecuteGetFilesets(Connection connection, EventPollNotify eventPollNotify, IBackup bk) + private static Dto.V2.ListFilesetsResponseDto ExecuteGetFilesets(Connection connection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IBackup bk) { var extra = new Dictionary(); @@ -87,7 +88,7 @@ public class BackupListing : IEndpointV2 try { - var r = Runner.Run(connection, eventPollNotify, Runner.CreateListFilesetsTask(bk, extra), false) as IListFilesetResults; + var r = Runner.Run(connection, eventPollNotify, notificationUpdateService, Runner.CreateListFilesetsTask(bk, extra), false) as IListFilesetResults; if (r == null) throw new ServerErrorException("No result from list operation"); @@ -112,9 +113,9 @@ public class BackupListing : IEndpointV2 } } - private static Dto.V2.ListFileVersionsOutputDto ExecuteListVersions(Connection connection, EventPollNotify eventPollNotify, IBackup bk, Dto.V2.ListFileVersionsRequestDto input) + private static Dto.V2.ListFileVersionsOutputDto ExecuteListVersions(Connection connection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IBackup bk, Dto.V2.ListFileVersionsRequestDto input) { - var r = Runner.Run(connection, eventPollNotify, Runner.ListFileVersionsTask(bk, input.Paths, input.PageSize ?? 1000, input.Page ?? 0), false) as IListFileVersionsResults; + var r = Runner.Run(connection, eventPollNotify, notificationUpdateService, Runner.ListFileVersionsTask(bk, input.Paths, input.PageSize ?? 1000, input.Page ?? 0), false) as IListFileVersionsResults; if (r == null) throw new ServerErrorException("No result from list operation"); @@ -135,13 +136,13 @@ public class BackupListing : IEndpointV2 r.FileVersions.TotalCount); } - private static Dto.V2.SearchEntriesResponseDto ExecuteSearch(Connection connection, EventPollNotify eventPollNotify, IBackup bk, Dto.V2.SearchEntriesRequestDto input) + private static Dto.V2.SearchEntriesResponseDto ExecuteSearch(Connection connection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IBackup bk, Dto.V2.SearchEntriesRequestDto input) { var time = string.IsNullOrWhiteSpace(input.Time) ? new DateTime(0) : Library.Utility.Timeparser.ParseTimeInterval(input.Time, DateTime.Now); - var r = Runner.Run(connection, eventPollNotify, Runner.CreateSearchEntriesTask(bk, input.Filters, input.Paths, time, input.PageSize ?? 1000, input.Page ?? 0), false) as ISearchFilesResults; + var r = Runner.Run(connection, eventPollNotify, notificationUpdateService, Runner.CreateSearchEntriesTask(bk, input.Filters, input.Paths, time, input.PageSize ?? 1000, input.Page ?? 0), false) as ISearchFilesResults; if (r == null) throw new ServerErrorException("No result from list operation"); diff --git a/Duplicati/WebserverCore/Services/QueueRunnerService.cs b/Duplicati/WebserverCore/Services/QueueRunnerService.cs index 4313c0814..0b63903ee 100644 --- a/Duplicati/WebserverCore/Services/QueueRunnerService.cs +++ b/Duplicati/WebserverCore/Services/QueueRunnerService.cs @@ -19,6 +19,7 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +using Duplicati.Library.RestAPI; using Duplicati.Library.Utility; using Duplicati.Server; using Duplicati.Server.Database; @@ -30,7 +31,7 @@ namespace Duplicati.WebserverCore.Services; /// /// Simple queue that will run the given task /// -public class QueueRunnerService(Connection connection, EventPollNotify eventPollNotify) : IQueueRunnerService +public class QueueRunnerService(Connection connection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService) : IQueueRunnerService { private readonly object _lock = new(); /// @@ -136,7 +137,7 @@ public class QueueRunnerService(Connection connection, EventPollNotify eventPoll if (task.OnStarting != null) await task.OnStarting().ConfigureAwait(false); - Runner.Run(connection, eventPollNotify, task, false); + Runner.Run(connection, eventPollNotify, notificationUpdateService, task, false); // If the task is completed, don't call OnFinished again completed = true; From eaf6fe5f263892498288553e2b912dcf510b3d31 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 3 May 2025 16:25:24 +0200 Subject: [PATCH 08/24] One more FIXMEGlobal removal --- .../IProgressStateProviderService.cs | 37 +++++++++++++++++++ .../Abstractions/IQueueRunnerService.cs | 8 ++++ Duplicati/Library/RestAPI/FIXMEGlobal.cs | 7 ---- Duplicati/Library/RestAPI/Runner.cs | 11 +++--- Duplicati/Server/Program.cs | 5 --- .../Endpoints/V1/Backup/BackupGet.cs | 20 +++++----- .../Endpoints/V1/ProgressState.cs | 11 +++--- .../Endpoints/V2/BackupListing.cs | 33 +++++++++-------- .../ServiceCollectionsExtensions.cs | 3 +- .../Services/LoginProvider copy.cs | 35 ++++++++++++++++++ .../Services/QueueRunnerService.cs | 15 +++++++- 11 files changed, 134 insertions(+), 51 deletions(-) create mode 100644 Duplicati/Library/RestAPI/Abstractions/IProgressStateProviderService.cs create mode 100644 Duplicati/WebserverCore/Services/LoginProvider copy.cs diff --git a/Duplicati/Library/RestAPI/Abstractions/IProgressStateProviderService.cs b/Duplicati/Library/RestAPI/Abstractions/IProgressStateProviderService.cs new file mode 100644 index 000000000..335bb5247 --- /dev/null +++ b/Duplicati/Library/RestAPI/Abstractions/IProgressStateProviderService.cs @@ -0,0 +1,37 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; + +namespace Duplicati.WebserverCore.Abstractions; + +/// +/// Produces system information. +/// +public interface IProgressStateProviderService +{ + /// + /// Handler for progress state generation. + /// + Func? GenerateProgressState { get; set; } +} diff --git a/Duplicati/Library/RestAPI/Abstractions/IQueueRunnerService.cs b/Duplicati/Library/RestAPI/Abstractions/IQueueRunnerService.cs index 554ae4bae..1ee4933de 100644 --- a/Duplicati/Library/RestAPI/Abstractions/IQueueRunnerService.cs +++ b/Duplicati/Library/RestAPI/Abstractions/IQueueRunnerService.cs @@ -23,6 +23,7 @@ using System; using System.Collections.Generic; +using Duplicati.Library.Interface; using Duplicati.Server.Serialization.Interface; namespace Duplicati.WebserverCore.Abstractions; @@ -96,4 +97,11 @@ public interface IQueueRunnerService /// /// A list of tuples containing the task ID and backup ID IList> GetQueueWithIds(); + + /// + /// Runs a task immediately, bypassing the queue. + /// Note that the task will run concurrently with the queue tasks and may cause database lock issues. + /// + /// The task to run + IBasicResults? RunImmediately(IQueuedTask task); } \ No newline at end of file diff --git a/Duplicati/Library/RestAPI/FIXMEGlobal.cs b/Duplicati/Library/RestAPI/FIXMEGlobal.cs index f91a6100b..781dc5311 100644 --- a/Duplicati/Library/RestAPI/FIXMEGlobal.cs +++ b/Duplicati/Library/RestAPI/FIXMEGlobal.cs @@ -21,8 +21,6 @@ using Duplicati.Server; using System; -using Duplicati.WebserverCore.Abstractions; -using Microsoft.Extensions.DependencyInjection; using Duplicati.Library.Interface; namespace Duplicati.Library.RestAPI @@ -45,11 +43,6 @@ namespace Duplicati.Library.RestAPI /// public static Server.Database.Connection DataConnection; - /// - /// A delegate method for creating a copy of the current progress state - /// - public static Func GenerateProgressState; - /// /// Checks if the server has started and is listening for events /// diff --git a/Duplicati/Library/RestAPI/Runner.cs b/Duplicati/Library/RestAPI/Runner.cs index ed99ee376..9374c6826 100644 --- a/Duplicati/Library/RestAPI/Runner.cs +++ b/Duplicati/Library/RestAPI/Runner.cs @@ -31,6 +31,7 @@ using Duplicati.Library.Utility; using System.Threading.Tasks; using Duplicati.Server.Database; using Duplicati.Server.Serialization.Interface; +using Duplicati.WebserverCore.Abstractions; namespace Duplicati.Server { @@ -496,15 +497,15 @@ namespace Duplicati.Server return parts.ToArray(); } - public static IBasicResults? Run(Connection databaseConnection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IQueuedTask data, bool fromQueue) + public static IBasicResults? Run(Connection databaseConnection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IProgressStateProviderService progressStateProviderService, IQueuedTask data, bool fromQueue) { if (data is IRunnerData runnerData) - return Run(databaseConnection, eventPollNotify, notificationUpdateService, runnerData, fromQueue); + return RunInternal(databaseConnection, eventPollNotify, notificationUpdateService, progressStateProviderService, runnerData, fromQueue); throw new ArgumentException("Invalid task type", nameof(data)); } - private static IBasicResults? Run(Connection databaseConnection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IRunnerData data, bool fromQueue) + private static IBasicResults? RunInternal(Connection databaseConnection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IProgressStateProviderService progressStateProviderService, IRunnerData data, bool fromQueue) { data.TaskStarted = DateTime.Now; if (data is CustomRunnerTask task) @@ -512,7 +513,7 @@ namespace Duplicati.Server try { var sink = new MessageSink(task.TaskID, null); - FIXMEGlobal.GenerateProgressState = sink.Copy; + progressStateProviderService.GenerateProgressState = sink.Copy; eventPollNotify.SignalNewEvent(); task.Run(sink); @@ -541,7 +542,7 @@ namespace Duplicati.Server var sink = new MessageSink(data.TaskID, backup.ID); if (fromQueue) { - FIXMEGlobal.GenerateProgressState = () => sink.Copy(); + progressStateProviderService.GenerateProgressState = () => sink.Copy(); eventPollNotify.SignalNewEvent(); } diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index 794ceb955..fa651829f 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -137,11 +137,6 @@ namespace Duplicati.Server /// public static readonly ManualResetEvent ServerStartedEvent = new ManualResetEvent(false); - /// - /// A delegate method for creating a copy of the current progress state - /// - public static Func GenerateProgressState { get => FIXMEGlobal.GenerateProgressState; set => FIXMEGlobal.GenerateProgressState = value; } - /// /// The log redirect handler /// diff --git a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs index 045fd9a45..413226cfe 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs @@ -42,8 +42,8 @@ public class BackupGet : IEndpointV1 => ExecuteGet(connection, GetBackup(connection, id))) .RequireAuthorization(); - group.MapGet("/backup/{id}/files", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromServices] INotificationUpdateService notificationUpdateService, [FromRoute] string id, [FromQuery] string? filter, [FromQuery] string? time, [FromQuery(Name = "all-versions")] bool? allVersions, [FromQuery(Name = "prefix-only")] bool? prefixOnly, [FromQuery(Name = "folder-contents")] bool? folderContents) - => ExecuteGetFiles(connection, eventPollNotify, notificationUpdateService, GetBackup(connection, id), filter, time, allVersions ?? false, prefixOnly ?? false, folderContents ?? false, new Dictionary())) + group.MapGet("/backup/{id}/files", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id, [FromQuery] string? filter, [FromQuery] string? time, [FromQuery(Name = "all-versions")] bool? allVersions, [FromQuery(Name = "prefix-only")] bool? prefixOnly, [FromQuery(Name = "folder-contents")] bool? folderContents) + => ExecuteGetFiles(queueRunnerService, GetBackup(connection, id), filter, time, allVersions ?? false, prefixOnly ?? false, folderContents ?? false, new Dictionary())) .RequireAuthorization(); group.MapGet("/backup/{id}/log", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery] long? offset, [FromQuery] long? pagesize) @@ -54,8 +54,8 @@ public class BackupGet : IEndpointV1 => ExecuteGetRemotelog(connection, GetBackup(connection, id), offset, pagesize ?? 100)) .RequireAuthorization(); - group.MapGet("/backup/{id}/filesets", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromServices] INotificationUpdateService notificationUpdateService, [FromRoute] string id, [FromQuery(Name = "include-metadata")] bool? includeMetadata, [FromQuery(Name = "from-remote-only")] bool? fromRemoteOnly) - => ExecuteGetFilesets(connection, eventPollNotify, notificationUpdateService, GetBackup(connection, id), includeMetadata ?? false, fromRemoteOnly ?? false)) + group.MapGet("/backup/{id}/filesets", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id, [FromQuery(Name = "include-metadata")] bool? includeMetadata, [FromQuery(Name = "from-remote-only")] bool? fromRemoteOnly) + => ExecuteGetFilesets(queueRunnerService, GetBackup(connection, id), includeMetadata ?? false, fromRemoteOnly ?? false)) .RequireAuthorization(); group.MapGet("/backup/{id}/export-argsonly", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery(Name = "export-passwords")] bool? exportPasswords, [FromQuery] string? passphrase) @@ -143,7 +143,7 @@ public class BackupGet : IEndpointV1 ); } - private static Dictionary SearchFiles(Connection connection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IBackup backup, string? filter, string? timestring, bool allVersions, bool prefixOnly, bool folderContents, Dictionary extraValues) + private static Dictionary SearchFiles(IQueueRunnerService queueRunnerService, IBackup backup, string? filter, string? timestring, bool allVersions, bool prefixOnly, bool folderContents, Dictionary extraValues) { if (string.IsNullOrWhiteSpace(timestring) && !allVersions) throw new BadRequestException("Invalid or missing time"); @@ -152,7 +152,7 @@ public class BackupGet : IEndpointV1 if (!allVersions) time = Library.Utility.Timeparser.ParseTimeInterval(timestring, DateTime.Now); - var r = Runner.Run(connection, eventPollNotify, notificationUpdateService, Runner.CreateListTask(backup, filter == null ? null : [filter], prefixOnly, allVersions, folderContents, time), false) as IListResults; + var r = queueRunnerService.RunImmediately(Runner.CreateListTask(backup, filter == null ? null : [filter], prefixOnly, allVersions, folderContents, time)) as IListResults; if (r == null) throw new ServerErrorException("No result from list operation"); @@ -171,8 +171,8 @@ public class BackupGet : IEndpointV1 return result; } - private static Dictionary ExecuteGetFiles(Connection connection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IBackup bk, string? filter, string? timestring, bool allVersions, bool prefixOnly, bool folderContents, Dictionary extraValues) - => SearchFiles(connection, eventPollNotify, notificationUpdateService, bk, filter, timestring, allVersions, prefixOnly, folderContents, extraValues); + private static Dictionary ExecuteGetFiles(IQueueRunnerService queueRunnerService, IBackup bk, string? filter, string? timestring, bool allVersions, bool prefixOnly, bool folderContents, Dictionary extraValues) + => SearchFiles(queueRunnerService, bk, filter, timestring, allVersions, prefixOnly, folderContents, extraValues); private static List> ExecuteGetLog(Connection connection, IBackup bk, long? offset, long pagesize) { @@ -203,7 +203,7 @@ public class BackupGet : IEndpointV1 } } - private static IEnumerable ExecuteGetFilesets(Connection connection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IBackup bk, bool includeMetadata, bool fromRemoteOnly) + private static IEnumerable ExecuteGetFilesets(IQueueRunnerService queueRunnerService, IBackup bk, bool includeMetadata, bool fromRemoteOnly) { var extra = new Dictionary { @@ -219,7 +219,7 @@ public class BackupGet : IEndpointV1 try { - var r = Runner.Run(connection, eventPollNotify, notificationUpdateService, Runner.CreateTask(DuplicatiOperation.List, bk, extra), false) as IListResults; + var r = queueRunnerService.RunImmediately(Runner.CreateTask(DuplicatiOperation.List, bk, extra)) as IListResults; if (r == null) throw new ServerErrorException("No result from list operation"); diff --git a/Duplicati/WebserverCore/Endpoints/V1/ProgressState.cs b/Duplicati/WebserverCore/Endpoints/V1/ProgressState.cs index b24980e9e..b178dd718 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/ProgressState.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/ProgressState.cs @@ -18,9 +18,9 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using Duplicati.Library.RestAPI; using Duplicati.WebserverCore.Abstractions; using Duplicati.WebserverCore.Exceptions; +using Microsoft.AspNetCore.Mvc; namespace Duplicati.WebserverCore.Endpoints.V1; @@ -28,15 +28,16 @@ public class ProgressState : IEndpointV1 { public static void Map(RouteGroupBuilder group) { - group.MapGet("/progressstate", Execute) + group.MapGet("/progressstate", ([FromServices] IProgressStateProviderService progressStateProviderService) => Execute(progressStateProviderService)) .RequireAuthorization(); } - private static Server.Serialization.Interface.IProgressEventData Execute() + private static Server.Serialization.Interface.IProgressEventData Execute(IProgressStateProviderService progressStateProviderService) { - if (FIXMEGlobal.GenerateProgressState == null) + var pgState = progressStateProviderService.GenerateProgressState; + if (pgState == null) throw new NotFoundException("No active backup"); - return FIXMEGlobal.GenerateProgressState(); + return pgState(); } } diff --git a/Duplicati/WebserverCore/Endpoints/V2/BackupListing.cs b/Duplicati/WebserverCore/Endpoints/V2/BackupListing.cs index d94fa043e..8466abffe 100644 --- a/Duplicati/WebserverCore/Endpoints/V2/BackupListing.cs +++ b/Duplicati/WebserverCore/Endpoints/V2/BackupListing.cs @@ -26,6 +26,7 @@ using Duplicati.Server.Database; using Duplicati.Server.Serialization.Interface; using Duplicati.WebserverCore.Abstractions; using Duplicati.WebserverCore.Exceptions; +using Duplicati.WebserverCore.Services; using Microsoft.AspNetCore.Mvc; namespace Duplicati.WebserverCore.Endpoints.V2.Backup; @@ -34,33 +35,33 @@ public class BackupListing : IEndpointV2 { public static void Map(RouteGroupBuilder group) { - group.MapPost("/backup/list-filesets", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromServices] INotificationUpdateService notificationUpdateService, [FromBody] Dto.V2.ListFilesetsRequestDto input) - => ExecuteGetFilesets(connection, eventPollNotify, notificationUpdateService, GetBackup(connection, input.BackupId))) + group.MapPost("/backup/list-filesets", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromBody] Dto.V2.ListFilesetsRequestDto input) + => ExecuteGetFilesets(queueRunnerService, GetBackup(connection, input.BackupId))) .RequireAuthorization(); - group.MapPost("/backup/list-folder", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromServices] INotificationUpdateService notificationUpdateService, [FromBody] Dto.V2.ListFolderContentRequestDto input) - => ExecuteListFolder(connection, eventPollNotify, notificationUpdateService, GetBackup(connection, input.BackupId), input)) + group.MapPost("/backup/list-folder", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromBody] Dto.V2.ListFolderContentRequestDto input) + => ExecuteListFolder(queueRunnerService, GetBackup(connection, input.BackupId), input)) .RequireAuthorization(); - group.MapPost("/backup/list-versions", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromServices] INotificationUpdateService notificationUpdateService, [FromBody] Dto.V2.ListFileVersionsRequestDto input) - => ExecuteListVersions(connection, eventPollNotify, notificationUpdateService, GetBackup(connection, input.BackupId), input)) + group.MapPost("/backup/list-versions", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromBody] Dto.V2.ListFileVersionsRequestDto input) + => ExecuteListVersions(queueRunnerService, GetBackup(connection, input.BackupId), input)) .RequireAuthorization(); - group.MapPost("/backup/search", ([FromServices] Connection connection, [FromServices] EventPollNotify eventPollNotify, [FromServices] INotificationUpdateService notificationUpdateService, [FromBody] Dto.V2.SearchEntriesRequestDto input) - => ExecuteSearch(connection, eventPollNotify, notificationUpdateService, GetBackup(connection, input.BackupId), input)) + group.MapPost("/backup/search", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromBody] Dto.V2.SearchEntriesRequestDto input) + => ExecuteSearch(queueRunnerService, GetBackup(connection, input.BackupId), input)) .RequireAuthorization(); } private static IBackup GetBackup(Connection connection, string id) => connection.GetBackup(id) ?? throw new NotFoundException("Backup not found"); - private static Dto.V2.ListFolderContentResponseDto ExecuteListFolder(Connection connection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IBackup bk, Dto.V2.ListFolderContentRequestDto input) + private static Dto.V2.ListFolderContentResponseDto ExecuteListFolder(IQueueRunnerService queueRunnerService, IBackup bk, Dto.V2.ListFolderContentRequestDto input) { var time = string.IsNullOrWhiteSpace(input.Time) ? new DateTime(0) : Library.Utility.Timeparser.ParseTimeInterval(input.Time, DateTime.Now); - var r = Runner.Run(connection, eventPollNotify, notificationUpdateService, Runner.CreateListFolderContents(bk, input.Paths, time, input.PageSize ?? 1000, input.Page ?? 0), false) as IListFolderResults; + var r = queueRunnerService.RunImmediately(Runner.CreateListFolderContents(bk, input.Paths, time, input.PageSize ?? 1000, input.Page ?? 0)) as IListFolderResults; if (r == null) throw new ServerErrorException("No result from list operation"); @@ -79,7 +80,7 @@ public class BackupListing : IEndpointV2 r.Entries.TotalCount); } - private static Dto.V2.ListFilesetsResponseDto ExecuteGetFilesets(Connection connection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IBackup bk) + private static Dto.V2.ListFilesetsResponseDto ExecuteGetFilesets(IQueueRunnerService queueRunnerService, IBackup bk) { var extra = new Dictionary(); @@ -88,7 +89,7 @@ public class BackupListing : IEndpointV2 try { - var r = Runner.Run(connection, eventPollNotify, notificationUpdateService, Runner.CreateListFilesetsTask(bk, extra), false) as IListFilesetResults; + var r = queueRunnerService.RunImmediately(Runner.CreateListFilesetsTask(bk, extra)) as IListFilesetResults; if (r == null) throw new ServerErrorException("No result from list operation"); @@ -113,9 +114,9 @@ public class BackupListing : IEndpointV2 } } - private static Dto.V2.ListFileVersionsOutputDto ExecuteListVersions(Connection connection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IBackup bk, Dto.V2.ListFileVersionsRequestDto input) + private static Dto.V2.ListFileVersionsOutputDto ExecuteListVersions(IQueueRunnerService queueRunnerService, IBackup bk, Dto.V2.ListFileVersionsRequestDto input) { - var r = Runner.Run(connection, eventPollNotify, notificationUpdateService, Runner.ListFileVersionsTask(bk, input.Paths, input.PageSize ?? 1000, input.Page ?? 0), false) as IListFileVersionsResults; + var r = queueRunnerService.RunImmediately(Runner.ListFileVersionsTask(bk, input.Paths, input.PageSize ?? 1000, input.Page ?? 0)) as IListFileVersionsResults; if (r == null) throw new ServerErrorException("No result from list operation"); @@ -136,13 +137,13 @@ public class BackupListing : IEndpointV2 r.FileVersions.TotalCount); } - private static Dto.V2.SearchEntriesResponseDto ExecuteSearch(Connection connection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IBackup bk, Dto.V2.SearchEntriesRequestDto input) + private static Dto.V2.SearchEntriesResponseDto ExecuteSearch(IQueueRunnerService queueRunnerService, IBackup bk, Dto.V2.SearchEntriesRequestDto input) { var time = string.IsNullOrWhiteSpace(input.Time) ? new DateTime(0) : Library.Utility.Timeparser.ParseTimeInterval(input.Time, DateTime.Now); - var r = Runner.Run(connection, eventPollNotify, notificationUpdateService, Runner.CreateSearchEntriesTask(bk, input.Filters, input.Paths, time, input.PageSize ?? 1000, input.Page ?? 0), false) as ISearchFilesResults; + var r = queueRunnerService.RunImmediately(Runner.CreateSearchEntriesTask(bk, input.Filters, input.Paths, time, input.PageSize ?? 1000, input.Page ?? 0)) as ISearchFilesResults; if (r == null) throw new ServerErrorException("No result from list operation"); diff --git a/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs b/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs index ae2a52593..f891663f4 100644 --- a/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs +++ b/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs @@ -66,7 +66,8 @@ public static class ServiceCollectionsExtensions .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton(); + .AddSingleton() + .AddSingleton(); return services; } diff --git a/Duplicati/WebserverCore/Services/LoginProvider copy.cs b/Duplicati/WebserverCore/Services/LoginProvider copy.cs new file mode 100644 index 000000000..a3b9731e4 --- /dev/null +++ b/Duplicati/WebserverCore/Services/LoginProvider copy.cs @@ -0,0 +1,35 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using Duplicati.Server.Serialization.Interface; +using Duplicati.WebserverCore.Abstractions; + +namespace Duplicati.WebserverCore.Services; + +/// +/// Provides a way to generate progress state. +/// +public class ProgressStateProviderService : IProgressStateProviderService +{ + /// + /// The current progress state generator function. + /// + public Func? GenerateProgressState { get; set; } +} diff --git a/Duplicati/WebserverCore/Services/QueueRunnerService.cs b/Duplicati/WebserverCore/Services/QueueRunnerService.cs index 0b63903ee..87cd8daba 100644 --- a/Duplicati/WebserverCore/Services/QueueRunnerService.cs +++ b/Duplicati/WebserverCore/Services/QueueRunnerService.cs @@ -19,6 +19,7 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +using Duplicati.Library.Interface; using Duplicati.Library.RestAPI; using Duplicati.Library.Utility; using Duplicati.Server; @@ -31,7 +32,11 @@ namespace Duplicati.WebserverCore.Services; /// /// Simple queue that will run the given task /// -public class QueueRunnerService(Connection connection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService) : IQueueRunnerService +public class QueueRunnerService( + Connection connection, + EventPollNotify eventPollNotify, + INotificationUpdateService notificationUpdateService, + IProgressStateProviderService progressStateProviderService) : IQueueRunnerService { private readonly object _lock = new(); /// @@ -137,7 +142,7 @@ public class QueueRunnerService(Connection connection, EventPollNotify eventPoll if (task.OnStarting != null) await task.OnStarting().ConfigureAwait(false); - Runner.Run(connection, eventPollNotify, notificationUpdateService, task, false); + Runner.Run(connection, eventPollNotify, notificationUpdateService, progressStateProviderService, task, true); // If the task is completed, don't call OnFinished again completed = true; @@ -206,4 +211,10 @@ public class QueueRunnerService(Connection connection, EventPollNotify eventPoll } } } + + /// + public IBasicResults? RunImmediately(IQueuedTask task) + { + return Runner.Run(connection, eventPollNotify, notificationUpdateService, progressStateProviderService, task, false); + } } \ No newline at end of file From 71e926beeaca4a41b51bae0e81e542383dac391f Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 4 May 2025 12:13:15 +0200 Subject: [PATCH 09/24] One more FIXMEGlobal property removed --- .../Duplicati.GUI.TrayIcon/HttpServerConnection.cs | 12 +++++++----- Duplicati/Library/RestAPI/FIXMEGlobal.cs | 5 ----- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/HttpServerConnection.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/HttpServerConnection.cs index 0f3ada47a..d0b9f497c 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/HttpServerConnection.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/HttpServerConnection.cs @@ -334,9 +334,10 @@ namespace Duplicati.GUI.TrayIcon private async Task ObtainAccessTokenAsync() { // If we host the server, issue the access token from the service - if (FIXMEGlobal.IsServerStarted && m_passwordSource == Program.PasswordSource.HostedServer) + var sp = Server.Program.DuplicatiWebserver?.Provider; + if (sp != null && m_passwordSource == Program.PasswordSource.HostedServer) { - var provider = FIXMEGlobal.Provider.GetRequiredService(); + var provider = sp.GetRequiredService(); m_accesstoken = provider.CreateAccessToken("trayicon", provider.TemporaryFamilyId); return; } @@ -412,12 +413,13 @@ namespace Duplicati.GUI.TrayIcon string signinjwt = null; // If we host the server, issue the token from the service - if (FIXMEGlobal.IsServerStarted && m_passwordSource == Program.PasswordSource.HostedServer) + var sp = Server.Program.DuplicatiWebserver?.Provider; + if (sp != null && m_passwordSource == Program.PasswordSource.HostedServer) { - if (FIXMEGlobal.DataConnection.ApplicationSettings.DisableSigninTokens) + if (sp.GetRequiredService().ApplicationSettings.DisableSigninTokens) return null; - signinjwt = FIXMEGlobal.Provider.GetRequiredService().CreateSigninToken("trayicon"); + signinjwt = sp.GetRequiredService().CreateSigninToken("trayicon"); } // If we have database access, grab the issuer key from the db and issue a token diff --git a/Duplicati/Library/RestAPI/FIXMEGlobal.cs b/Duplicati/Library/RestAPI/FIXMEGlobal.cs index 781dc5311..a9b9829ee 100644 --- a/Duplicati/Library/RestAPI/FIXMEGlobal.cs +++ b/Duplicati/Library/RestAPI/FIXMEGlobal.cs @@ -43,11 +43,6 @@ namespace Duplicati.Library.RestAPI /// public static Server.Database.Connection DataConnection; - /// - /// Checks if the server has started and is listening for events - /// - public static bool IsServerStarted => Provider != null; - public static Action StartOrStopUsageReporter; /// From 1947b93a735d5a3635be15a62fd8abea642bbb1a Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 4 May 2025 12:17:44 +0200 Subject: [PATCH 10/24] One more FIXMEGlobal fix --- Duplicati/Library/RestAPI/FIXMEGlobal.cs | 5 --- .../RestAPI/Serializable/ServerSettings.cs | 35 +++++-------------- Duplicati/Server/Program.cs | 2 +- .../Services/RemoteControllerHandler.cs | 4 +-- 4 files changed, 12 insertions(+), 34 deletions(-) diff --git a/Duplicati/Library/RestAPI/FIXMEGlobal.cs b/Duplicati/Library/RestAPI/FIXMEGlobal.cs index a9b9829ee..d4e6009d2 100644 --- a/Duplicati/Library/RestAPI/FIXMEGlobal.cs +++ b/Duplicati/Library/RestAPI/FIXMEGlobal.cs @@ -38,11 +38,6 @@ namespace Duplicati.Library.RestAPI public static IServiceProvider Provider { get; set; } - /// - /// This is the only access to the database - /// - public static Server.Database.Connection DataConnection; - public static Action StartOrStopUsageReporter; /// diff --git a/Duplicati/Library/RestAPI/Serializable/ServerSettings.cs b/Duplicati/Library/RestAPI/Serializable/ServerSettings.cs index 180e7f0fd..6174fab70 100644 --- a/Duplicati/Library/RestAPI/Serializable/ServerSettings.cs +++ b/Duplicati/Library/RestAPI/Serializable/ServerSettings.cs @@ -18,9 +18,8 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; + using System.Linq; -using Duplicati.Library.RestAPI; using Duplicati.Server.Serialization.Interface; namespace Duplicati.Server.Serializable @@ -38,7 +37,7 @@ namespace Duplicati.Server.Serializable /// /// Constructor for backend interface /// - public DynamicModule(Duplicati.Library.Interface.IBackend backend) + public DynamicModule(Library.Interface.IBackend backend) { this.Key = backend.ProtocolKey; this.Description = backend.Description; @@ -49,7 +48,7 @@ namespace Duplicati.Server.Serializable /// /// Constructor for compression module interface /// - public DynamicModule(Duplicati.Library.Interface.ICompression module) + public DynamicModule(Library.Interface.ICompression module) { this.Key = module.FilenameExtension; this.Description = module.Description; @@ -60,7 +59,7 @@ namespace Duplicati.Server.Serializable /// /// Constructor for encryption module interface /// - public DynamicModule(Duplicati.Library.Interface.IEncryption module) + public DynamicModule(Library.Interface.IEncryption module) { this.Key = module.FilenameExtension; this.Description = module.Description; @@ -71,7 +70,7 @@ namespace Duplicati.Server.Serializable /// /// Constructor for generic module interface /// - public DynamicModule(Duplicati.Library.Interface.IGenericModule module) + public DynamicModule(Library.Interface.IGenericModule module) { this.Key = module.Key; this.Description = module.Description; @@ -82,7 +81,7 @@ namespace Duplicati.Server.Serializable /// /// Constructor for webmodule interface /// - public DynamicModule(Duplicati.Library.Interface.IWebModule module) + public DynamicModule(Library.Interface.IWebModule module) { this.Key = module.Key; this.Description = module.Description; @@ -93,7 +92,7 @@ namespace Duplicati.Server.Serializable /// /// Constructor for sercretprovider interface /// - public DynamicModule(Duplicati.Library.Interface.ISecretProvider module) + public DynamicModule(Library.Interface.ISecretProvider module) { this.Key = module.Key; this.Description = module.Description; @@ -116,13 +115,13 @@ namespace Duplicati.Server.Serializable /// /// The options supported by the module /// - public Duplicati.Library.Interface.ICommandLineArgument[] Options { get; private set; } + public Library.Interface.ICommandLineArgument[] Options { get; private set; } } /// /// Gets all supported options /// - public static Duplicati.Library.Interface.ICommandLineArgument[] Options + public static Library.Interface.ICommandLineArgument[] Options { get { @@ -242,21 +241,5 @@ namespace Duplicati.Server.Serializable .ToArray(); } } - - /// - /// The filters that are applied to all backups - /// - public static IFilter[] Filters - { - get { return FIXMEGlobal.DataConnection.Filters; } - } - - /// - /// The settings applied to all backups by default - /// - public static ISetting[] Settings - { - get { return FIXMEGlobal.DataConnection.Settings; } - } } } diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index fa651829f..69aec6ccd 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -97,7 +97,7 @@ namespace Duplicati.Server /// /// This is the only access to the database /// - public static Database.Connection DataConnection { get => FIXMEGlobal.DataConnection; set => FIXMEGlobal.DataConnection = value; } + public static Database.Connection DataConnection { get; set; } /// /// The thread running the ping-pong handler diff --git a/Duplicati/WebserverCore/Services/RemoteControllerHandler.cs b/Duplicati/WebserverCore/Services/RemoteControllerHandler.cs index 6664e2918..5259eef67 100644 --- a/Duplicati/WebserverCore/Services/RemoteControllerHandler.cs +++ b/Duplicati/WebserverCore/Services/RemoteControllerHandler.cs @@ -86,10 +86,10 @@ public class RemoteControllerHandler(Connection connection, IHttpClientFactory h try { if (string.Equals(message.ControlRequestMessage.Command, ControlRequestMessage.ConfigureReportUrlSet, StringComparison.OrdinalIgnoreCase)) - FIXMEGlobal.DataConnection.ApplicationSettings.AdditionalReportUrl = message.ControlRequestMessage.Parameters.GetValueOrDefault(ControlRequestMessage.ConfigureReportUrlParameter); + connection.ApplicationSettings.AdditionalReportUrl = message.ControlRequestMessage.Parameters.GetValueOrDefault(ControlRequestMessage.ConfigureReportUrlParameter); if (string.Equals(message.ControlRequestMessage.Command, ControlRequestMessage.ConfigureReportUrlGet, StringComparison.OrdinalIgnoreCase)) - result = new Dictionary { { ControlRequestMessage.ConfigureReportUrlParameter, FIXMEGlobal.DataConnection.ApplicationSettings.AdditionalReportUrl } }; + result = new Dictionary { { ControlRequestMessage.ConfigureReportUrlParameter, connection.ApplicationSettings.AdditionalReportUrl } }; } catch (Exception ex) { From 467c6eb44cbbc8a70630d6a3cab2fb73881e2a5c Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 4 May 2025 12:21:06 +0200 Subject: [PATCH 11/24] One more removal from FIXMEGlobal --- Duplicati/Agent/Program.cs | 6 +++--- Duplicati/Library/RestAPI/FIXMEGlobal.cs | 2 -- Duplicati/Server/Program.cs | 1 - Duplicati/UnitTest/ImportExportTests.cs | 4 ---- 4 files changed, 3 insertions(+), 10 deletions(-) diff --git a/Duplicati/Agent/Program.cs b/Duplicati/Agent/Program.cs index 53df9f5c4..ff1513016 100644 --- a/Duplicati/Agent/Program.cs +++ b/Duplicati/Agent/Program.cs @@ -320,13 +320,13 @@ public static class Program } private static Task> OnConnect(Dictionary metadata) - => FIXMEGlobal.Provider.GetRequiredService().OnConnect(metadata); + => Server.Program.DuplicatiWebserver.Provider.GetRequiredService().OnConnect(metadata); private static Task OnControl(KeepRemoteConnection.ControlMessage message) - => FIXMEGlobal.Provider.GetRequiredService().OnControl(message); + => Server.Program.DuplicatiWebserver.Provider.GetRequiredService().OnControl(message); private static Task OnMessage(KeepRemoteConnection.CommandMessage message) - => FIXMEGlobal.Provider.GetRequiredService().OnMessage(message); + => Server.Program.DuplicatiWebserver.Provider.GetRequiredService().OnMessage(message); private static Task ReKey(ClaimedClientData keydata, CommandLineArguments agentConfig) diff --git a/Duplicati/Library/RestAPI/FIXMEGlobal.cs b/Duplicati/Library/RestAPI/FIXMEGlobal.cs index d4e6009d2..02a401878 100644 --- a/Duplicati/Library/RestAPI/FIXMEGlobal.cs +++ b/Duplicati/Library/RestAPI/FIXMEGlobal.cs @@ -36,8 +36,6 @@ namespace Duplicati.Library.RestAPI /// public const string PASSWORD_PLACEHOLDER = "**********"; - public static IServiceProvider Provider { get; set; } - public static Action StartOrStopUsageReporter; /// diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index 69aec6ccd..7a7287dcb 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -413,7 +413,6 @@ namespace Duplicati.Server return server; }).ConfigureAwait(false); - FIXMEGlobal.Provider = server.Provider; ServerPortChanged |= server.Port != DataConnection.ApplicationSettings.LastWebserverPort; DataConnection.ApplicationSettings.LastWebserverPort = server.Port; diff --git a/Duplicati/UnitTest/ImportExportTests.cs b/Duplicati/UnitTest/ImportExportTests.cs index 50f6e49f4..40f99644d 100644 --- a/Duplicati/UnitTest/ImportExportTests.cs +++ b/Duplicati/UnitTest/ImportExportTests.cs @@ -131,10 +131,6 @@ namespace Duplicati.UnitTest var advancedOptions = new Dictionary { { "server-datafolder", this.serverDatafolder } }; // Mock the setup enough to get the import/export working - var serviceCollection = new ServiceCollection(); - serviceCollection.AddSingleton(); - serviceCollection.AddSingleton(new EventPollNotify()); - FIXMEGlobal.Provider = new DefaultServiceProviderFactory().CreateServiceProvider(serviceCollection); FIXMEGlobal.DataFolder = this.serverDatafolder; var dbpath = Path.Combine(this.serverDatafolder, Library.AutoUpdater.DataFolderManager.SERVER_DATABASE_FILENAME); From 0bc335598a0015e81c2f1670a3de64eb282390c2 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 5 May 2025 13:41:28 +0200 Subject: [PATCH 12/24] Removed the LogWriteHandler from FIXMEGlobal --- Duplicati/Library/Logging/ILogDestination.cs | 34 ++-- Duplicati/Library/Logging/LogEntry.cs | 38 ++--- .../RestAPI/Abstractions/ILogWriteHandler.cs | 143 ++++++++++++++++ Duplicati/Library/RestAPI/FIXMEGlobal.cs | 5 - Duplicati/Library/RestAPI/LogWriteHandler.cs | 159 ++++-------------- Duplicati/Library/RestAPI/Runner.cs | 4 +- Duplicati/Server/Program.cs | 34 ++-- Duplicati/WebserverCore/Dto/LogEntry.cs | 8 +- Duplicati/WebserverCore/DuplicatiWebserver.cs | 9 +- .../WebserverCore/Endpoints/V1/LogData.cs | 9 +- .../ServiceCollectionsExtensions.cs | 5 +- .../Services/CommandlineRunService.cs | 37 ++-- .../Services/QueueRunnerService.cs | 27 ++- 13 files changed, 269 insertions(+), 243 deletions(-) create mode 100644 Duplicati/Library/RestAPI/Abstractions/ILogWriteHandler.cs diff --git a/Duplicati/Library/Logging/ILogDestination.cs b/Duplicati/Library/Logging/ILogDestination.cs index 5eca15ab3..c92302cd5 100644 --- a/Duplicati/Library/Logging/ILogDestination.cs +++ b/Duplicati/Library/Logging/ILogDestination.cs @@ -18,22 +18,18 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. - -using System; -using System.Collections.Generic; -using System.Text; - -namespace Duplicati.Library.Logging -{ - /// - /// Interface for a log destination - /// - public interface ILogDestination - { - /// - /// The function called when a message is logged - /// - /// The message to log - void WriteMessage(LogEntry entry); - } -} + +namespace Duplicati.Library.Logging +{ + /// + /// Interface for a log destination + /// + public interface ILogDestination + { + /// + /// The function called when a message is logged + /// + /// The message to log + void WriteMessage(LogEntry entry); + } +} diff --git a/Duplicati/Library/Logging/LogEntry.cs b/Duplicati/Library/Logging/LogEntry.cs index e12441052..2b49e2b86 100644 --- a/Duplicati/Library/Logging/LogEntry.cs +++ b/Duplicati/Library/Logging/LogEntry.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; diff --git a/Duplicati/Library/RestAPI/Abstractions/ILogWriteHandler.cs b/Duplicati/Library/RestAPI/Abstractions/ILogWriteHandler.cs new file mode 100644 index 000000000..d041ad7b9 --- /dev/null +++ b/Duplicati/Library/RestAPI/Abstractions/ILogWriteHandler.cs @@ -0,0 +1,143 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using Duplicati.Library.Interface; +using Duplicati.Library.Logging; + +namespace Duplicati.WebserverCore.Abstractions; + +/// +/// Handles logging from the server, +/// and provides an entry point for the runner +/// to redirect log output to a file +/// +public interface ILogWriteHandler : ILogDestination, IDisposable +{ + /// + /// Represents a single log event + /// + public struct LiveLogEntry + { + /// + /// The context key used for conveying the backup ID + /// + public const string LOG_EXTRA_BACKUPID = "BackupID"; + /// + /// The context key used for conveying the task ID + /// + public const string LOG_EXTRA_TASKID = "TaskID"; + + /// + /// A unique ID that sequentially increments + /// + private static long _id; + + /// + /// The time the message was logged + /// + public readonly DateTime When; + + /// + /// The ID assigned to the message + /// + public readonly long ID; + + /// + /// The logged message + /// + public readonly string Message; + + /// + /// The log tag + /// + public readonly string Tag; + + /// + /// The message ID + /// + public readonly string MessageID; + + /// + /// The message ID + /// + public readonly string? ExceptionID; + + /// + /// The message type + /// + public readonly LogMessageType Type; + + /// + /// Exception data attached to the message + /// + public readonly Exception? Exception; + + /// + /// The backup ID, if any + /// + public readonly string? BackupID; + + /// + /// The task ID, if any + /// + public readonly string TaskID; + + /// + /// Initializes a new instance of the struct. + /// + /// The log entry to store + public LiveLogEntry(LogEntry entry) + { + this.ID = System.Threading.Interlocked.Increment(ref _id); + this.When = entry.When; + this.Message = entry.FormattedMessage; + this.Type = entry.Level; + this.Exception = entry.Exception; + this.Tag = entry.FilterTag; + this.MessageID = entry.Id; + this.BackupID = entry[LOG_EXTRA_BACKUPID]; + this.TaskID = entry[LOG_EXTRA_TASKID]; + + if (entry.Exception == null) + this.ExceptionID = null; + else if (entry.Exception is UserInformationException exception) + this.ExceptionID = exception.HelpID; + else + this.ExceptionID = entry.Exception.GetType().FullName; + + } + } + + + void RenewTimeout(LogMessageType type); + + void SetServerFile(string path, LogMessageType level); + + void AppendLogDestination(ILogDestination destination, LogMessageType level); + + LiveLogEntry[] AfterTime(DateTime offset, LogMessageType level); + + LiveLogEntry[] AfterID(long id, LogMessageType level, int pagesize); +} + diff --git a/Duplicati/Library/RestAPI/FIXMEGlobal.cs b/Duplicati/Library/RestAPI/FIXMEGlobal.cs index 02a401878..882e50dd3 100644 --- a/Duplicati/Library/RestAPI/FIXMEGlobal.cs +++ b/Duplicati/Library/RestAPI/FIXMEGlobal.cs @@ -43,11 +43,6 @@ namespace Duplicati.Library.RestAPI /// public static string DataFolder; - /// - /// The log redirect handler - /// - public static readonly LogWriteHandler LogHandler = new LogWriteHandler(); - /// /// Used to check the origin of the web server (e.g. Tray icon or a stand alone Server) /// diff --git a/Duplicati/Library/RestAPI/LogWriteHandler.cs b/Duplicati/Library/RestAPI/LogWriteHandler.cs index 72aa92f60..6b0613d67 100644 --- a/Duplicati/Library/RestAPI/LogWriteHandler.cs +++ b/Duplicati/Library/RestAPI/LogWriteHandler.cs @@ -1,30 +1,30 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Linq; -using Duplicati.Library.Logging; using System.Collections.Generic; -using Duplicati.Library.Interface; using Duplicati.Library.Main; +using Duplicati.WebserverCore.Abstractions; +using Duplicati.Library.Logging; namespace Duplicati.Server { @@ -33,7 +33,7 @@ namespace Duplicati.Server /// and provides an entry point for the runner /// to redirect log output to a file /// - public class LogWriteHandler : ILogDestination, IDisposable + public class LogWriteHandler : ILogWriteHandler { /// /// The number of messages to keep when inactive @@ -44,101 +44,6 @@ namespace Duplicati.Server /// private const int ACTIVE_SIZE = 5000; - /// - /// The context key used for conveying the backup ID - /// - public const string LOG_EXTRA_BACKUPID = "BackupID"; - /// - /// The context key used for conveying the task ID - /// - public const string LOG_EXTRA_TASKID = "TaskID"; - - /// - /// Represents a single log event - /// - public struct LogEntry - { - /// - /// A unique ID that sequentially increments - /// - private static long _id; - - /// - /// The time the message was logged - /// - public readonly DateTime When; - - /// - /// The ID assigned to the message - /// - public readonly long ID; - - /// - /// The logged message - /// - public readonly string Message; - - /// - /// The log tag - /// - public readonly string Tag; - - /// - /// The message ID - /// - public readonly string MessageID; - - /// - /// The message ID - /// - public readonly string ExceptionID; - - /// - /// The message type - /// - public readonly LogMessageType Type; - - /// - /// Exception data attached to the message - /// - public readonly Exception Exception; - - /// - /// The backup ID, if any - /// - public readonly string BackupID; - - /// - /// The task ID, if any - /// - public readonly string TaskID; - - /// - /// Initializes a new instance of the struct. - /// - /// The log entry to store - public LogEntry(Duplicati.Library.Logging.LogEntry entry) - { - this.ID = System.Threading.Interlocked.Increment(ref _id); - this.When = entry.When; - this.Message = entry.FormattedMessage; - this.Type = entry.Level; - this.Exception = entry.Exception; - this.Tag = entry.FilterTag; - this.MessageID = entry.Id; - this.BackupID = entry[LOG_EXTRA_BACKUPID]; - this.TaskID = entry[LOG_EXTRA_TASKID]; - - if (entry.Exception == null) - this.ExceptionID = null; - else if (entry.Exception is UserInformationException exception) - this.ExceptionID = exception.HelpID; - else - this.ExceptionID = entry.Exception.GetType().FullName; - - } - } - /// /// Basic implementation of a ring-buffer /// @@ -208,7 +113,7 @@ namespace Duplicati.Server private readonly DateTime[] m_timeouts; private readonly object m_lock = new object(); private volatile bool m_anytimeouts = false; - private RingBuffer m_buffer; + private RingBuffer m_buffer; private readonly ControllerMultiLogTarget m_target = new ControllerMultiLogTarget(null, LogMessageType.Warning, null); @@ -218,7 +123,7 @@ namespace Duplicati.Server { var fields = Enum.GetValues(typeof(LogMessageType)); m_timeouts = new DateTime[fields.Length]; - m_buffer = new RingBuffer(INACTIVE_SIZE); + m_buffer = new RingBuffer(INACTIVE_SIZE); } public void RenewTimeout(LogMessageType type) @@ -228,7 +133,7 @@ namespace Duplicati.Server m_timeouts[(int)type] = DateTime.Now.AddSeconds(30); m_anytimeouts = true; if (m_buffer == null || m_buffer.Size == INACTIVE_SIZE) - m_buffer = new RingBuffer(ACTIVE_SIZE, m_buffer); + m_buffer = new RingBuffer(ACTIVE_SIZE, m_buffer); } } @@ -248,7 +153,7 @@ namespace Duplicati.Server UpdateLogLevel(); } - public LogEntry[] AfterTime(DateTime offset, LogMessageType level) + public ILogWriteHandler.LiveLogEntry[] AfterTime(DateTime offset, LogMessageType level) { RenewTimeout(level); UpdateLogLevel(); @@ -257,13 +162,13 @@ namespace Duplicati.Server lock (m_lock) { if (m_buffer == null) - return new LogEntry[0]; + return new ILogWriteHandler.LiveLogEntry[0]; return m_buffer.FlatArray((x) => x.When > offset && x.Type >= level); } } - public LogEntry[] AfterID(long id, LogMessageType level, int pagesize) + public ILogWriteHandler.LiveLogEntry[] AfterID(long id, LogMessageType level, int pagesize) { RenewTimeout(level); UpdateLogLevel(); @@ -271,7 +176,7 @@ namespace Duplicati.Server lock (m_lock) { if (m_buffer == null) - return new LogEntry[0]; + return []; var buffer = m_buffer.FlatArray((x) => x.ID > id && x.Type >= level); // Return the newest entries @@ -305,7 +210,7 @@ namespace Duplicati.Server #region ILog implementation - public void WriteMessage(Duplicati.Library.Logging.LogEntry entry) + public void WriteMessage(LogEntry entry) { if (entry.Level < m_logLevel) return; @@ -329,13 +234,13 @@ namespace Duplicati.Server UpdateLogLevel(); m_anytimeouts = false; if (m_buffer == null || m_buffer.Size != INACTIVE_SIZE) - m_buffer = new RingBuffer(INACTIVE_SIZE, m_buffer); + m_buffer = new RingBuffer(INACTIVE_SIZE, m_buffer); } } if (m_buffer != null) - m_buffer.Enqueue(new LogEntry(entry)); + m_buffer.Enqueue(new ILogWriteHandler.LiveLogEntry(entry)); } } diff --git a/Duplicati/Library/RestAPI/Runner.cs b/Duplicati/Library/RestAPI/Runner.cs index 9374c6826..15ccc4898 100644 --- a/Duplicati/Library/RestAPI/Runner.cs +++ b/Duplicati/Library/RestAPI/Runner.cs @@ -560,8 +560,8 @@ namespace Duplicati.Server // Attach a log scope that tags all messages to relay the TaskID and BackupID using (Library.Logging.Log.StartScope(log => { - log[LogWriteHandler.LOG_EXTRA_TASKID] = data.TaskID.ToString(); - log[LogWriteHandler.LOG_EXTRA_BACKUPID] = data.BackupID; + log[ILogWriteHandler.LiveLogEntry.LOG_EXTRA_TASKID] = data.TaskID.ToString(); + log[ILogWriteHandler.LiveLogEntry.LOG_EXTRA_BACKUPID] = data.BackupID; })) using (tempfolder) diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index 7a7287dcb..d51e1aaad 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -137,11 +137,6 @@ namespace Duplicati.Server /// public static readonly ManualResetEvent ServerStartedEvent = new ManualResetEvent(false); - /// - /// The log redirect handler - /// - public static LogWriteHandler LogHandler { get => FIXMEGlobal.LogHandler; } - private static System.Threading.Timer PurgeTempFilesTimer = null; public static int ServerPort @@ -216,7 +211,8 @@ namespace Duplicati.Server return 100; } - ConfigureLogging(commandlineOptions); + var logHandler = new LogWriteHandler(); + using var logScope = ConfigureLogging(logHandler, commandlineOptions); // Validate after logging is configured CommandLineArgumentValidator.ValidateArguments(SupportedCommands, commandlineOptions, KnownDuplicateOptions, ValidationIgnoredOptions); @@ -248,7 +244,7 @@ namespace Duplicati.Server DataConnection.LogError(null, "Error in updater", obj); }; - DuplicatiWebserver = StartWebServer(commandlineOptions, DataConnection).Await(); + DuplicatiWebserver = StartWebServer(commandlineOptions, DataConnection, logHandler).Await(); DataConnection.SetServiceProvider(DuplicatiWebserver.Provider); queueRunner = DuplicatiWebserver.Provider.GetRequiredService(); @@ -358,8 +354,8 @@ namespace Duplicati.Server () => PingPongThread?.Interrupt(), () => { - Library.Logging.Log.WriteInformationMessage(LOGTAG, "ServerStopped", Strings.Program.ServerStopped); - LogHandler?.Dispose(); + Log.WriteInformationMessage(LOGTAG, "ServerStopped", Strings.Program.ServerStopped); + logHandler?.Dispose(); } }; @@ -385,7 +381,7 @@ namespace Duplicati.Server return 0; } - private static async Task StartWebServer(IReadOnlyDictionary options, Connection connection) + private static async Task StartWebServer(IReadOnlyDictionary options, Connection connection, ILogWriteHandler logWriteHandler) { var server = await WebServerLoader.TryRunServer(options, connection, async parsedOptions => { @@ -402,7 +398,7 @@ namespace Duplicati.Server parsedOptions.PreAuthTokens ); - var server = DuplicatiWebserver.CreateWebServer(mappedSettings, connection); + var server = DuplicatiWebserver.CreateWebServer(mappedSettings, connection, logWriteHandler); // Start the server, but catch any configuration issues var task = server.Start(); @@ -639,8 +635,9 @@ namespace Duplicati.Server } } - private static void ConfigureLogging(Dictionary commandlineOptions) + private static IDisposable ConfigureLogging(ILogWriteHandler logWriteHandler, Dictionary commandlineOptions) { + IDisposable logScope; //Log various information in the logfile if (DEBUG_MODE && !commandlineOptions.ContainsKey(LOG_FILE_OPTION)) { @@ -651,7 +648,7 @@ namespace Duplicati.Server System.IO.File.Delete(commandlineOptions[LOG_FILE_OPTION]); } - Log.StartScope(LogHandler, null); + logScope = Log.StartScope(logWriteHandler, null); if (commandlineOptions.ContainsKey(LOG_FILE_OPTION)) { @@ -659,7 +656,7 @@ namespace Duplicati.Server if (commandlineOptions.ContainsKey(LOG_LEVEL_OPTION)) Enum.TryParse(commandlineOptions[LOG_LEVEL_OPTION], true, out loglevel); - LogHandler.SetServerFile(commandlineOptions[LOG_FILE_OPTION], loglevel); + logWriteHandler.SetServerFile(commandlineOptions[LOG_FILE_OPTION], loglevel); } if (Library.Utility.Utility.ParseBoolOption(commandlineOptions, LOG_CONSOLE_OPTION)) @@ -668,7 +665,7 @@ namespace Duplicati.Server if (commandlineOptions.ContainsKey(LOG_LEVEL_OPTION)) Enum.TryParse(commandlineOptions[LOG_LEVEL_OPTION], true, out loglevel); - LogHandler.AppendLogDestination(new ConsoleLogDestination(loglevel), loglevel); + logWriteHandler.AppendLogDestination(new ConsoleLogDestination(loglevel), loglevel); } if (commandlineOptions.TryGetValue(WINDOWS_EVENTLOG_OPTION, out var source) && !string.IsNullOrEmpty(source)) @@ -694,16 +691,17 @@ namespace Duplicati.Server if (WindowsEventLogSource.SourceExists(source)) { - var loglevel = Library.Logging.LogMessageType.Information; + var loglevel = LogMessageType.Information; if (commandlineOptions.ContainsKey(WINDOWS_EVENTLOG_LEVEL_OPTION)) Enum.TryParse(commandlineOptions[WINDOWS_EVENTLOG_LEVEL_OPTION], true, out loglevel); - LogHandler.AppendLogDestination(new WindowsEventLogSource(source), loglevel); + logWriteHandler.AppendLogDestination(new WindowsEventLogSource(source), loglevel); } } } - CrashlogHelper.OnUnobservedTaskException += (ex) => LogHandler.WriteMessage(new Library.Logging.LogEntry(ex.Message, null, Library.Logging.LogMessageType.Error, LOGTAG, "UnobservedTaskException", ex)); + CrashlogHelper.OnUnobservedTaskException += (ex) => logWriteHandler.WriteMessage(new LogEntry(ex.Message, null, Library.Logging.LogMessageType.Error, LOGTAG, "UnobservedTaskException", ex)); + return logScope; } private static int ShowHelp(bool writeToConsoleOnExceptionw) diff --git a/Duplicati/WebserverCore/Dto/LogEntry.cs b/Duplicati/WebserverCore/Dto/LogEntry.cs index 0affbfeb9..1e19fab8b 100644 --- a/Duplicati/WebserverCore/Dto/LogEntry.cs +++ b/Duplicati/WebserverCore/Dto/LogEntry.cs @@ -19,7 +19,7 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Logging; -using Duplicati.Server; +using Duplicati.WebserverCore.Abstractions; namespace Duplicati.WebserverCore.Dto; @@ -56,7 +56,7 @@ public sealed record LogEntry /// /// The message ID /// - public required string ExceptionID { get; init; } + public required string? ExceptionID { get; init; } /// /// The message type @@ -71,7 +71,7 @@ public sealed record LogEntry /// /// The backup ID, if any /// - public required string BackupID { get; init; } + public required string? BackupID { get; init; } /// /// The task ID, if any @@ -83,7 +83,7 @@ public sealed record LogEntry /// /// The internal record /// The DTO record - public static LogEntry FromInternalEntry(LogWriteHandler.LogEntry entry) + public static LogEntry FromInternalEntry(ILogWriteHandler.LiveLogEntry entry) { return new LogEntry { diff --git a/Duplicati/WebserverCore/DuplicatiWebserver.cs b/Duplicati/WebserverCore/DuplicatiWebserver.cs index 1482319f3..967b55103 100644 --- a/Duplicati/WebserverCore/DuplicatiWebserver.cs +++ b/Duplicati/WebserverCore/DuplicatiWebserver.cs @@ -132,8 +132,9 @@ public class DuplicatiWebserver /// /// The settings for the server /// The connection to the database + /// The log write handler /// The new webserver instance - public static DuplicatiWebserver CreateWebServer(InitSettings settings, Connection connection) + public static DuplicatiWebserver CreateWebServer(InitSettings settings, Connection connection, ILogWriteHandler logWriteHandler) { var builder = WebApplication.CreateBuilder(new WebApplicationOptions() { @@ -279,7 +280,7 @@ public class DuplicatiWebserver builder.Services.AddHealthChecks() .AddCheck("Basic", () => HealthCheckResult.Healthy("Service is running")); - builder.Services.AddDuplicati(connection); + builder.Services.AddDuplicati(connection, logWriteHandler); // Prevent logs from spamming the console, but allow enabling for debugging if (Environment.GetEnvironmentVariable("DUPLICATI_WEBSERVER_LOGGING") != "1") @@ -387,9 +388,9 @@ public class DuplicatiWebserver var _ = Task.Run(() => app.Services.GetRequiredService().GetSystemInfo(null)); // Get a string description of the listen interface used - var listenInterface = settings.Interface == System.Net.IPAddress.Any + var listenInterface = settings.Interface == IPAddress.Any ? "*" - : settings.Interface == System.Net.IPAddress.Loopback + : settings.Interface == IPAddress.Loopback ? "localhost" : settings.Interface.ToString(); diff --git a/Duplicati/WebserverCore/Endpoints/V1/LogData.cs b/Duplicati/WebserverCore/Endpoints/V1/LogData.cs index 13c39324a..69c4c929e 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/LogData.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/LogData.cs @@ -19,7 +19,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Crashlog; -using Duplicati.Library.RestAPI; using Duplicati.Server.Database; using Duplicati.WebserverCore.Abstractions; using Duplicati.WebserverCore.Dto; @@ -31,8 +30,8 @@ public class LogData : IEndpointV1 { public static void Map(RouteGroupBuilder group) { - group.MapGet("/logdata/poll", ([FromServices] Connection connection, [FromQuery] Library.Logging.LogMessageType level, [FromQuery] long id, [FromQuery] long? offset, [FromQuery] int? pagesize) - => ExecuteLogPoll(level, id, offset ?? 0, pagesize ?? 100)) + group.MapGet("/logdata/poll", ([FromServices] ILogWriteHandler logWriteHandler, [FromQuery] Library.Logging.LogMessageType level, [FromQuery] long id, [FromQuery] int? pagesize) + => ExecuteLogPoll(logWriteHandler, level, id, pagesize ?? 100)) .RequireAuthorization(); group.MapGet("/logdata/log", ([FromServices] Connection connection, [FromQuery] long? offset, [FromQuery] int? pagesize) @@ -44,10 +43,10 @@ public class LogData : IEndpointV1 .RequireAuthorization(); } - private static LogEntry[] ExecuteLogPoll(Library.Logging.LogMessageType level, long id, long offset, int pagesize) + private static LogEntry[] ExecuteLogPoll(ILogWriteHandler logWriteHandler, Library.Logging.LogMessageType level, long id, int pagesize) { pagesize = Math.Max(1, Math.Min(500, pagesize)); - return FIXMEGlobal.LogHandler.AfterID(id, level, pagesize).Select(x => Dto.LogEntry.FromInternalEntry(x)).ToArray(); + return logWriteHandler.AfterID(id, level, pagesize).Select(x => Dto.LogEntry.FromInternalEntry(x)).ToArray(); } private static List>? ExecuteGetLog(Connection connection, long? offset, long pagesize) diff --git a/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs b/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs index f891663f4..d8ebfcc6d 100644 --- a/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs +++ b/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs @@ -35,7 +35,7 @@ namespace Duplicati.WebserverCore.Extensions; public static class ServiceCollectionsExtensions { - public static IServiceCollection AddDuplicati(this IServiceCollection services, Connection connection) + public static IServiceCollection AddDuplicati(this IServiceCollection services, Connection connection, ILogWriteHandler logWriteHandler) { //old part services @@ -67,7 +67,8 @@ public static class ServiceCollectionsExtensions .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton(); + .AddSingleton() + .AddSingleton(logWriteHandler); return services; } diff --git a/Duplicati/WebserverCore/Services/CommandlineRunService.cs b/Duplicati/WebserverCore/Services/CommandlineRunService.cs index 3d9747ab1..741bf31c1 100644 --- a/Duplicati/WebserverCore/Services/CommandlineRunService.cs +++ b/Duplicati/WebserverCore/Services/CommandlineRunService.cs @@ -19,34 +19,27 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.Text; -using Duplicati.Library.RestAPI; using Duplicati.Server; using Duplicati.WebserverCore.Abstractions; namespace Duplicati.WebserverCore.Services; -public class CommandlineRunService(IQueueRunnerService queueRunnerService) : ICommandlineRunService +public class CommandlineRunService(IQueueRunnerService queueRunnerService, ILogWriteHandler logWriteHandler) : ICommandlineRunService { private static readonly string LOGTAG = Library.Logging.Log.LogTagFromType(); - private class LogWriter : TextWriter + private class LogWriter(ActiveRun target, ILogWriteHandler logWriteHandler) : TextWriter { - private readonly ActiveRun m_target; - private readonly StringBuilder m_sb = new StringBuilder(); + private readonly StringBuilder sb = new StringBuilder(); private int m_newlinechars = 0; - public LogWriter(ActiveRun target) - { - m_target = target; - } - public override Encoding Encoding => Encoding.UTF8; public override void Write(char value) { - lock (m_target.Lock) + lock (target.Lock) { - m_sb.Append(value); + sb.Append(value); if (NewLine[m_newlinechars] == value) { m_newlinechars++; @@ -61,34 +54,34 @@ public class CommandlineRunService(IQueueRunnerService queueRunnerService) : ICo public override void WriteLine(string? value) { value ??= string.Empty; - lock (m_target.Lock) + lock (target.Lock) { - m_target.LastAccess = DateTime.Now; + target.LastAccess = DateTime.Now; //Avoid writing the log if it does not exist - if (m_target.IsLogDisposed) + if (target.IsLogDisposed) { - FIXMEGlobal.LogHandler.WriteMessage(new Library.Logging.LogEntry("Attempted to write message after closing: {0}", new object[] { value }, Library.Logging.LogMessageType.Warning, LOGTAG, "CommandLineOutputAfterLogClosed", null)); + logWriteHandler.WriteMessage(new Library.Logging.LogEntry("Attempted to write message after closing: {0}", new object[] { value }, Library.Logging.LogMessageType.Warning, LOGTAG, "CommandLineOutputAfterLogClosed", null)); return; } try { - if (m_sb.Length != 0) + if (sb.Length != 0) { - m_target.Log.Add(m_sb + value); - m_sb.Length = 0; + target.Log.Add(sb + value); + sb.Length = 0; m_newlinechars = 0; } else { - m_target.Log.Add(value); + target.Log.Add(value); } } catch (Exception ex) { // This can happen on a very unlucky race where IsLogDisposed is set right after the check - FIXMEGlobal.LogHandler.WriteMessage(new Library.Logging.LogEntry("Failed to forward commandline message: {0}", new object[] { value }, Library.Logging.LogMessageType.Warning, LOGTAG, "CommandLineOutputAfterLogClosed", ex)); + logWriteHandler.WriteMessage(new Library.Logging.LogEntry("Failed to forward commandline message: {0}", new object[] { value }, Library.Logging.LogMessageType.Warning, LOGTAG, "CommandLineOutputAfterLogClosed", ex)); } } } @@ -140,7 +133,7 @@ public class CommandlineRunService(IQueueRunnerService queueRunnerService) : ICo public string StartTask(string[] args) { var k = new ActiveRun(); - k.Writer = new LogWriter(k); + k.Writer = new LogWriter(k, logWriteHandler); m_activeItems[k.ID] = k; StartCleanupTask(); diff --git a/Duplicati/WebserverCore/Services/QueueRunnerService.cs b/Duplicati/WebserverCore/Services/QueueRunnerService.cs index 87cd8daba..57fa6a3e4 100644 --- a/Duplicati/WebserverCore/Services/QueueRunnerService.cs +++ b/Duplicati/WebserverCore/Services/QueueRunnerService.cs @@ -50,8 +50,7 @@ public class QueueRunnerService( private static readonly int MAX_TASK_RESULT_CACHE_SIZE = 100; private readonly List _tasks = new(); - private Task? _currentTask; - private IQueuedTask? _activeTask; + private (Task? Task, IQueuedTask? QueuedTask) _current; private bool _isPaused; private bool _isTerminated; @@ -72,10 +71,10 @@ public class QueueRunnerService( } public bool GetIsActive() - => _activeTask != null; + => _current.Task != null; public IQueuedTask? GetCurrentTask() - => _activeTask; + => _current.QueuedTask; public List GetCurrentTasks() { @@ -102,7 +101,7 @@ public class QueueRunnerService( _isTerminated = true; if (wait) { - var task = _currentTask; + var task = _current.Task; if (task != null) task.Await(); } @@ -112,23 +111,19 @@ public class QueueRunnerService( { lock (_lock) { - if (_isTerminated || _isPaused || (_currentTask != null && !_currentTask.IsCompleted)) + if (_isTerminated || _isPaused || (_current.Task != null && !_current.Task.IsCompleted)) return; // Clean up completed tasks - if (_currentTask != null && _currentTask.IsCompleted) - { - _currentTask = null; - _activeTask = null; - } + if (_current.Task != null && _current.Task.IsCompleted) + _current = (null, null); if (_tasks.Count == 0) return; - _activeTask = _tasks[0]; + var nextTask = _tasks[0]; _tasks.RemoveAt(0); - var task = _activeTask; - _currentTask = Task.Run(() => RunTask(task), CancellationToken.None); + _current = (Task.Run(() => RunTask(nextTask), CancellationToken.None), nextTask); } } @@ -163,8 +158,8 @@ public class QueueRunnerService( finally { task.TaskFinished = DateTime.UtcNow; - _currentTask = null; - _activeTask = null; + lock (_lock) + _current = (null, null); eventPollNotify.SignalNewEvent(); StartNextTask(); } From c91ff5705dea92343e9e097b62f6dc23fbac898a Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 5 May 2025 13:46:44 +0200 Subject: [PATCH 13/24] Fixed issue with runner tasks not logging to livelog --- Duplicati/WebserverCore/Services/QueueRunnerService.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Duplicati/WebserverCore/Services/QueueRunnerService.cs b/Duplicati/WebserverCore/Services/QueueRunnerService.cs index 57fa6a3e4..71dbe4a5e 100644 --- a/Duplicati/WebserverCore/Services/QueueRunnerService.cs +++ b/Duplicati/WebserverCore/Services/QueueRunnerService.cs @@ -36,7 +36,8 @@ public class QueueRunnerService( Connection connection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, - IProgressStateProviderService progressStateProviderService) : IQueueRunnerService + IProgressStateProviderService progressStateProviderService, + ILogWriteHandler logWriteHandler) : IQueueRunnerService { private readonly object _lock = new(); /// @@ -132,6 +133,8 @@ public class QueueRunnerService( var completed = false; try { + using var log = Library.Logging.Log.StartScope(logWriteHandler); + eventPollNotify.SignalNewEvent(); task.TaskStarted = DateTime.UtcNow; if (task.OnStarting != null) From 4191ed31cb7567950b66007e7edf37356d750276 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 5 May 2025 14:20:59 +0200 Subject: [PATCH 14/24] Removed unused ServerName property --- Duplicati/Server/Program.cs | 1 - Duplicati/Server/WebServerLoader.cs | 3 --- Duplicati/WebserverCore/DuplicatiWebserver.cs | 2 -- 3 files changed, 6 deletions(-) diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index d51e1aaad..b4349c2ab 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -390,7 +390,6 @@ namespace Duplicati.Server parsedOptions.Port, parsedOptions.Interface, parsedOptions.Certificate, - parsedOptions.Servername, parsedOptions.AllowedHostnames, parsedOptions.DisableStaticFiles, parsedOptions.SPAPaths, diff --git a/Duplicati/Server/WebServerLoader.cs b/Duplicati/Server/WebServerLoader.cs index cfb537d19..c043059ae 100644 --- a/Duplicati/Server/WebServerLoader.cs +++ b/Duplicati/Server/WebServerLoader.cs @@ -158,7 +158,6 @@ public static class WebServerLoader /// The listining port /// The listening interface /// SSL certificate, if any - /// The servername to report /// The allowed hostnames /// If static files should be disabled /// The paths to serve as SPAs @@ -168,7 +167,6 @@ public static class WebServerLoader int Port, System.Net.IPAddress Interface, X509Certificate2Collection? Certificate, - string Servername, IEnumerable AllowedHostnames, bool DisableStaticFiles, IEnumerable SPAPaths, @@ -264,7 +262,6 @@ public static class WebServerLoader -1, listenInterface, connection.ApplicationSettings.UseHTTPS ? connection.ApplicationSettings.ServerSSLCertificate : null, - string.Format("{0} v{1}", Library.AutoUpdater.AutoUpdateSettings.AppName, Library.AutoUpdater.UpdaterManager.SelfVersion.Version), (connection.ApplicationSettings.AllowedHostnames ?? string.Empty).Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries), Utility.ParseBoolOption(options, OPTION_WEBSERVICE_API_ONLY), spaPathsString.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries), diff --git a/Duplicati/WebserverCore/DuplicatiWebserver.cs b/Duplicati/WebserverCore/DuplicatiWebserver.cs index 967b55103..ccc4c53e1 100644 --- a/Duplicati/WebserverCore/DuplicatiWebserver.cs +++ b/Duplicati/WebserverCore/DuplicatiWebserver.cs @@ -108,7 +108,6 @@ public class DuplicatiWebserver /// The listining port /// The listening interface /// The certificate, if using SSL - /// The servername to report /// The allowed hostnames /// If static files should be disabled /// The paths to serve as SPAs @@ -119,7 +118,6 @@ public class DuplicatiWebserver int Port, System.Net.IPAddress Interface, X509Certificate2Collection? Certificate, - string Servername, IEnumerable AllowedHostnames, bool DisableStaticFiles, IEnumerable SPAPaths, From 365621f4b9cba60fc312167b773a17d2c03f922e Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 5 May 2025 14:21:49 +0200 Subject: [PATCH 15/24] Fixed log scope to track all requests --- Duplicati/WebserverCore/DuplicatiWebserver.cs | 7 +++++++ Duplicati/WebserverCore/Services/QueueRunnerService.cs | 2 -- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Duplicati/WebserverCore/DuplicatiWebserver.cs b/Duplicati/WebserverCore/DuplicatiWebserver.cs index ccc4c53e1..30b45e63c 100644 --- a/Duplicati/WebserverCore/DuplicatiWebserver.cs +++ b/Duplicati/WebserverCore/DuplicatiWebserver.cs @@ -330,6 +330,13 @@ public class DuplicatiWebserver if (!settings.DisableStaticFiles) app.UseDefaultStaticFiles(settings.WebRoot, settings.SPAPaths); + app.Use(async (context, next) => + { + // Set up the log scope to capture any log messages for the livelog / server log + using (var scope = Library.Logging.Log.StartScope(logWriteHandler)) + await next(); + }); + app.UseExceptionHandler(app => { app.Run(async context => diff --git a/Duplicati/WebserverCore/Services/QueueRunnerService.cs b/Duplicati/WebserverCore/Services/QueueRunnerService.cs index 71dbe4a5e..4075e41b7 100644 --- a/Duplicati/WebserverCore/Services/QueueRunnerService.cs +++ b/Duplicati/WebserverCore/Services/QueueRunnerService.cs @@ -133,8 +133,6 @@ public class QueueRunnerService( var completed = false; try { - using var log = Library.Logging.Log.StartScope(logWriteHandler); - eventPollNotify.SignalNewEvent(); task.TaskStarted = DateTime.UtcNow; if (task.OnStarting != null) From f47f218d110539e5776d3f9ae49480bc07278311 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 5 May 2025 14:22:24 +0200 Subject: [PATCH 16/24] Don't run DB commands on no-op delete --- .../Library/Main/Operation/DeleteHandler.cs | 80 ++++++++++--------- 1 file changed, 43 insertions(+), 37 deletions(-) diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs index 0466b2fc5..6e291fc4e 100644 --- a/Duplicati/Library/Main/Operation/DeleteHandler.cs +++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs @@ -89,50 +89,56 @@ namespace Duplicati.Library.Main.Operation versionsToDelete = versionsToDelete.OrderBy(x => x.Version).Skip(1).ToList(); } - if (versionsToDelete.Count > 0) - Logging.Log.WriteInformationMessage(LOGTAG, "DeleteRemoteFileset", "Deleting {0} remote fileset(s) ...", versionsToDelete.Count); - - var lst = db.DropFilesetsFromTable(versionsToDelete.Select(x => x.Time).ToArray(), rtr.Transaction).ToArray(); - foreach (var f in lst) - db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, rtr.Transaction); - - if (!m_options.Dryrun) - rtr.Commit("CommitBeforeDelete"); - - foreach (var f in lst) + if (versionsToDelete.Count == 0) { - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - { - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - return; - } - - if (!m_options.Dryrun) - await backendManager.DeleteAsync(f.Key, f.Value, false, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - else - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFileset", "Would delete remote fileset: {0}", f.Key); - } - - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - - var count = lst.Length; - if (!m_options.Dryrun) - { - if (count == 0) - Logging.Log.WriteInformationMessage(LOGTAG, "DeleteResults", "No remote filesets were deleted"); - else - Logging.Log.WriteInformationMessage(LOGTAG, "DeleteResults", "Deleted {0} remote fileset(s)", count); + Logging.Log.WriteInformationMessage(LOGTAG, "NoFilesetsToDelete", "No remote filesets should be deleted"); } else { + Logging.Log.WriteInformationMessage(LOGTAG, "DeleteRemoteFileset", "Deleting {0} remote fileset(s) ...", versionsToDelete.Count); - if (count == 0) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteResults", "No remote filesets would be deleted"); + var lst = db.DropFilesetsFromTable(versionsToDelete.Select(x => x.Time).ToArray(), rtr.Transaction).ToArray(); + foreach (var f in lst) + db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, rtr.Transaction); + + if (!m_options.Dryrun) + rtr.Commit("CommitBeforeDelete"); + + foreach (var f in lst) + { + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + { + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + return; + } + + if (!m_options.Dryrun) + await backendManager.DeleteAsync(f.Key, f.Value, false, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + else + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFileset", "Would delete remote fileset: {0}", f.Key); + } + + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + + var count = lst.Length; + if (!m_options.Dryrun) + { + if (count == 0) + Logging.Log.WriteInformationMessage(LOGTAG, "DeleteResults", "No remote filesets were deleted"); + else + Logging.Log.WriteInformationMessage(LOGTAG, "DeleteResults", "Deleted {0} remote fileset(s)", count); + } else - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteResults", "{0} remote fileset(s) would be deleted", count); + { - if (count > 0 && m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteHelp", "Remove --dry-run to actually delete files"); + if (count == 0) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteResults", "No remote filesets would be deleted"); + else + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteResults", "{0} remote fileset(s) would be deleted", count); + + if (count > 0 && m_options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteHelp", "Remove --dry-run to actually delete files"); + } } if (!m_options.NoAutoCompact && (forceCompact || versionsToDelete.Count > 0)) From c9f985ecdd4a7c25d090f8a93e90c27167756060 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 5 May 2025 16:13:46 +0200 Subject: [PATCH 17/24] Final removal of FIXMEGlobal --- Duplicati/Agent/Program.cs | 29 ++- .../CommandLine/ServerUtil/Connection.cs | 4 +- .../HostedInstanceKeeper.cs | 11 +- .../GUI/Duplicati.GUI.TrayIcon/Program.cs | 27 +- .../Abstractions/IApplicationSettings.cs | 64 +++++ .../Library/RestAPI/Database/Connection.cs | 17 +- .../RestAPI/Database/ServerSettings.cs | 19 +- Duplicati/Library/RestAPI/FIXMEGlobal.cs | 66 ----- Duplicati/Library/RestAPI/Runner.cs | 8 +- Duplicati/Server/Program.cs | 236 ++++++++---------- Duplicati/UnitTest/ImportExportTests.cs | 13 +- Duplicati/WebserverCore/DuplicatiWebserver.cs | 5 +- .../Endpoints/V1/BackupDefaults.cs | 2 +- .../Endpoints/V1/RemoteControl.cs | 6 +- .../Endpoints/V1/RemoteOperation.cs | 13 +- .../WebserverCore/Endpoints/V1/WebModules.cs | 9 +- .../Endpoints/V2/DestinationVerify.cs | 13 +- .../ServiceCollectionsExtensions.cs | 5 +- .../Extensions/WebApplicationExtensions.cs | 4 + .../Services/ApplicationSettings.cs | 46 ++++ .../Services/QueueRunnerService.cs | 6 +- .../Services/RemoteControllerHandler.cs | 5 +- .../Services/SystemInfoProvider.cs | 14 +- 23 files changed, 324 insertions(+), 298 deletions(-) create mode 100644 Duplicati/Library/RestAPI/Abstractions/IApplicationSettings.cs delete mode 100644 Duplicati/Library/RestAPI/FIXMEGlobal.cs create mode 100644 Duplicati/WebserverCore/Services/ApplicationSettings.cs diff --git a/Duplicati/Agent/Program.cs b/Duplicati/Agent/Program.cs index ff1513016..d5d0351c5 100644 --- a/Duplicati/Agent/Program.cs +++ b/Duplicati/Agent/Program.cs @@ -30,9 +30,9 @@ using Duplicati.Library.Interface; using Duplicati.Library.Logging; using Duplicati.Library.Main; using Duplicati.Library.RemoteControl; -using Duplicati.Library.RestAPI; using Duplicati.Library.Utility; using Duplicati.Server; +using Duplicati.WebserverCore.Abstractions; using Duplicati.WebserverCore.Services; using Microsoft.Extensions.DependencyInjection; @@ -225,6 +225,9 @@ public static class Program if (string.IsNullOrWhiteSpace(agentConfig.SettingsEncryptionKey)) agentConfig = agentConfig with { SettingsEncryptionKey = Environment.GetEnvironmentVariable(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME) }; + // Prepare the application settings for the server + var applicationSettings = new ApplicationSettings(); + // Apply the secret provider, if present if (!string.IsNullOrWhiteSpace(agentConfig.SecretProvider)) { @@ -245,7 +248,7 @@ public static class Program options["secret-provider-cache"] = agentConfig.SecretProviderCache.ToString(); options["secret-provider-pattern"] = agentConfig.SecretProviderPattern; - FIXMEGlobal.SecretProvider = await SecretProviderHelper.ApplySecretProviderAsync([], [], options, TempFolder.SystemTempPath, null, CancellationToken.None); + applicationSettings.SecretProvider = await SecretProviderHelper.ApplySecretProviderAsync([], [], options, TempFolder.SystemTempPath, null, CancellationToken.None); // Apply the secret provider to the agent configuration foreach (var prop in agentProps) @@ -299,7 +302,7 @@ public static class Program } var t = await Task.WhenAny( - Task.Run(() => StartLocalServer(agentConfig, settings, cts.Token)), + Task.Run(() => StartLocalServer(agentConfig, applicationSettings, settings, cts.Token)), KeepRemoteConnection.Start( settings.ServerUrl, settings.JWT, @@ -307,7 +310,7 @@ public static class Program settings.ServerCertificates, cts.Token, OnConnect, - m => ReKey(m, agentConfig), + m => ReKey(applicationSettings, m, agentConfig), OnControl, OnMessage ) @@ -329,7 +332,7 @@ public static class Program => Server.Program.DuplicatiWebserver.Provider.GetRequiredService().OnMessage(message); - private static Task ReKey(ClaimedClientData keydata, CommandLineArguments agentConfig) + private static Task ReKey(IApplicationSettings applicationSettings, ClaimedClientData keydata, CommandLineArguments agentConfig) { // ReKey is handled here because we store the config outside of the database Log.WriteMessage(LogMessageType.Verbose, LogTag, "ReKey", "Rekeying the settings"); @@ -340,7 +343,7 @@ public static class Program settings = settings with { ServerCertificates = keydata.ServerCertificates }; // Only allow re-keying if the settings encryption key is not set - if (!FIXMEGlobal.SettingsEncryptionKeyProvidedExternally) + if (!applicationSettings.SettingsEncryptionKeyProvidedExternally) { if (!string.IsNullOrWhiteSpace(keydata.LocalEncryptionKey) && settings.SettingsEncryptionKey != keydata.LocalEncryptionKey) { @@ -358,14 +361,15 @@ public static class Program /// /// Runs the server, restarting on crashes /// + /// The application settings for the server /// The commandline arguments passed to the server /// The cancellation token to use for the process /// An awaitable task - private static async Task RunServer(string[] args, CancellationToken cancellationToken) + private static async Task RunServer(IApplicationSettings applicationSettings, string[] args, CancellationToken cancellationToken) { cancellationToken.Register(() => { - Server.Program.ApplicationExitEvent.Set(); + applicationSettings.ApplicationExitEvent.Set(); }); var lastRestart = DateTime.Now; @@ -374,7 +378,7 @@ public static class Program try { lastRestart = DateTime.Now; - Server.Program.Main(args); + Server.Program.Main(applicationSettings, args); } catch (Exception ex) { @@ -445,10 +449,11 @@ public static class Program /// Starts the local webserver in locked down mode, restarting on crashes /// /// The agent configuration + /// The application settings for the server /// The settings for the agent /// The cancellation token that stops the server /// An awaitable task - private static async Task StartLocalServer(CommandLineArguments agentConfig, Settings settings, CancellationToken cancellationToken) + private static async Task StartLocalServer(CommandLineArguments agentConfig, IApplicationSettings applicationSettings, Settings settings, CancellationToken cancellationToken) { // TODO: Look into pipes for Kestrel to prevent network access @@ -487,10 +492,10 @@ public static class Program .ToArray(); // Set the global origin - FIXMEGlobal.Origin = "Agent"; + applicationSettings.Origin = "Agent"; // Start the server - await RunServer(args, cancellationToken); + await RunServer(applicationSettings, args, cancellationToken); } } diff --git a/Duplicati/CommandLine/ServerUtil/Connection.cs b/Duplicati/CommandLine/ServerUtil/Connection.cs index 809cba5de..0df198122 100644 --- a/Duplicati/CommandLine/ServerUtil/Connection.cs +++ b/Duplicati/CommandLine/ServerUtil/Connection.cs @@ -24,6 +24,7 @@ using System.Net.Security; using System.Text.Json; using Duplicati.Library.AutoUpdater; using Duplicati.WebserverCore.Middlewares; +using Duplicati.WebserverCore.Services; namespace Duplicati.CommandLine.ServerUtil; @@ -141,7 +142,6 @@ public class Connection /// The connection public static async Task Connect(Settings settings, bool obtainRefreshToken = false, OutputInterceptor? console = null) { - if (console != null) console.AppendConsoleMessage($"Connecting to {settings.HostUrl}..."); else @@ -201,7 +201,7 @@ public class Connection if (File.Exists(Path.Combine(DataFolderManager.GetDataFolder(DataFolderManager.AccessMode.ProbeOnly), DataFolderManager.SERVER_DATABASE_FILENAME))) { string? cfg = null; - using (var connection = Server.Program.GetDatabaseConnection(opts, true)) + using (var connection = Server.Program.GetDatabaseConnection(new ApplicationSettings(), opts, true)) { cfg = connection.ApplicationSettings.JWTConfig; if (settings.HostUrl.Scheme == "https" && connection.ApplicationSettings.ServerSSLCertificate != null && trustedCertificateHashes.Count == 0) diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/HostedInstanceKeeper.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/HostedInstanceKeeper.cs index 2eea00385..d8e5f6550 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/HostedInstanceKeeper.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/HostedInstanceKeeper.cs @@ -21,6 +21,7 @@ using System; using System.Linq; +using Duplicati.WebserverCore.Abstractions; namespace Duplicati.GUI.TrayIcon { @@ -32,9 +33,11 @@ namespace Duplicati.GUI.TrayIcon private readonly System.Threading.Thread m_runner; private Exception m_runnerException = null; public Action InstanceShutdown; + private readonly IApplicationSettings applicationSettings; - public HostedInstanceKeeper(string[] args) + public HostedInstanceKeeper(IApplicationSettings applicationSettings, string[] args) { + this.applicationSettings = applicationSettings; m_runner = new System.Threading.Thread(_ => { try @@ -44,13 +47,13 @@ namespace Duplicati.GUI.TrayIcon if (args == null || !args.Any(x => x.Trim().StartsWith("--" + Server.WebServerLoader.OPTION_PORT + "=", StringComparison.OrdinalIgnoreCase))) args = (args ?? new string[0]).Union(new string[] { "--" + Server.WebServerLoader.OPTION_PORT + "=8200,8300,8400,8500,8600,8700,8800,8900,8989" }).ToArray(); - Server.Program.Main(args); + Server.Program.Main(applicationSettings, args); } catch (Exception ex) { m_runnerException = ex; Server.Program.ServerStartedEvent?.Set(); - Server.Program.ApplicationExitEvent?.Set(); + applicationSettings.ApplicationExitEvent?.Set(); } finally { @@ -85,7 +88,7 @@ namespace Duplicati.GUI.TrayIcon { try { - Server.Program.ApplicationExitEvent.Set(); + applicationSettings.ApplicationExitEvent.Set(); if (!m_runner.Join(TimeSpan.FromSeconds(10))) { m_runner.Interrupt(); diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs index 6fccabd6c..b16ccfad0 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs @@ -30,9 +30,11 @@ using CoCoL; using Duplicati.CommandLine; using Duplicati.Library.AutoUpdater; using Duplicati.Library.Interface; -using Duplicati.Library.RestAPI; using Duplicati.Library.Utility; using Duplicati.Server; +using Duplicati.Server.Database; +using Duplicati.WebserverCore.Services; +using Microsoft.Extensions.DependencyInjection; using Uri = System.Uri; namespace Duplicati.GUI.TrayIcon @@ -144,12 +146,13 @@ namespace Duplicati.GUI.TrayIcon try { // Tell the hosted server it was started by the TrayIcon - FIXMEGlobal.Origin = "Tray icon"; + var applicationSettings = new ApplicationSettings(); + applicationSettings.Origin = "Tray icon"; passwordSource = PasswordSource.HostedServer; // Ignore TrayIcon specific settings foreach (var c in BasicSupportedCommands.Select(x => x.Name)) Server.Program.ValidationIgnoredOptions.Add(c); - hosted = new HostedInstanceKeeper(_args); + hosted = new HostedInstanceKeeper(applicationSettings, _args); } catch (Exception ex) { @@ -164,17 +167,18 @@ namespace Duplicati.GUI.TrayIcon // We have a hosted server, if this is the first run, // we should open the main page - openui = Server.Program.IsFirstRun || Server.Program.ServerPortChanged; + var connection = Server.Program.DuplicatiWebserver.Provider.GetRequiredService(); + openui = connection.ApplicationSettings.IsFirstRun || connection.ApplicationSettings.ServerPortChanged; - var scheme = Server.Program.DataConnection.ApplicationSettings.UseHTTPS ? "https" : "http"; + var scheme = connection.ApplicationSettings.UseHTTPS ? "https" : "http"; serverURL = new UriBuilder(serverURL) { - Port = Server.Program.ServerPort, + Port = Server.Program.DuplicatiWebserver.Port, Scheme = scheme }.Uri; - if (Server.Program.DataConnection.ApplicationSettings.UseHTTPS && string.IsNullOrWhiteSpace(acceptedHostCertificate)) - acceptedHostCertificate = Server.Program.DataConnection.ApplicationSettings.ServerSSLCertificate?.FirstOrDefault(x => x.HasPrivateKey)?.GetCertHashString(); + if (connection.ApplicationSettings.UseHTTPS && string.IsNullOrWhiteSpace(acceptedHostCertificate)) + acceptedHostCertificate = connection.ApplicationSettings.ServerSSLCertificate?.FirstOrDefault(x => x.HasPrivateKey)?.GetCertHashString(); } else if (Utility.ParseBoolOption(options, READCONFIGFROMDB_OPTION)) @@ -182,7 +186,7 @@ namespace Duplicati.GUI.TrayIcon if (File.Exists(Path.Combine(DataFolderManager.GetDataFolder(DataFolderManager.AccessMode.ReadWritePermissionSet), DataFolderManager.SERVER_DATABASE_FILENAME))) { passwordSource = PasswordSource.Database; - databaseConnection = Server.Program.GetDatabaseConnection(options, true); + databaseConnection = Server.Program.GetDatabaseConnection(new ApplicationSettings(), options, true); if (databaseConnection != null) { @@ -267,8 +271,9 @@ No password provided, unable to connect to server, exiting"); } tk.ShowUrlInWindow(t.Result); - Server.Program.IsFirstRun = false; - Server.Program.ServerPortChanged = false; + var connection = Server.Program.DuplicatiWebserver.Provider.GetRequiredService(); + connection.ApplicationSettings.IsFirstRun = false; + connection.ApplicationSettings.ServerPortChanged = false; }); } diff --git a/Duplicati/Library/RestAPI/Abstractions/IApplicationSettings.cs b/Duplicati/Library/RestAPI/Abstractions/IApplicationSettings.cs new file mode 100644 index 000000000..67282546b --- /dev/null +++ b/Duplicati/Library/RestAPI/Abstractions/IApplicationSettings.cs @@ -0,0 +1,64 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +#nullable enable + +using System; +using System.Threading; +using Duplicati.Library.Interface; + +namespace Duplicati.WebserverCore.Abstractions; + +/// +/// Interface for application-wide settings for the server. +/// +public interface IApplicationSettings +{ + /// + /// Action to start or stop the usage reporter + /// + Action? StartOrStopUsageReporter { get; set; } + + /// + /// Gets the folder where Duplicati data is stored + /// + string DataFolder { get; } + + /// + /// Used to check the origin of the web server (e.g. Tray icon or a stand alone Server) + /// + string Origin { get; set; } + + /// + /// The application exit event + /// + ManualResetEvent ApplicationExitEvent { get; } + + /// + /// The shared secret provider from the server invocation + /// + ISecretProvider? SecretProvider { get; set; } + + /// + /// Flag to indicate if the settings encryption key was provided externally + /// + bool SettingsEncryptionKeyProvidedExternally { get; set; } +} + diff --git a/Duplicati/Library/RestAPI/Database/Connection.cs b/Duplicati/Library/RestAPI/Database/Connection.cs index 2c4a8a91b..421031e14 100644 --- a/Duplicati/Library/RestAPI/Database/Connection.cs +++ b/Duplicati/Library/RestAPI/Database/Connection.cs @@ -40,6 +40,11 @@ namespace Duplicati.Server.Database { public class Connection : IDisposable { + /// + /// The placeholder for passwords in the UI + /// + public const string PASSWORD_PLACEHOLDER = "**********"; + private readonly IDbConnection m_connection; private readonly IDbCommand m_errorcmd; public readonly object m_lock = new object(); @@ -51,6 +56,7 @@ namespace Duplicati.Server.Database private IServiceProvider? m_serviceProvider; private INotificationUpdateService? m_notificationUpdateService; private EventPollNotify? m_eventPollNotifyer; + private readonly string m_dataFolder; private static readonly HashSet _encryptedFields = BackendLoader.Backends.SelectMany(x => x.SupportedCommands ?? []) @@ -71,14 +77,15 @@ namespace Duplicati.Server.Database ]) .ToHashSet(StringComparer.OrdinalIgnoreCase); - public Connection(IDbConnection connection, bool disableFieldEncryption, EncryptedFieldHelper.KeyInstance? key) + public Connection(IDbConnection connection, bool disableFieldEncryption, EncryptedFieldHelper.KeyInstance? key, string dataFolder, Action startOrStopUsageReporter) { + m_dataFolder = dataFolder; m_encryptSensitiveFields = !disableFieldEncryption; m_key = key; m_connection = connection; m_errorcmd = m_connection.CreateCommand(@"INSERT INTO ""ErrorLog"" (""BackupID"", ""Message"", ""Exception"", ""Timestamp"") VALUES (@BackupId,@Message,@Exception,@Timestamp)"); - this.ApplicationSettings = new ServerSettings(this); + this.ApplicationSettings = new ServerSettings(this, startOrStopUsageReporter); } /// @@ -331,7 +338,7 @@ namespace Duplicati.Server.Database cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Option"" (""BackupID"", ""Filter"", ""Name"", ""Value"") VALUES (@BackupId, @Filter, @Name, @Value)"), (cmd, f) => { - if (FIXMEGlobal.PASSWORD_PLACEHOLDER.Equals(f.Value)) + if (PASSWORD_PLACEHOLDER.Equals(f.Value)) throw new Exception("Attempted to save a property with the placeholder password"); cmd.SetParameterValue("@BackupId", id) @@ -640,7 +647,7 @@ namespace Duplicati.Server.Database bool update = item.ID != null; if (!update && item.DBPath == null) { - var folder = FIXMEGlobal.DataFolder; + var folder = m_dataFolder; if (!System.IO.Directory.Exists(folder)) System.IO.Directory.CreateDirectory(folder); @@ -673,7 +680,7 @@ namespace Duplicati.Server.Database }, (cmd, n) => { - if (n.TargetURL.IndexOf(FIXMEGlobal.PASSWORD_PLACEHOLDER, StringComparison.Ordinal) >= 0) + if (n.TargetURL.IndexOf(PASSWORD_PLACEHOLDER, StringComparison.Ordinal) >= 0) throw new Exception("Attempted to save a backup with the password placeholder"); if (update && long.Parse(n.ID) <= 0) throw new Exception("Invalid update, cannot update application settings through update method"); diff --git a/Duplicati/Library/RestAPI/Database/ServerSettings.cs b/Duplicati/Library/RestAPI/Database/ServerSettings.cs index c1fc38b30..94f28764c 100644 --- a/Duplicati/Library/RestAPI/Database/ServerSettings.cs +++ b/Duplicati/Library/RestAPI/Database/ServerSettings.cs @@ -80,10 +80,12 @@ namespace Duplicati.Server.Database private readonly Dictionary settings; private readonly Connection databaseConnection; - private UpdateInfo? m_latestUpdate; + private UpdateInfo? latestUpdate; + private readonly Action? startOrStopUsageReporter; - internal ServerSettings(Connection con) + internal ServerSettings(Connection con, Action startOrStopUsageReporter) { + this.startOrStopUsageReporter = startOrStopUsageReporter; settings = new Dictionary(); databaseConnection = con; ReloadSettings(); @@ -115,7 +117,7 @@ namespace Duplicati.Server.Database lock (databaseConnection.m_lock) { - m_latestUpdate = null; + latestUpdate = null; if (clearExisting) settings.Clear(); @@ -154,8 +156,7 @@ namespace Duplicati.Server.Database } // In case the usage reporter is enabled or disabled, refresh now - if (FIXMEGlobal.StartOrStopUsageReporter != null) - FIXMEGlobal.StartOrStopUsageReporter(); + startOrStopUsageReporter?.Invoke(); } public string? StartupDelayDuration @@ -634,11 +635,11 @@ namespace Duplicati.Server.Database try { - if (m_latestUpdate != null) - return m_latestUpdate; + if (latestUpdate != null) + return latestUpdate; using (var tr = new System.IO.StringReader(updateNew)) - return m_latestUpdate = Server.Serialization.Serializer.Deserialize(tr); + return latestUpdate = Server.Serialization.Serializer.Deserialize(tr); } catch { @@ -658,7 +659,7 @@ namespace Duplicati.Server.Database result = sb.ToString(); } - m_latestUpdate = value; + latestUpdate = value; lock (databaseConnection.m_lock) settings[CONST.UPDATE_CHECK_NEW_VERSION] = result; diff --git a/Duplicati/Library/RestAPI/FIXMEGlobal.cs b/Duplicati/Library/RestAPI/FIXMEGlobal.cs deleted file mode 100644 index 882e50dd3..000000000 --- a/Duplicati/Library/RestAPI/FIXMEGlobal.cs +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -using Duplicati.Server; -using System; -using Duplicati.Library.Interface; - -namespace Duplicati.Library.RestAPI -{ - /** - * In the absense of dependancy injection, there is a significant amount of variables exposed through Program as globals. - * This causes a problem decoupling classes and leads to circular dependancies. - */ - public static class FIXMEGlobal - { - /// - /// The placeholder for passwords in the UI - /// - public const string PASSWORD_PLACEHOLDER = "**********"; - - public static Action StartOrStopUsageReporter; - - /// - /// Gets the folder where Duplicati data is stored - /// - public static string DataFolder; - - /// - /// Used to check the origin of the web server (e.g. Tray icon or a stand alone Server) - /// - public static string Origin = "Server"; - - /// - /// The application exit event - /// - public static System.Threading.ManualResetEvent ApplicationExitEvent; - - /// - /// The shared secret provider from the server invocation - /// - public static ISecretProvider SecretProvider { get; set; } - /// - /// Flag to indicate if the settings encryption key was provided externally - /// - public static bool SettingsEncryptionKeyProvidedExternally { get; set; } - - } -} diff --git a/Duplicati/Library/RestAPI/Runner.cs b/Duplicati/Library/RestAPI/Runner.cs index 15ccc4898..bd9e0df2e 100644 --- a/Duplicati/Library/RestAPI/Runner.cs +++ b/Duplicati/Library/RestAPI/Runner.cs @@ -497,15 +497,15 @@ namespace Duplicati.Server return parts.ToArray(); } - public static IBasicResults? Run(Connection databaseConnection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IProgressStateProviderService progressStateProviderService, IQueuedTask data, bool fromQueue) + public static IBasicResults? Run(Connection databaseConnection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IProgressStateProviderService progressStateProviderService, IApplicationSettings applicationSettings, IQueuedTask data, bool fromQueue) { if (data is IRunnerData runnerData) - return RunInternal(databaseConnection, eventPollNotify, notificationUpdateService, progressStateProviderService, runnerData, fromQueue); + return RunInternal(databaseConnection, eventPollNotify, notificationUpdateService, progressStateProviderService, applicationSettings, runnerData, fromQueue); throw new ArgumentException("Invalid task type", nameof(data)); } - private static IBasicResults? RunInternal(Connection databaseConnection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IProgressStateProviderService progressStateProviderService, IRunnerData data, bool fromQueue) + private static IBasicResults? RunInternal(Connection databaseConnection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IProgressStateProviderService progressStateProviderService, IApplicationSettings applicationSettings, IRunnerData data, bool fromQueue) { data.TaskStarted = DateTime.Now; if (data is CustomRunnerTask task) @@ -586,7 +586,7 @@ namespace Duplicati.Server data.UpdateThrottleSpeeds(appSettings.UploadSpeedLimit, appSettings.DownloadSpeedLimit); // Pass on the provider, will be replaced if configured in the backup - controller.SetSecretProvider(FIXMEGlobal.SecretProvider); + controller.SetSecretProvider(applicationSettings.SecretProvider); if (backup.Metadata.ContainsKey("LastCompactFinished")) controller.LastCompact = Utility.DeserializeDateTime(backup.Metadata["LastCompactFinished"]); diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index b4349c2ab..d9505180b 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -32,11 +32,11 @@ using Duplicati.Library.Interface; using Duplicati.Library.Logging; using Duplicati.Library.Main; using Duplicati.Library.Main.Database; -using Duplicati.Library.RestAPI; using Duplicati.Library.Utility; using Duplicati.Server.Database; using Duplicati.WebserverCore; using Duplicati.WebserverCore.Abstractions; +using Duplicati.WebserverCore.Services; using Microsoft.Extensions.DependencyInjection; namespace Duplicati.Server @@ -84,41 +84,21 @@ namespace Duplicati.Server /// private static readonly string ENV_NAME_PREFIX = AutoUpdateSettings.AppName.ToUpperInvariant(); - /// - /// Gets the folder where Duplicati data is stored - /// - public static string DataFolder { get => FIXMEGlobal.DataFolder; private set => FIXMEGlobal.DataFolder = value; } - /// /// The single instance /// public static SingleInstance ApplicationInstance = null; - /// - /// This is the only access to the database - /// - public static Database.Connection DataConnection { get; set; } - /// /// The thread running the ping-pong handler /// - private static System.Threading.Thread PingPongThread; - - /// - /// The path to the file that contains the current database - /// - private static string DatabasePath; + private static Thread PingPongThread; /// /// The controller interface for pause/resume and throttle options /// public static LiveControls LiveControl { get => DuplicatiWebserver.Provider.GetRequiredService(); } - /// - /// The application exit event - /// - public static System.Threading.ManualResetEvent ApplicationExitEvent { get => FIXMEGlobal.ApplicationExitEvent; set => FIXMEGlobal.ApplicationExitEvent = value; } - /// /// Duplicati webserver instance /// @@ -137,43 +117,33 @@ namespace Duplicati.Server /// public static readonly ManualResetEvent ServerStartedEvent = new ManualResetEvent(false); + /// + /// Timer for purging temp files and log data + /// private static System.Threading.Timer PurgeTempFilesTimer = null; - public static int ServerPort - { - get - { - return DuplicatiWebserver.Port; - } - } - - public static bool IsFirstRun - { - get { return DataConnection.ApplicationSettings.IsFirstRun; } - set { DataConnection.ApplicationSettings.IsFirstRun = value; } - } - - public static bool ServerPortChanged - { - get { return DataConnection.ApplicationSettings.ServerPortChanged; } - set { DataConnection.ApplicationSettings.ServerPortChanged = value; } - } - - static Program() - { - FIXMEGlobal.StartOrStopUsageReporter = Program.StartOrStopUsageReporter; - } - /// /// The main entry point for the application. /// + /// The command line arguments [STAThread] public static int Main(string[] _args) + => Main(null, _args); + + /// + /// The main entry point for the application. + /// + /// The application settings + /// The command line arguments + [STAThread] + public static int Main(IApplicationSettings applicationSettings, string[] _args) { PreloadSettingsLoader.ConfigurePreloadSettings(ref _args, PackageHelper.NamedExecutable.Server, out var preloadDbSettings); + applicationSettings ??= new ApplicationSettings(); + //If this executable is invoked directly, write to console, otherwise throw exceptions - var writeToConsoleOnException = FIXMEGlobal.Origin == "Server"; + var writeToConsoleOnException = applicationSettings.Origin == "Server"; // Prepared for the future, where we might want to have a silent console mode var silentConsole = false; @@ -198,7 +168,7 @@ namespace Duplicati.Server SystemContextSettings.StartSession(); ApplyEnvironmentVariables(commandlineOptions); - ApplySecretProvider(commandlineOptions, CancellationToken.None).Await(); + ApplySecretProvider(applicationSettings, commandlineOptions, CancellationToken.None).Await(); var parameterFileOption = PARAMETERS_FILE_OPTION_EXTRAS.Prepend(PARAMETERS_FILE_OPTION) .FirstOrDefault(x => commandlineOptions.ContainsKey(x)); @@ -225,28 +195,27 @@ namespace Duplicati.Server ISchedulerService scheduler = null; try { - DataConnection = GetDatabaseConnection(commandlineOptions, silentConsole); + var connection = GetDatabaseConnection(applicationSettings, commandlineOptions, silentConsole); - if (!DataConnection.ApplicationSettings.FixedInvalidBackupId) - DataConnection.FixInvalidBackupId(); + if (!connection.ApplicationSettings.FixedInvalidBackupId) + connection.FixInvalidBackupId(); - DataConnection.ApplicationSettings.UpgradePasswordToKBDF(); - CreateApplicationInstance(writeToConsoleOnException); + connection.ApplicationSettings.UpgradePasswordToKBDF(); + CreateApplicationInstance(applicationSettings.DataFolder, writeToConsoleOnException); - StartOrStopUsageReporter(); + applicationSettings.StartOrStopUsageReporter = () => StartOrStopUsageReporter(connection); + applicationSettings.StartOrStopUsageReporter?.Invoke(); - AdjustApplicationSettings(commandlineOptions); - - ApplicationExitEvent = new System.Threading.ManualResetEvent(false); + AdjustApplicationSettings(connection, commandlineOptions); UpdaterManager.OnError += obj => { - DataConnection.LogError(null, "Error in updater", obj); + connection.LogError(null, "Error in updater", obj); }; - DuplicatiWebserver = StartWebServer(commandlineOptions, DataConnection, logHandler).Await(); + DuplicatiWebserver = StartWebServer(commandlineOptions, connection, logHandler, applicationSettings).Await(); - DataConnection.SetServiceProvider(DuplicatiWebserver.Provider); + connection.SetServiceProvider(DuplicatiWebserver.Provider); queueRunner = DuplicatiWebserver.Provider.GetRequiredService(); updatePollThread = DuplicatiWebserver.Provider.GetRequiredService(); eventPollNotify = DuplicatiWebserver.Provider.GetRequiredService(); @@ -254,28 +223,28 @@ namespace Duplicati.Server updatePollThread.Init(Library.Utility.Utility.ParseBoolOption(commandlineOptions, DISABLE_UPDATE_CHECK_OPTION)); - SetPurgeTempFilesTimer(commandlineOptions); + SetPurgeTempFilesTimer(connection, commandlineOptions); - LiveControl.StateChanged = (e) => { LiveControl_StateChanged(queueRunner, DataConnection, eventPollNotify, e); }; + LiveControl.StateChanged = (e) => { LiveControl_StateChanged(queueRunner, connection, eventPollNotify, e); }; if (Library.Utility.Utility.ParseBoolOption(commandlineOptions, PING_PONG_KEEPALIVE_OPTION)) { - PingPongThread = new System.Threading.Thread(PingPongMethod) { IsBackground = true }; + PingPongThread = new Thread(() => PingPongMethod(applicationSettings)) { IsBackground = true }; PingPongThread.Start(); } - DataConnection.ReWriteAllFieldsIfEncryptionChanged(); - DataConnection.SetPreloadSettingsIfChanged(preloadDbSettings); - EmitWarningsForConfigurationIssues(commandlineOptions); + connection.ReWriteAllFieldsIfEncryptionChanged(); + connection.SetPreloadSettingsIfChanged(preloadDbSettings); + EmitWarningsForConfigurationIssues(connection, applicationSettings, commandlineOptions); Log.WriteInformationMessage(LOGTAG, "ServerStarted", Strings.Program.ServerStarted(DuplicatiWebserver.Interface, DuplicatiWebserver.Port)); logMessageToConsole(Strings.Program.ServerStarted(DuplicatiWebserver.Interface, DuplicatiWebserver.Port)); - if (FIXMEGlobal.Origin == "Server" && DataConnection.ApplicationSettings.AutogeneratedPassphrase) + if (applicationSettings.Origin == "Server" && connection.ApplicationSettings.AutogeneratedPassphrase) { var signinToken = DuplicatiWebserver.Provider.GetRequiredService().CreateSigninToken("server-cli"); - var hostname = (DataConnection.ApplicationSettings.AllowedHostnames ?? string.Empty).Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries).FirstOrDefault(x => x != "*") ?? "localhost"; - var scheme = DataConnection.ApplicationSettings.UseHTTPS ? "https" : "http"; + var hostname = (connection.ApplicationSettings.AllowedHostnames ?? string.Empty).Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries).FirstOrDefault(x => x != "*") ?? "localhost"; + var scheme = connection.ApplicationSettings.UseHTTPS ? "https" : "http"; var url = $"{scheme}://{hostname}:{DuplicatiWebserver.Port}/signin.html?token={signinToken}"; Log.WriteWarningMessage(LOGTAG, "ServerStartedSignin", null, Strings.Program.ServerStartedSignin(url)); @@ -291,7 +260,7 @@ namespace Duplicati.Server } terminated = true; - ApplicationExitEvent.Set(); + applicationSettings.ApplicationExitEvent.Set(); }); var stopCounter = 0; @@ -313,7 +282,7 @@ namespace Duplicati.Server }; ServerStartedEvent.Set(); - ApplicationExitEvent.WaitOne(); + applicationSettings.ApplicationExitEvent.WaitOne(); } catch (SingleInstance.MultipleInstanceException mex) { @@ -381,7 +350,7 @@ namespace Duplicati.Server return 0; } - private static async Task StartWebServer(IReadOnlyDictionary options, Connection connection, ILogWriteHandler logWriteHandler) + private static async Task StartWebServer(IReadOnlyDictionary options, Connection connection, ILogWriteHandler logWriteHandler, IApplicationSettings applicationSettings) { var server = await WebServerLoader.TryRunServer(options, connection, async parsedOptions => { @@ -397,7 +366,7 @@ namespace Duplicati.Server parsedOptions.PreAuthTokens ); - var server = DuplicatiWebserver.CreateWebServer(mappedSettings, connection, logWriteHandler); + var server = DuplicatiWebserver.CreateWebServer(mappedSettings, connection, logWriteHandler, applicationSettings); // Start the server, but catch any configuration issues var task = server.Start(); @@ -408,13 +377,13 @@ namespace Duplicati.Server return server; }).ConfigureAwait(false); - ServerPortChanged |= server.Port != DataConnection.ApplicationSettings.LastWebserverPort; - DataConnection.ApplicationSettings.LastWebserverPort = server.Port; + connection.ApplicationSettings.ServerPortChanged |= server.Port != connection.ApplicationSettings.LastWebserverPort; + connection.ApplicationSettings.LastWebserverPort = server.Port; return server; } - private static void SetPurgeTempFilesTimer(Dictionary commandlineOptions) + private static void SetPurgeTempFilesTimer(Connection connection, Dictionary commandlineOptions) { var lastPurge = new DateTime(0); @@ -429,7 +398,7 @@ namespace Duplicati.Server lastPurge = DateTime.Now; - foreach (var e in DataConnection.GetTempFiles().Where((f) => f.Expires < DateTime.Now)) + foreach (var e in connection.GetTempFiles().Where((f) => f.Expires < DateTime.Now)) { try { @@ -438,16 +407,16 @@ namespace Duplicati.Server } catch (Exception ex) { - DataConnection.LogError(null, $"Failed to delete temp file: {e.Path}", ex); + connection.LogError(null, $"Failed to delete temp file: {e.Path}", ex); } - DataConnection.DeleteTempFile(e.ID); + connection.DeleteTempFile(e.ID); } Library.Utility.TempFile.RemoveOldApplicationTempFiles((path, ex) => { - DataConnection.LogError(null, $"Failed to delete temp file: {path}", ex); + connection.LogError(null, $"Failed to delete temp file: {path}", ex); }); if (!commandlineOptions.TryGetValue("log-retention", out string pts)) @@ -455,11 +424,11 @@ namespace Duplicati.Server pts = DEFAULT_LOG_RETENTION; } - DataConnection.PurgeLogData(Library.Utility.Timeparser.ParseTimeInterval(pts, DateTime.Now, true)); + connection.PurgeLogData(Library.Utility.Timeparser.ParseTimeInterval(pts, DateTime.Now, true)); } catch (Exception ex) { - DataConnection.LogError(null, "Failed during temp file cleanup", ex); + connection.LogError(null, "Failed during temp file cleanup", ex); } }; @@ -469,37 +438,37 @@ namespace Duplicati.Server DEBUG_MODE ? TimeSpan.FromHours(1) : TimeSpan.FromDays(1)); } - private static void AdjustApplicationSettings(Dictionary commandlineOptions) + private static void AdjustApplicationSettings(Connection connection, Dictionary commandlineOptions) { // This clears the JWT config, and a new will be generated, invalidating all existing tokens if (Library.Utility.Utility.ParseBoolOption(commandlineOptions, WebServerLoader.OPTION_WEBSERVICE_RESET_JWT_CONFIG)) { - DataConnection.ApplicationSettings.JWTConfig = null; + connection.ApplicationSettings.JWTConfig = null; // Clean up stored tokens as they are now invalid - DataConnection.ExecuteWithCommand((con) => con.ExecuteNonQuery("DELETE FROM TokenFamily")); + connection.ExecuteWithCommand((con) => con.ExecuteNonQuery("DELETE FROM TokenFamily")); } if (Library.Utility.Utility.ParseBoolOption(commandlineOptions, WebServerLoader.OPTION_WEBSERVICE_ENABLE_FOREVER_TOKEN)) - DataConnection.ApplicationSettings.EnableForeverTokens(); + connection.ApplicationSettings.EnableForeverTokens(); if (commandlineOptions.ContainsKey(WebServerLoader.OPTION_WEBSERVICE_DISABLE_VISUAL_CAPTCHA)) - DataConnection.ApplicationSettings.DisableVisualCaptcha = Library.Utility.Utility.ParseBool(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_DISABLE_VISUAL_CAPTCHA], true); + connection.ApplicationSettings.DisableVisualCaptcha = Library.Utility.Utility.ParseBool(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_DISABLE_VISUAL_CAPTCHA], true); if (commandlineOptions.ContainsKey(WebServerLoader.OPTION_WEBSERVICE_DISABLE_SIGNIN_TOKENS)) - DataConnection.ApplicationSettings.DisableSigninTokens = Library.Utility.Utility.ParseBool(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_DISABLE_SIGNIN_TOKENS], true); + connection.ApplicationSettings.DisableSigninTokens = Library.Utility.Utility.ParseBool(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_DISABLE_SIGNIN_TOKENS], true); if (commandlineOptions.ContainsKey(WebServerLoader.OPTION_WEBSERVICE_PASSWORD)) - DataConnection.ApplicationSettings.SetWebserverPassword(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_PASSWORD]); + connection.ApplicationSettings.SetWebserverPassword(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_PASSWORD]); if (commandlineOptions.ContainsKey(WebServerLoader.OPTION_WEBSERVICE_ALLOWEDHOSTNAMES)) - DataConnection.ApplicationSettings.SetAllowedHostnames(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_ALLOWEDHOSTNAMES]); + connection.ApplicationSettings.SetAllowedHostnames(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_ALLOWEDHOSTNAMES]); else if (commandlineOptions.ContainsKey(WebServerLoader.OPTION_WEBSERVICE_ALLOWEDHOSTNAMES_ALT)) - DataConnection.ApplicationSettings.SetAllowedHostnames(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_ALLOWEDHOSTNAMES_ALT]); + connection.ApplicationSettings.SetAllowedHostnames(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_ALLOWEDHOSTNAMES_ALT]); if (commandlineOptions.ContainsKey(WebServerLoader.OPTION_WEBSERVICE_TIMEZONE) && !string.IsNullOrEmpty(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_TIMEZONE])) try { - DataConnection.ApplicationSettings.Timezone = TimeZoneHelper.FindTimeZone(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_TIMEZONE]); + connection.ApplicationSettings.Timezone = TimeZoneHelper.FindTimeZone(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_TIMEZONE]); } catch (Exception ex) { @@ -507,33 +476,33 @@ namespace Duplicati.Server } // The database has recorded a new version - if (DataConnection.ApplicationSettings.UpdatedVersion != null) + if (connection.ApplicationSettings.UpdatedVersion != null) { // Check if the running version is newer than the recorded version - if (UpdaterManager.TryParseVersion(DataConnection.ApplicationSettings.UpdatedVersion.Version) <= UpdaterManager.TryParseVersion(UpdaterManager.SelfVersion.Version)) + if (UpdaterManager.TryParseVersion(connection.ApplicationSettings.UpdatedVersion.Version) <= UpdaterManager.TryParseVersion(UpdaterManager.SelfVersion.Version)) { // Clean up lingering update notifications - var updateNotifications = DataConnection.GetNotifications().Where(x => x.Action == "update:new").ToList(); + var updateNotifications = connection.GetNotifications().Where(x => x.Action == "update:new").ToList(); foreach (var n in updateNotifications) - DataConnection.DismissNotification(n.ID); + connection.DismissNotification(n.ID); // Clear up the recorded version - DataConnection.ApplicationSettings.UpdatedVersion = null; + connection.ApplicationSettings.UpdatedVersion = null; } } } - private static void EmitWarningsForConfigurationIssues(Dictionary commandlineOptions) + private static void EmitWarningsForConfigurationIssues(Connection connection, IApplicationSettings applicationSettings, Dictionary commandlineOptions) { - if (DataConnection.ApplicationSettings.LastConfigIssueCheckVersion != UpdaterManager.SelfVersion.Version) + if (connection.ApplicationSettings.LastConfigIssueCheckVersion != UpdaterManager.SelfVersion.Version) { - var updateNotifications = DataConnection.GetNotifications().Where(x => x.Action.StartsWith("config:issue:")).ToList(); + var updateNotifications = connection.GetNotifications().Where(x => x.Action.StartsWith("config:issue:")).ToList(); foreach (var n in updateNotifications) - DataConnection.DismissNotification(n.ID); + connection.DismissNotification(n.ID); - if (!DataConnection.IsEncryptingFields && !Library.Utility.Utility.ParseBoolOption(commandlineOptions, DISABLE_DB_ENCRYPTION_OPTION)) + if (!connection.IsEncryptingFields && !Library.Utility.Utility.ParseBoolOption(commandlineOptions, DISABLE_DB_ENCRYPTION_OPTION)) { - DataConnection.RegisterNotification( + connection.RegisterNotification( Serialization.NotificationType.Warning, "Unencrypted database", "The database is not encrypted. This is a security risk and should be fixed as soon as possible.", @@ -550,9 +519,9 @@ namespace Duplicati.Server ); } - if (OperatingSystem.IsWindows() && DataFolder.StartsWith(Util.AppendDirSeparator(Environment.GetFolderPath(Environment.SpecialFolder.Windows)), StringComparison.OrdinalIgnoreCase)) + if (OperatingSystem.IsWindows() && applicationSettings.DataFolder.StartsWith(Util.AppendDirSeparator(Environment.GetFolderPath(Environment.SpecialFolder.Windows)), StringComparison.OrdinalIgnoreCase)) { - DataConnection.RegisterNotification( + connection.RegisterNotification( Serialization.NotificationType.Warning, "Incorrect storage folder", "The server configuraion is stored inside the Windows folder. Please move the configuration to a different location, or it may be deleted on Windows version upgrades.", @@ -569,16 +538,16 @@ namespace Duplicati.Server ); } - DataConnection.ApplicationSettings.LastConfigIssueCheckVersion = UpdaterManager.SelfVersion.Version; + connection.ApplicationSettings.LastConfigIssueCheckVersion = UpdaterManager.SelfVersion.Version; } } - private static void CreateApplicationInstance(bool writeToConsoleOnExceptionw) + private static void CreateApplicationInstance(string dataFolder, bool writeToConsoleOnExceptionw) { try { //This will also create DATAFOLDER if it does not exist - ApplicationInstance = new SingleInstance(DataFolder); + ApplicationInstance = new SingleInstance(dataFolder); } catch (Exception ex) { @@ -622,8 +591,8 @@ namespace Duplicati.Server commandlineOptions[SETTINGS_ENCRYPTION_KEY_OPTION] = Environment.GetEnvironmentVariable(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME); } - private static async Task ApplySecretProvider(Dictionary commandlineOptions, CancellationToken cancellationToken) - => FIXMEGlobal.SecretProvider = await SecretProviderHelper.ApplySecretProviderAsync([], [], commandlineOptions, TempFolder.SystemTempPath, FIXMEGlobal.SecretProvider, cancellationToken).ConfigureAwait(false); + private static async Task ApplySecretProvider(IApplicationSettings applicationSettings, Dictionary commandlineOptions, CancellationToken cancellationToken) + => applicationSettings.SecretProvider = await SecretProviderHelper.ApplySecretProviderAsync([], [], commandlineOptions, TempFolder.SystemTempPath, applicationSettings.SecretProvider, cancellationToken).ConfigureAwait(false); private class ConsoleLogDestination(LogMessageType level) : ILogDestination { @@ -718,15 +687,13 @@ namespace Duplicati.Server throw new Exception("Server invoked with --help"); } - public static Connection GetDatabaseConnection(Dictionary commandlineOptions, bool silentConsole) + public static Connection GetDatabaseConnection(IApplicationSettings applicationSettings, Dictionary commandlineOptions, bool silentConsole) { - DataFolder = DataFolderManager.GetDataFolder(DataFolderManager.AccessMode.ReadWritePermissionSet); - // Emit a warning if the database is stored in the Windows folder - if (Util.IsPathUnderWindowsFolder(DataFolder)) + if (Util.IsPathUnderWindowsFolder(applicationSettings.DataFolder)) Log.WriteWarningMessage(LOGTAG, "DatabaseInWindowsFolder", null, "The database is stored in the Windows folder, this is not recommended as it will be deleted on Windows upgrades."); - CrashlogHelper.DefaultLogDir = DataFolder; + CrashlogHelper.DefaultLogDir = applicationSettings.DataFolder; var sqliteVersion = new Version(Duplicati.Library.SQLiteHelper.SQLiteLoader.SQLiteVersion); if (sqliteVersion < new Version(3, 6, 3)) @@ -740,15 +707,15 @@ namespace Duplicati.Server try { - DatabasePath = System.IO.Path.Combine(DataFolder, DataFolderManager.SERVER_DATABASE_FILENAME); + var databasePath = System.IO.Path.Combine(applicationSettings.DataFolder, DataFolderManager.SERVER_DATABASE_FILENAME); - if (!System.IO.Directory.Exists(System.IO.Path.GetDirectoryName(DatabasePath))) - System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(DatabasePath)); + if (!System.IO.Directory.Exists(System.IO.Path.GetDirectoryName(databasePath))) + System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(databasePath)); // Attempt to open the database, removing any encryption present - Library.SQLiteHelper.SQLiteLoader.OpenDatabase(con, DatabasePath, Library.SQLiteHelper.SQLiteRC4Decrypter.GetEncryptionPassword(commandlineOptions)); + Library.SQLiteHelper.SQLiteLoader.OpenDatabase(con, databasePath, Library.SQLiteHelper.SQLiteRC4Decrypter.GetEncryptionPassword(commandlineOptions)); - Library.SQLiteHelper.DatabaseUpgrader.UpgradeDatabase(con, DatabasePath, typeof(Library.RestAPI.Database.DatabaseSchemaMarker)); + Library.SQLiteHelper.DatabaseUpgrader.UpgradeDatabase(con, databasePath, typeof(Library.RestAPI.Database.DatabaseSchemaMarker)); } catch (Exception ex) { @@ -765,7 +732,7 @@ namespace Duplicati.Server var usingBlacklistedKey = encKey?.IsBlacklisted ?? false; var hasValidEncryptionKey = encKey != null; - FIXMEGlobal.SettingsEncryptionKeyProvidedExternally = hasValidEncryptionKey; + applicationSettings.SettingsEncryptionKeyProvidedExternally = hasValidEncryptionKey; if (requireDbEncryptionKey && !(hasValidEncryptionKey || disableDbEncryption)) throw new UserInformationException(Strings.Program.DatabaseEncryptionKeyRequired(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION), "RequireDbEncryptionKey"); @@ -782,7 +749,7 @@ namespace Duplicati.Server if (hasEncryptedFields) { - Library.Logging.Log.WriteWarningMessage(LOGTAG, "EncryptionKeyMissing", null, Strings.Program.EncryptionKeyMissing(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME)); + Log.WriteWarningMessage(LOGTAG, "EncryptionKeyMissing", null, Strings.Program.EncryptionKeyMissing(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME)); if (!silentConsole) Console.WriteLine(Strings.Program.EncryptionKeyMissing(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME)); } @@ -797,30 +764,30 @@ namespace Duplicati.Server if (!hasValidEncryptionKey && !disableDbEncryption) { disableDbEncryption = true; - Log.WriteWarningMessage(LOGTAG, "MissingEncryptionKey", null, Strings.Program.NoEncryptionKeySpecified(Library.Encryption.EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION)); + Log.WriteWarningMessage(LOGTAG, "MissingEncryptionKey", null, Strings.Program.NoEncryptionKeySpecified(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION)); if (!silentConsole) - Console.WriteLine(Strings.Program.NoEncryptionKeySpecified(Library.Encryption.EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION)); + Console.WriteLine(Strings.Program.NoEncryptionKeySpecified(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION)); } if (usingBlacklistedKey && !disableDbEncryption) { disableDbEncryption = true; - Log.WriteErrorMessage(LOGTAG, "BlacklistedEncryptionKey", null, Strings.Program.BlacklistedEncryptionKey(Library.Encryption.EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION)); + Log.WriteErrorMessage(LOGTAG, "BlacklistedEncryptionKey", null, Strings.Program.BlacklistedEncryptionKey(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION)); if (!silentConsole) - Console.WriteLine(Strings.Program.BlacklistedEncryptionKey(Library.Encryption.EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION)); + Console.WriteLine(Strings.Program.BlacklistedEncryptionKey(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION)); } - return new Connection(con, disableDbEncryption, encKey); + return new Connection(con, disableDbEncryption, encKey, applicationSettings.DataFolder, applicationSettings.StartOrStopUsageReporter); } - public static void StartOrStopUsageReporter() + private static void StartOrStopUsageReporter(Connection connection) { var disableUsageReporter = - string.Equals(DataConnection.ApplicationSettings.UsageReporterLevel, "none", StringComparison.OrdinalIgnoreCase) + string.Equals(connection.ApplicationSettings.UsageReporterLevel, "none", StringComparison.OrdinalIgnoreCase) || - string.Equals(DataConnection.ApplicationSettings.UsageReporterLevel, "disabled", StringComparison.OrdinalIgnoreCase); + string.Equals(connection.ApplicationSettings.UsageReporterLevel, "disabled", StringComparison.OrdinalIgnoreCase); - if (!Enum.TryParse(DataConnection.ApplicationSettings.UsageReporterLevel, true, out var reportLevel)) + if (!Enum.TryParse(connection.ApplicationSettings.UsageReporterLevel, true, out var reportLevel)) Library.UsageReporter.Reporter.SetReportLevel(null, disableUsageReporter); else Library.UsageReporter.Reporter.SetReportLevel(reportLevel, disableUsageReporter); @@ -861,7 +828,8 @@ namespace Duplicati.Server /// /// Simple method for tracking if the server has crashed /// - private static void PingPongMethod() + /// The application settings + private static void PingPongMethod(IApplicationSettings applicationSettings) { var rd = new System.IO.StreamReader(Console.OpenStandardInput()); var wr = new System.IO.StreamWriter(Console.OpenStandardOutput()); @@ -872,7 +840,7 @@ namespace Duplicati.Server { // TODO: All calls to ApplicationExitEvent and TrayIcon->Quit // should check if we are running something - ApplicationExitEvent.Set(); + applicationSettings.ApplicationExitEvent.Set(); } else { diff --git a/Duplicati/UnitTest/ImportExportTests.cs b/Duplicati/UnitTest/ImportExportTests.cs index 40f99644d..1d61312f3 100644 --- a/Duplicati/UnitTest/ImportExportTests.cs +++ b/Duplicati/UnitTest/ImportExportTests.cs @@ -29,7 +29,7 @@ using Duplicati.Server.Database; using Duplicati.Server.Serializable; using Duplicati.Server.Serialization; using Duplicati.Server.Serialization.Interface; -using Microsoft.Extensions.DependencyInjection; +using Duplicati.WebserverCore.Services; using NUnit.Framework; using Backup = Duplicati.Server.Database.Backup; @@ -107,10 +107,8 @@ namespace Duplicati.UnitTest } byte[] jsonByteArray; - using (Program.DataConnection = Program.GetDatabaseConnection(advancedOptions, true)) - { - jsonByteArray = BackupImportExportHandler.ExportToJSON(Program.DataConnection, backup, null); - } + using (var con = Program.GetDatabaseConnection(new ApplicationSettings(), advancedOptions, true)) + jsonByteArray = BackupImportExportHandler.ExportToJSON(con, backup, null); // The username should not have the '%40' converted to '@' since the import code // cannot handle it (see issue #3619). @@ -130,9 +128,6 @@ namespace Duplicati.UnitTest var metadata = new Dictionary { { "SourceFilesCount", "1" } }; var advancedOptions = new Dictionary { { "server-datafolder", this.serverDatafolder } }; - // Mock the setup enough to get the import/export working - FIXMEGlobal.DataFolder = this.serverDatafolder; - var dbpath = Path.Combine(this.serverDatafolder, Library.AutoUpdater.DataFolderManager.SERVER_DATABASE_FILENAME); if (File.Exists(dbpath)) File.Delete(dbpath); @@ -141,7 +136,7 @@ namespace Duplicati.UnitTest SQLiteLoader.OpenDatabase(con, dbpath, null); DatabaseUpgrader.UpgradeDatabase(con, dbpath, typeof(Library.RestAPI.Database.DatabaseSchemaMarker)); - using (var connection = new Connection(con, true, null)) + using (var connection = new Connection(con, true, null, this.serverDatafolder, null)) { // Unencrypted file, don't import metadata. string unencryptedWithoutMetadata = Path.Combine(this.serverDatafolder, Path.GetRandomFileName()); diff --git a/Duplicati/WebserverCore/DuplicatiWebserver.cs b/Duplicati/WebserverCore/DuplicatiWebserver.cs index 30b45e63c..ccaf2bcf6 100644 --- a/Duplicati/WebserverCore/DuplicatiWebserver.cs +++ b/Duplicati/WebserverCore/DuplicatiWebserver.cs @@ -131,8 +131,9 @@ public class DuplicatiWebserver /// The settings for the server /// The connection to the database /// The log write handler + /// The application settings /// The new webserver instance - public static DuplicatiWebserver CreateWebServer(InitSettings settings, Connection connection, ILogWriteHandler logWriteHandler) + public static DuplicatiWebserver CreateWebServer(InitSettings settings, Connection connection, ILogWriteHandler logWriteHandler, IApplicationSettings applicationSettings) { var builder = WebApplication.CreateBuilder(new WebApplicationOptions() { @@ -278,7 +279,7 @@ public class DuplicatiWebserver builder.Services.AddHealthChecks() .AddCheck("Basic", () => HealthCheckResult.Healthy("Service is running")); - builder.Services.AddDuplicati(connection, logWriteHandler); + builder.Services.AddDuplicati(connection, logWriteHandler, applicationSettings); // Prevent logs from spamming the console, but allow enabling for debugging if (Environment.GetEnvironmentVariable("DUPLICATI_WEBSERVER_LOGGING") != "1") diff --git a/Duplicati/WebserverCore/Endpoints/V1/BackupDefaults.cs b/Duplicati/WebserverCore/Endpoints/V1/BackupDefaults.cs index 918eb7a46..85b1d162f 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/BackupDefaults.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/BackupDefaults.cs @@ -60,7 +60,7 @@ public class BackupDefaults : IEndpointV1 { // Add built-in defaults Newtonsoft.Json.Linq.JObject? n; - using (var s = new StreamReader(typeof(FIXMEGlobal).Assembly.GetManifestResourceStream(typeof(FIXMEGlobal), "newbackup.json")!)) + using (var s = new StreamReader(typeof(Server.Runner).Assembly.GetManifestResourceStream(typeof(Server.Runner), "newbackup.json")!)) n = (Newtonsoft.Json.Linq.JObject)Newtonsoft.Json.JsonConvert.DeserializeObject(s.ReadToEnd())!; o.Merge(n); diff --git a/Duplicati/WebserverCore/Endpoints/V1/RemoteControl.cs b/Duplicati/WebserverCore/Endpoints/V1/RemoteControl.cs index b8e57796b..19af1cc1e 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/RemoteControl.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/RemoteControl.cs @@ -18,7 +18,6 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using Duplicati.Library.RestAPI; using Duplicati.WebserverCore.Abstractions; using Duplicati.WebserverCore.Exceptions; using Microsoft.AspNetCore.Mvc; @@ -32,9 +31,12 @@ public class RemoteControl : IEndpointV1 group.MapGet("/remotecontrol/status", ([FromServices] IRemoteControllerRegistration registration, [FromServices] IRemoteController remoteController) => GetStatus(registration, remoteController)) .RequireAuthorization(); + } + public static void MapConditional(RouteGroupBuilder group, string origin) + { // Don't allow these in agent-mode - if (FIXMEGlobal.Origin == "Agent") + if (origin == "Agent") return; group.MapPost("/remotecontrol/enable", ([FromServices] IRemoteControllerRegistration registration, [FromServices] IRemoteController remoteController) diff --git a/Duplicati/WebserverCore/Endpoints/V1/RemoteOperation.cs b/Duplicati/WebserverCore/Endpoints/V1/RemoteOperation.cs index 3abd2d4ef..ed8bf15fa 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/RemoteOperation.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/RemoteOperation.cs @@ -20,7 +20,6 @@ // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Interface; using Duplicati.Library.Main; -using Duplicati.Library.RestAPI; using Duplicati.Server; using Duplicati.Server.Database; using Duplicati.WebserverCore.Abstractions; @@ -39,8 +38,8 @@ namespace Duplicati.WebserverCore.Endpoints.V1 => ExecuteDbPath(input.path)) .RequireAuthorization(); - group.MapPost("/remoteoperation/test", ([FromServices] Connection connection, [FromQuery] bool? autocreate, [FromBody] RemoteOperationInput input, CancellationToken cancelToken) - => ExecuteTest(connection, input.path, autocreate ?? false, cancelToken)) + group.MapPost("/remoteoperation/test", ([FromServices] Connection connection, [FromServices] IApplicationSettings applicationSettings, [FromQuery] bool? autocreate, [FromBody] RemoteOperationInput input, CancellationToken cancelToken) + => ExecuteTest(connection, applicationSettings, input.path, autocreate ?? false, cancelToken)) .RequireAuthorization(); group.MapPost("/remoteoperation/create", ([FromBody] RemoteOperationInput input, CancellationToken cancelToken) @@ -88,13 +87,13 @@ namespace Duplicati.WebserverCore.Endpoints.V1 } } - private static async Task GetBackend(Connection connection, string url, CancellationToken cancelToken) + private static async Task GetBackend(Connection connection, IApplicationSettings applicationSettings, string url, CancellationToken cancelToken) { var uri = new Library.Utility.Uri(url); var opts = ParseUrlOptions(connection, uri); var tmp = new[] { uri }; - await SecretProviderHelper.ApplySecretProviderAsync([], tmp, opts, Library.Utility.TempFolder.SystemTempPath, FIXMEGlobal.SecretProvider, cancelToken); + await SecretProviderHelper.ApplySecretProviderAsync([], tmp, opts, Library.Utility.TempFolder.SystemTempPath, applicationSettings.SecretProvider, cancelToken); url = tmp[0].ToString(); var modules = ConfigureModules(opts); @@ -102,13 +101,13 @@ namespace Duplicati.WebserverCore.Endpoints.V1 return new TupleDisposeWrapper(backend, modules); } - private static async Task ExecuteTest(Connection connection, string url, bool autoCreate, CancellationToken cancelToken) + private static async Task ExecuteTest(Connection connection, IApplicationSettings applicationSettings, string url, bool autoCreate, CancellationToken cancelToken) { TupleDisposeWrapper? wrapper = null; try { - wrapper = await GetBackend(connection, url, cancelToken); + wrapper = await GetBackend(connection, applicationSettings, url, cancelToken); using (var b = wrapper.Backend) { diff --git a/Duplicati/WebserverCore/Endpoints/V1/WebModules.cs b/Duplicati/WebserverCore/Endpoints/V1/WebModules.cs index 20421f853..4ef47e40e 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/WebModules.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/WebModules.cs @@ -20,7 +20,6 @@ // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Interface; using Duplicati.Library.Main; -using Duplicati.Library.RestAPI; using Duplicati.Server; using Duplicati.Server.Database; using Duplicati.WebserverCore.Abstractions; @@ -34,15 +33,15 @@ public record WebModules : IEndpointV1 public static void Map(RouteGroupBuilder group) { group.MapGet("/webmodules", ExecuteGet).RequireAuthorization(); - group.MapPost("/webmodule/{modulekey}", ([FromServices] Connection connection, [FromRoute] string modulekey, [FromBody] Dictionary options, CancellationToken cancellationToken) - => ExecutePost(connection, modulekey, options, cancellationToken)) + group.MapPost("/webmodule/{modulekey}", ([FromServices] Connection connection, [FromServices] IApplicationSettings applicationSettings, [FromRoute] string modulekey, [FromBody] Dictionary options, CancellationToken cancellationToken) + => ExecutePost(connection, applicationSettings, modulekey, options, cancellationToken)) .RequireAuthorization(); } private static IEnumerable ExecuteGet() => Library.DynamicLoader.WebLoader.Modules; - private static async Task ExecutePost(Connection connection, string modulekey, Dictionary inputOptions, CancellationToken cancellationToken) + private static async Task ExecutePost(Connection connection, IApplicationSettings applicationSettings, string modulekey, Dictionary inputOptions, CancellationToken cancellationToken) { var m = Library.DynamicLoader.WebLoader.Modules.FirstOrDefault(x => x.Key.Equals(modulekey, StringComparison.OrdinalIgnoreCase)) ?? throw new NotFoundException("No such module found"); @@ -51,7 +50,7 @@ public record WebModules : IEndpointV1 foreach (var k in inputOptions.Keys) options[k] = inputOptions[k]; - await SecretProviderHelper.ApplySecretProviderAsync([], [], options, Library.Utility.TempFolder.SystemTempPath, FIXMEGlobal.SecretProvider, cancellationToken); + await SecretProviderHelper.ApplySecretProviderAsync([], [], options, Library.Utility.TempFolder.SystemTempPath, applicationSettings.SecretProvider, cancellationToken); return new Dto.WebModuleOutputDto( Status: "OK", diff --git a/Duplicati/WebserverCore/Endpoints/V2/DestinationVerify.cs b/Duplicati/WebserverCore/Endpoints/V2/DestinationVerify.cs index c44725bf7..504043fdb 100644 --- a/Duplicati/WebserverCore/Endpoints/V2/DestinationVerify.cs +++ b/Duplicati/WebserverCore/Endpoints/V2/DestinationVerify.cs @@ -22,7 +22,6 @@ using Duplicati.Library.Interface; using Duplicati.Library.Main; using Duplicati.Library.Main.Volumes; -using Duplicati.Library.RestAPI; using Duplicati.Server; using Duplicati.Server.Database; using Duplicati.WebserverCore.Abstractions; @@ -35,8 +34,8 @@ public class DestinationVerify : IEndpointV2 { public static void Map(RouteGroupBuilder group) { - group.MapPost("/destination/test", ([FromServices] Connection connection, [FromBody] Dto.V2.DestinationTestRequestDto input, CancellationToken cancelToken) - => ExecuteTest(connection, input, cancelToken)) + group.MapPost("/destination/test", ([FromServices] Connection connection, [FromServices] IApplicationSettings applicationSettings, [FromBody] Dto.V2.DestinationTestRequestDto input, CancellationToken cancelToken) + => ExecuteTest(connection, applicationSettings, input, cancelToken)) .RequireAuthorization(); } @@ -75,13 +74,13 @@ public class DestinationVerify : IEndpointV2 return modules; } - private static async Task GetBackend(Connection connection, string url, CancellationToken cancelToken) + private static async Task GetBackend(Connection connection, IApplicationSettings applicationSettings, string url, CancellationToken cancelToken) { var uri = new Library.Utility.Uri(url); var opts = ParseUrlOptions(connection, uri); var tmp = new[] { uri }; - await SecretProviderHelper.ApplySecretProviderAsync([], tmp, opts, Library.Utility.TempFolder.SystemTempPath, FIXMEGlobal.SecretProvider, cancelToken); + await SecretProviderHelper.ApplySecretProviderAsync([], tmp, opts, Library.Utility.TempFolder.SystemTempPath, applicationSettings.SecretProvider, cancelToken); url = tmp[0].ToString(); var modules = ConfigureModules(opts); @@ -90,13 +89,13 @@ public class DestinationVerify : IEndpointV2 } - private static async Task ExecuteTest(Connection connection, DestinationTestRequestDto input, CancellationToken cancelToken) + private static async Task ExecuteTest(Connection connection, IApplicationSettings applicationSettings, DestinationTestRequestDto input, CancellationToken cancelToken) { TupleDisposeWrapper? wrapper = null; try { - wrapper = await GetBackend(connection, input.DestinationUrl, cancelToken); + wrapper = await GetBackend(connection, applicationSettings, input.DestinationUrl, cancelToken); using (var b = wrapper.Backend) { diff --git a/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs b/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs index d8ebfcc6d..39065eafd 100644 --- a/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs +++ b/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs @@ -35,7 +35,7 @@ namespace Duplicati.WebserverCore.Extensions; public static class ServiceCollectionsExtensions { - public static IServiceCollection AddDuplicati(this IServiceCollection services, Connection connection, ILogWriteHandler logWriteHandler) + public static IServiceCollection AddDuplicati(this IServiceCollection services, Connection connection, ILogWriteHandler logWriteHandler, IApplicationSettings applicationSettings) { //old part services @@ -68,7 +68,8 @@ public static class ServiceCollectionsExtensions .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton(logWriteHandler); + .AddSingleton(logWriteHandler) + .AddSingleton(applicationSettings); return services; } diff --git a/Duplicati/WebserverCore/Extensions/WebApplicationExtensions.cs b/Duplicati/WebserverCore/Extensions/WebApplicationExtensions.cs index 1b92eddac..ede913a4a 100644 --- a/Duplicati/WebserverCore/Extensions/WebApplicationExtensions.cs +++ b/Duplicati/WebserverCore/Extensions/WebApplicationExtensions.cs @@ -56,6 +56,10 @@ public static class WebApplicationExtensions methodMap!.Invoke(null, [group]); } + // Special handling for endpoints that are not supported in all modes + var applicationSettings = application.Services.GetRequiredService(); + Endpoints.V1.RemoteControl.MapConditional(group, applicationSettings.Origin); + return application; } diff --git a/Duplicati/WebserverCore/Services/ApplicationSettings.cs b/Duplicati/WebserverCore/Services/ApplicationSettings.cs new file mode 100644 index 000000000..6e02163bc --- /dev/null +++ b/Duplicati/WebserverCore/Services/ApplicationSettings.cs @@ -0,0 +1,46 @@ + +using Duplicati.Library.AutoUpdater; +using Duplicati.Library.Interface; +using Duplicati.WebserverCore.Abstractions; +namespace Duplicati.WebserverCore.Services; + +/// +/// Application settings for the Duplicati server +/// +public class ApplicationSettings : IApplicationSettings +{ + /// + /// The application exit event + /// + private readonly ManualResetEvent _applicationExitEvent = new ManualResetEvent(false); + /// + /// The folder where Duplicati data is stored + /// + private readonly string _dataFolder; + + /// + /// Initializes a new instance of the class. + /// + public ApplicationSettings() + { + _dataFolder = DataFolderManager.GetDataFolder(DataFolderManager.AccessMode.ReadWritePermissionSet); + } + + /// + public bool SettingsEncryptionKeyProvidedExternally { get; set; } + + /// + public Action? StartOrStopUsageReporter { get; set; } + + /// + public string DataFolder => _dataFolder; + + /// + public string Origin { get; set; } = "Server"; + + /// + public ManualResetEvent ApplicationExitEvent => _applicationExitEvent; + + /// + public ISecretProvider? SecretProvider { get; set; } +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Services/QueueRunnerService.cs b/Duplicati/WebserverCore/Services/QueueRunnerService.cs index 4075e41b7..f30fd70c0 100644 --- a/Duplicati/WebserverCore/Services/QueueRunnerService.cs +++ b/Duplicati/WebserverCore/Services/QueueRunnerService.cs @@ -37,7 +37,7 @@ public class QueueRunnerService( EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IProgressStateProviderService progressStateProviderService, - ILogWriteHandler logWriteHandler) : IQueueRunnerService + IApplicationSettings applicationSettings) : IQueueRunnerService { private readonly object _lock = new(); /// @@ -138,7 +138,7 @@ public class QueueRunnerService( if (task.OnStarting != null) await task.OnStarting().ConfigureAwait(false); - Runner.Run(connection, eventPollNotify, notificationUpdateService, progressStateProviderService, task, true); + Runner.Run(connection, eventPollNotify, notificationUpdateService, progressStateProviderService, applicationSettings, task, true); // If the task is completed, don't call OnFinished again completed = true; @@ -211,6 +211,6 @@ public class QueueRunnerService( /// public IBasicResults? RunImmediately(IQueuedTask task) { - return Runner.Run(connection, eventPollNotify, notificationUpdateService, progressStateProviderService, task, false); + return Runner.Run(connection, eventPollNotify, notificationUpdateService, progressStateProviderService, applicationSettings, task, false); } } \ No newline at end of file diff --git a/Duplicati/WebserverCore/Services/RemoteControllerHandler.cs b/Duplicati/WebserverCore/Services/RemoteControllerHandler.cs index 5259eef67..371c987e6 100644 --- a/Duplicati/WebserverCore/Services/RemoteControllerHandler.cs +++ b/Duplicati/WebserverCore/Services/RemoteControllerHandler.cs @@ -22,7 +22,6 @@ using System.Net.Http.Headers; using Duplicati.Library.Logging; using Duplicati.Library.RemoteControl; -using Duplicati.Library.RestAPI; using Duplicati.Server.Database; using Duplicati.WebserverCore.Abstractions; using Newtonsoft.Json; @@ -35,7 +34,7 @@ namespace Duplicati.WebserverCore.Services; /// The connection to the database /// The HTTP client factory /// The JWT token provider -public class RemoteControllerHandler(Connection connection, IHttpClientFactory httpClientFactory, IJWTTokenProvider jwtTokenProvider) : IRemoteControllerHandler +public class RemoteControllerHandler(Connection connection, IHttpClientFactory httpClientFactory, IJWTTokenProvider jwtTokenProvider, IApplicationSettings applicationSettings) : IRemoteControllerHandler { /// /// The log tag for this class. @@ -64,7 +63,7 @@ public class RemoteControllerHandler(Connection connection, IHttpClientFactory h CertificateUrl = data.CertificateUrl }); - if (!FIXMEGlobal.SettingsEncryptionKeyProvidedExternally) + if (!applicationSettings.SettingsEncryptionKeyProvidedExternally) { // TODO: Implement changing the encryption key // if (!string.IsNullOrWhiteSpace(data.LocalEncryptionKey) && data.LocalEncryptionKey != connection.ApplicationSettings.SettingsEncryptionKey) diff --git a/Duplicati/WebserverCore/Services/SystemInfoProvider.cs b/Duplicati/WebserverCore/Services/SystemInfoProvider.cs index faacc34b9..b4e0e7b2d 100644 --- a/Duplicati/WebserverCore/Services/SystemInfoProvider.cs +++ b/Duplicati/WebserverCore/Services/SystemInfoProvider.cs @@ -21,8 +21,8 @@ using System.Globalization; using Duplicati.Library.AutoUpdater; using Duplicati.Library.Localization; -using Duplicati.Library.RestAPI; using Duplicati.Server; +using Duplicati.Server.Database; using Duplicati.Server.Serialization.Interface; using Duplicati.WebserverCore.Abstractions; using Duplicati.WebserverCore.Dto; @@ -32,7 +32,7 @@ namespace Duplicati.WebserverCore.Services; /// /// Produces system information. /// -public class SystemInfoProvider : ISystemInfoProvider +public class SystemInfoProvider(IApplicationSettings applicationSettings) : ISystemInfoProvider { /// /// The API extensions that are available @@ -87,11 +87,6 @@ public class SystemInfoProvider : ISystemInfoProvider /// public required string RemoteControlRegistrationUrl { get; init; } - /// - /// Gets or sets the started by. - /// - public required string StartedBy { get; init; } - /// /// Gets or sets the default update channel. /// @@ -231,12 +226,11 @@ public class SystemInfoProvider : ISystemInfoProvider => new StaticSystemInformation { APIVersion = 1, - PasswordPlaceholder = FIXMEGlobal.PASSWORD_PLACEHOLDER, + PasswordPlaceholder = Connection.PASSWORD_PLACEHOLDER, ServerVersion = UpdaterManager.SelfVersion.Version, ServerVersionName = License.VersionNumbers.VERSION_NAME, ServerVersionType = UpdaterManager.SelfVersion.ReleaseType, RemoteControlRegistrationUrl = Library.RemoteControl.RegisterForRemote.DefaultRegisterationUrl, - StartedBy = FIXMEGlobal.Origin, DefaultUpdateChannel = AutoUpdateSettings.DefaultUpdateChannel.ToString(), DefaultUsageReportLevel = Library.UsageReporter.Reporter.DefaultReportLevel, OSType = UpdaterManager.OperatingSystemName, @@ -315,7 +309,7 @@ public class SystemInfoProvider : ISystemInfoProvider ServerVersionName = systeminfo.ServerVersionName, ServerVersionType = systeminfo.ServerVersionType, RemoteControlRegistrationUrl = systeminfo.RemoteControlRegistrationUrl, - StartedBy = systeminfo.StartedBy, + StartedBy = applicationSettings.Origin, DefaultUpdateChannel = systeminfo.DefaultUpdateChannel, DefaultUsageReportLevel = systeminfo.DefaultUsageReportLevel, ServerTime = DateTime.Now, From b7600401f35daf8f128f1dff2ac16e06079ff029 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 5 May 2025 16:18:21 +0200 Subject: [PATCH 18/24] Bump again to get flushing working --- Duplicati/Library/Utility/Duplicati.Library.Utility.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj index a37f25bff..ee6d149bb 100644 --- a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj +++ b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj @@ -8,7 +8,7 @@ - + From 18b2df2e7eed492f6ba34d88b0286c54e99b9a28 Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Thu, 8 May 2025 10:08:21 -0300 Subject: [PATCH 19/24] Switching to constant rather than string literal --- Duplicati/Library/Backend/S3/S3Backend.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index 017cce158..8d4a2be0e 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -328,8 +328,8 @@ namespace Duplicati.Library.Backend var exts = S3AwsClient.GetAwsExtendedOptions(); return [ - new CommandLineArgument("aws-access-key-id", CommandLineArgument.ArgumentType.String, Strings.S3Backend.AMZUserIDDescriptionShort, Strings.S3Backend.AMZUserIDDescriptionLong, null, [AuthOptionsHelper.AuthUsernameOption], null), - new CommandLineArgument("aws-secret-access-key", CommandLineArgument.ArgumentType.Password, Strings.S3Backend.AMZKeyDescriptionShort, Strings.S3Backend.AMZKeyDescriptionLong,null, [AuthOptionsHelper.AuthPasswordOption], null ), + new CommandLineArgument(AUTH_USERNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.S3Backend.AMZUserIDDescriptionShort, Strings.S3Backend.AMZUserIDDescriptionLong, null, [AuthOptionsHelper.AuthUsernameOption], null), + new CommandLineArgument(AUTH_PASSWORD_OPTION, CommandLineArgument.ArgumentType.Password, Strings.S3Backend.AMZKeyDescriptionShort, Strings.S3Backend.AMZKeyDescriptionLong,null, [AuthOptionsHelper.AuthPasswordOption], null ), new CommandLineArgument(STORAGECLASS_OPTION, CommandLineArgument.ArgumentType.String, Strings.S3Backend.S3StorageclassDescriptionShort, Strings.S3Backend.S3StorageclassDescriptionLong, "", null, KNOWN_S3_STORAGE_CLASSES.Select(x => x.Value).WhereNotNullOrWhiteSpace().ToArray()), new CommandLineArgument(SERVER_NAME, CommandLineArgument.ArgumentType.String, Strings.S3Backend.S3ServerNameDescriptionShort, Strings.S3Backend.S3ServerNameDescriptionLong(hostnames.ToString()), DEFAULT_S3_HOST), new CommandLineArgument(LOCATION_OPTION, CommandLineArgument.ArgumentType.String, Strings.S3Backend.S3LocationDescriptionShort, Strings.S3Backend.S3LocationDescriptionLong(locations.ToString())), From 9f6a24ad4f3f134bcb97bf7d80dbb04ab612fc0f Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 8 May 2025 15:34:21 +0200 Subject: [PATCH 20/24] Update AuthIdOptionsHelper.cs Change type to Password --- Duplicati/Library/Utility/Options/AuthIdOptionsHelper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Utility/Options/AuthIdOptionsHelper.cs b/Duplicati/Library/Utility/Options/AuthIdOptionsHelper.cs index 2689d19d0..33a2a7653 100644 --- a/Duplicati/Library/Utility/Options/AuthIdOptionsHelper.cs +++ b/Duplicati/Library/Utility/Options/AuthIdOptionsHelper.cs @@ -44,7 +44,7 @@ public static class AuthIdOptionsHelper /// The authentication options public static CommandLineArgument[] GetOptions(string tokenurl, string? prefix = null) => [ - new CommandLineArgument($"{prefix}{AuthIdOption}", CommandLineArgument.ArgumentType.String, Strings.AuthIdSettingsHelper.AuthidShort, Strings.AuthIdSettingsHelper.AuthidLong(tokenurl)) + new CommandLineArgument($"{prefix}{AuthIdOption}", CommandLineArgument.ArgumentType.Password, Strings.AuthIdSettingsHelper.AuthidShort, Strings.AuthIdSettingsHelper.AuthidLong(tokenurl)) ]; /// From c1b051177bf8653c7f9c193116c9c8f4301c23b3 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 9 May 2025 11:35:52 +0200 Subject: [PATCH 21/24] Remove CloudFiles backend This relates to #6134 --- Duplicati.sln | 7 - .../Library/Backend/CloudFiles/CloudFiles.cs | 450 ------------------ ...uplicati.Library.Backend.CloudFiles.csproj | 25 - .../Library/Backend/CloudFiles/Strings.cs | 44 -- Duplicati/Library/Backends/BackendModules.cs | 1 - 5 files changed, 527 deletions(-) delete mode 100644 Duplicati/Library/Backend/CloudFiles/CloudFiles.cs delete mode 100644 Duplicati/Library/Backend/CloudFiles/Duplicati.Library.Backend.CloudFiles.csproj delete mode 100644 Duplicati/Library/Backend/CloudFiles/Strings.cs diff --git a/Duplicati.sln b/Duplicati.sln index 40d5d5bb1..ff0f27dae 100644 --- a/Duplicati.sln +++ b/Duplicati.sln @@ -21,8 +21,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Duplicati.CommandLine", "Du EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Duplicati.Library.Backend.WEBDAV", "Duplicati\Library\Backend\WEBDAV\Duplicati.Library.Backend.WEBDAV.csproj", "{BAE27510-8B5D-44B2-B33E-372A98908041}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Duplicati.Library.Backend.CloudFiles", "Duplicati\Library\Backend\CloudFiles\Duplicati.Library.Backend.CloudFiles.csproj", "{1BFAE226-8364-4086-825C-BB83F6F3EE4C}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Duplicati.CommandLine.BackendTester", "Duplicati\CommandLine\BackendTester\Duplicati.CommandLine.BackendTester.csproj", "{E7280DCA-7776-4A73-B9B5-41FD77FC8799}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Duplicati.Library.Interface", "Duplicati\Library\Interface\Duplicati.Library.Interface.csproj", "{C5899F45-B0FF-483C-9D38-24A9FCAAB237}" @@ -234,10 +232,6 @@ Global {BAE27510-8B5D-44B2-B33E-372A98908041}.Debug|Any CPU.Build.0 = Debug|Any CPU {BAE27510-8B5D-44B2-B33E-372A98908041}.Release|Any CPU.ActiveCfg = Release|Any CPU {BAE27510-8B5D-44B2-B33E-372A98908041}.Release|Any CPU.Build.0 = Release|Any CPU - {1BFAE226-8364-4086-825C-BB83F6F3EE4C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {1BFAE226-8364-4086-825C-BB83F6F3EE4C}.Debug|Any CPU.Build.0 = Debug|Any CPU - {1BFAE226-8364-4086-825C-BB83F6F3EE4C}.Release|Any CPU.ActiveCfg = Release|Any CPU - {1BFAE226-8364-4086-825C-BB83F6F3EE4C}.Release|Any CPU.Build.0 = Release|Any CPU {E7280DCA-7776-4A73-B9B5-41FD77FC8799}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {E7280DCA-7776-4A73-B9B5-41FD77FC8799}.Debug|Any CPU.Build.0 = Debug|Any CPU {E7280DCA-7776-4A73-B9B5-41FD77FC8799}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -553,7 +547,6 @@ Global {C03F6DFD-805A-4BE0-9338-64870ADDB4A2} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4} {81765A64-3661-4E3E-B850-2F6F87A51F74} = {D19A38DD-68F1-4EF5-BF5F-8966CE0D9A5B} {BAE27510-8B5D-44B2-B33E-372A98908041} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4} - {1BFAE226-8364-4086-825C-BB83F6F3EE4C} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4} {E7280DCA-7776-4A73-B9B5-41FD77FC8799} = {D19A38DD-68F1-4EF5-BF5F-8966CE0D9A5B} {0CA86ECF-5BEC-4909-B4F6-110A03B30B92} = {566EBBDA-19A4-4056-A615-D901D57D2439} {C0270709-2A40-43B5-8CF1-69581B9FA2A1} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4} diff --git a/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs b/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs deleted file mode 100644 index 956cbd1f3..000000000 --- a/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs +++ /dev/null @@ -1,450 +0,0 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -using Duplicati.Library.Common.IO; -using Duplicati.Library.Interface; -using Duplicati.Library.Utility; -using Duplicati.Library.Utility.Options; -using System.Net; -using System.Runtime.CompilerServices; -using System.Security.Cryptography; - -namespace Duplicati.Library.Backend -{ - // ReSharper disable once UnusedMember.Global - // This class is instantiated dynamically in the BackendLoader. - public class CloudFiles : IBackend, IStreamingBackend - { - /// - /// The log tag for this class - /// - private static readonly string LOGTAG = Logging.Log.LogTagFromType(); - - public const string AUTH_URL_US = "https://identity.api.rackspacecloud.com/auth"; - public const string AUTH_URL_UK = "https://lon.auth.api.rackspacecloud.com/v1.0"; - private const string DUMMY_HOSTNAME = "api.mosso.com"; - - private const string AUTH_USERNAME_OPTION = "cloudfiles-username"; - private const string AUTH_PASSWORD_OPTION = "cloudfiles-accesskey"; - - private const int ITEM_LIST_LIMIT = 1000; - private readonly string m_username; - private readonly string m_password; - private readonly string m_path; - - private string? m_storageUrl = null; - private string? m_authToken = null; - private readonly string m_authUrl; - private readonly TimeoutOptionsHelper.Timeouts m_timeouts; - - // ReSharper disable once UnusedMember.Global - // This constructor is needed by the BackendLoader. - public CloudFiles() - { - m_username = null!; - m_password = null!; - m_path = null!; - m_authUrl = null!; - m_timeouts = null!; - } - - // ReSharper disable once UnusedMember.Global - // This constructor is needed by the BackendLoader. - public CloudFiles(string url, Dictionary options) - { - var uri = new Utility.Uri(url); - var auth = AuthOptionsHelper.ParseWithAlias(options, uri, AUTH_USERNAME_OPTION, AUTH_PASSWORD_OPTION); - - if (!auth.HasUsername) - throw new UserInformationException(Strings.CloudFiles.NoUserIDError, "CloudFilesNoUserID"); - if (auth.HasPassword) - throw new UserInformationException(Strings.CloudFiles.NoAPIKeyError, "CloudFilesNoApiKey"); - - (m_username, m_password) = auth.GetCredentials(); - - //Fallback to the previous format - if (url.Contains(DUMMY_HOSTNAME)) - { - Logging.Log.WriteWarningMessage(LOGTAG, "CloudFilesDeprecatedFormat", null, Strings.CloudFiles.DeprecatedFormatWarning(DUMMY_HOSTNAME)); - var u = new System.Uri(url); - - if (!string.IsNullOrEmpty(u.UserInfo)) - { - if (u.UserInfo.IndexOf(":", StringComparison.Ordinal) >= 0) - { - m_username = u.UserInfo.Substring(0, u.UserInfo.IndexOf(":", StringComparison.Ordinal)); - m_password = u.UserInfo.Substring(u.UserInfo.IndexOf(":", StringComparison.Ordinal) + 1); - } - else - { - m_username = u.UserInfo; - } - } - - //We use the api.mosso.com hostname. - //This allows the use of containers that have names that are not valid hostnames, - // such as container names with spaces in them - if (u.Host.Equals(DUMMY_HOSTNAME)) - m_path = Library.Utility.Uri.UrlDecode(u.PathAndQuery); - else - m_path = u.Host + Library.Utility.Uri.UrlDecode(u.PathAndQuery); - } - else - { - m_path = uri.HostAndPath; - } - - if (m_path.EndsWith("/", StringComparison.Ordinal)) - m_path = m_path.Substring(0, m_path.Length - 1); - if (!m_path.StartsWith("/", StringComparison.Ordinal)) - m_path = "/" + m_path; - - var authUrl = options.GetValueOrDefault("cloudfiles-authentication-url"); - if (string.IsNullOrEmpty(authUrl)) - authUrl = Utility.Utility.ParseBoolOption(options, "cloudfiles-uk-account") ? AUTH_URL_UK : AUTH_URL_US; - m_authUrl = authUrl; - m_timeouts = TimeoutOptionsHelper.Parse(options); - } - - #region IBackend Members - - public string DisplayName - { - get { return Strings.CloudFiles.DisplayName; } - } - - public string ProtocolKey - { - get { return "cloudfiles"; } - } - - public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) - { - var extraUrl = "?format=xml&limit=" + ITEM_LIST_LIMIT.ToString(); - var markerUrl = ""; - - bool repeat; - do - { - var doc = new System.Xml.XmlDocument(); - var req = await CreateRequest("", extraUrl + markerUrl, cancelToken).ConfigureAwait(false); - - try - { - await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ => - { - var areq = new Utility.AsyncHttpRequest(req); - using (var resp = (HttpWebResponse)areq.GetResponse()) - using (var s = areq.GetResponseStream()) - doc.Load(s); - }).ConfigureAwait(false); - } - catch (WebException wex) - { - if (markerUrl == "") //Only check on first iteration - if (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound) - throw new FolderMissingException(wex); - - //Other error, just re-throw - throw; - } - - var lst = doc.SelectNodes("container/object"); - - //Perhaps the folder does not exist? - //The response should be 404 from the server, but it is not :( - if (lst == null || lst.Count == 0 && markerUrl == "") //Only on first iteration - { - try { await CreateFolderAsync(cancelToken).ConfigureAwait(false); } - catch { } //Ignore - } - - if (lst == null) - yield break; - - var lastItemName = ""; - foreach (System.Xml.XmlNode n in lst) - { - var name = n["name"]?.InnerText; - if (!long.TryParse(n["bytes"]?.InnerText, out var size)) - size = -1; - if (!DateTime.TryParse(n["last_modified"]?.InnerText, out var mod)) - mod = new DateTime(); - - lastItemName = name; - yield return new FileEntry(name, size, mod, mod); - } - - repeat = lst.Count == ITEM_LIST_LIMIT; - - if (repeat) - markerUrl = "&marker=" + Library.Utility.Uri.UrlEncode(lastItemName); - - } while (repeat); - } - - public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) - { - using (var fs = File.OpenRead(filename)) - await PutAsync(remotename, fs, cancelToken); - } - - public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken) - { - using (var fs = File.Create(filename)) - await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false); - } - - public async Task DeleteAsync(string remotename, CancellationToken cancelToken) - { - try - { - var req = await CreateRequest("/" + remotename, "", cancelToken).ConfigureAwait(false); - await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => - { - req.Method = "DELETE"; - var areq = new AsyncHttpRequest(req); - using (var resp = (HttpWebResponse)areq.GetResponse()) - { - if (resp.StatusCode == HttpStatusCode.NotFound) - throw new FileMissingException(); - - if ((int)resp.StatusCode >= 300) - throw new WebException(Strings.CloudFiles.FileDeleteError, null, WebExceptionStatus.ProtocolError, resp); - else - using (areq.GetResponseStream()) - { } - } - }).ConfigureAwait(false); - } - catch (WebException wex) - { - if (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound) - throw new FileMissingException(wex); - else - throw; - } - } - - public IList SupportedCommands => - [ - new CommandLineArgument(AUTH_USERNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.CloudFiles.DescriptionUsernameShort, Strings.CloudFiles.DescriptionUsernameLong, null, [AuthOptionsHelper.AuthUsernameOption] ), - new CommandLineArgument(AUTH_PASSWORD_OPTION, CommandLineArgument.ArgumentType.Password, Strings.CloudFiles.DescriptionPasswordShort, Strings.CloudFiles.DescriptionPasswordLong, null, [AuthOptionsHelper.AuthPasswordOption]), - new CommandLineArgument("cloudfiles-uk-account", CommandLineArgument.ArgumentType.Boolean, Strings.CloudFiles.DescriptionUKAccountShort, Strings.CloudFiles.DescriptionUKAccountLong("cloudfiles-authentication-url", AUTH_URL_UK)), - new CommandLineArgument("cloudfiles-authentication-url", CommandLineArgument.ArgumentType.String, Strings.CloudFiles.DescriptionAuthenticationURLShort, Strings.CloudFiles.DescriptionAuthenticationURLLong_v2("cloudfiles-uk-account"), AUTH_URL_US), - .. TimeoutOptionsHelper.GetOptions(), - ]; - - public string Description - { - get { return Strings.CloudFiles.Description_v2; } - } - - #endregion - - #region IBackend_v2 Members - - public Task TestAsync(CancellationToken cancelToken) => - //The "Folder not found" is not detectable :( - this.TestListAsync(cancelToken); - - public async Task CreateFolderAsync(CancellationToken cancelToken) - { - var createReq = await CreateRequest("", "", cancelToken).ConfigureAwait(false); - await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => - { - createReq.Method = "PUT"; - var areq = new AsyncHttpRequest(createReq); - using (var resp = (HttpWebResponse)areq.GetResponse()) - { } - }).ConfigureAwait(false); - } - - #endregion - - #region IDisposable Members - - public void Dispose() - { - } - - #endregion - - #region IStreamingBackend Members - - public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult( - new string?[] { - new System.Uri(m_authUrl).Host, - string.IsNullOrWhiteSpace(m_storageUrl) ? null : new System.Uri(m_storageUrl).Host - } - .WhereNotNullOrWhiteSpace() - .ToArray() - ); - - public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) - { - var req = await CreateRequest("/" + remotename, "", cancelToken).ConfigureAwait(false); - req.Method = "GET"; - - var areq = new AsyncHttpRequest(req); - using (var resp = areq.GetResponse()) - using (var s = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponseStream()).ConfigureAwait(false)) - using (var timeoutStream = s.ObserveReadTimeout(m_timeouts.ReadWriteTimeout)) - using (var hasher = MD5.Create()) - using (var mds = new HashCalculatingStream(timeoutStream, hasher)) - { - var md5Hash = resp.Headers["ETag"]; - await Utility.Utility.CopyStreamAsync(mds, stream, true, cancelToken).ConfigureAwait(false); - - if (!string.Equals(mds.GetFinalHashString(), md5Hash, StringComparison.OrdinalIgnoreCase)) - throw new Exception(Strings.CloudFiles.ETagVerificationError); - } - } - - public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) - { - var req = await CreateRequest("/" + remotename, "", cancelToken).ConfigureAwait(false); - req.Method = "PUT"; - req.ContentType = "application/octet-stream"; - - try { req.ContentLength = stream.Length; } - catch { } - - // TODO: When reviewing this, lets build a common method to unwrap the stream passed from - // BackendManager.PutOperation, so we can use the same logic to compute hashes in all backends - - //If we can pre-calculate the MD5 hash before transmission, do so - /*if (stream.CanSeek) - { - System.Security.Cryptography.MD5 md5 = System.Security.Cryptography.MD5.Create(); - req.Headers["ETag"] = Core.Utility.ByteArrayAsHexString(md5.ComputeHash(stream)).ToLower(System.Globalization.CultureInfo.InvariantCulture); - stream.Seek(0, System.IO.SeekOrigin.Begin); - - using (System.IO.Stream s = req.GetRequestStream()) - Core.Utility.CopyStream(stream, s); - - //Reset the timeout to the default value of 100 seconds to - // avoid blocking the GetResponse() call - req.Timeout = 100000; - - //The server handles the eTag verification for us, and gives an error if the hash was a mismatch - using (HttpWebResponse resp = (HttpWebResponse)req.GetResponse()) - if ((int)resp.StatusCode >= 300) - throw new WebException(Strings.CloudFiles.FileUploadError, null, WebExceptionStatus.ProtocolError, resp); - - } - else //Otherwise use a client-side calculation - */ - //TODO: We cannot use the local MD5 calculation, because that could involve a throttled read, - // and may invoke various events - { - string? fileHash = null; - - long streamLen = -1; - try { streamLen = stream.Length; } - catch { } - - var areq = new AsyncHttpRequest(req); - using (var s = areq.GetRequestStream(streamLen)) - using (var timeoutStream = s.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout)) - using (var hasher = MD5.Create()) - using (var mds = new HashCalculatingStream(timeoutStream, hasher)) - { - await Utility.Utility.CopyStreamAsync(stream, mds, tryRewindSource: true, cancelToken: cancelToken); - fileHash = mds.GetFinalHashString(); - } - - string? md5Hash = null; - - //We need to verify the eTag locally - try - { - using (HttpWebResponse resp = (HttpWebResponse)areq.GetResponse()) - if ((int)resp.StatusCode >= 300) - throw new WebException(Strings.CloudFiles.FileUploadError, null, WebExceptionStatus.ProtocolError, resp); - else - md5Hash = resp.Headers["ETag"]; - } - catch (WebException wex) - { - //Catch 404 and turn it into a FolderNotFound error - if (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound) - throw new FolderMissingException(wex); - - //Other error, just re-throw - throw; - } - - - if (md5Hash == null || !string.Equals(md5Hash, fileHash, StringComparison.OrdinalIgnoreCase)) - { - //Remove the broken file - try { await DeleteAsync(remotename, cancelToken); } - catch { } - - throw new Exception(Strings.CloudFiles.ETagVerificationError); - } - } - } - - #endregion - - private async Task CreateRequest(string remotename, string query, CancellationToken cancelToken) - { - //If this is the first call, get an authentication token - if (string.IsNullOrEmpty(m_authToken) || string.IsNullOrEmpty(m_storageUrl)) - { - await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => - { - var authReq = (HttpWebRequest)HttpWebRequest.Create(m_authUrl); - authReq.Headers.Add("X-Auth-User", m_username); - authReq.Headers.Add("X-Auth-Key", m_password); - authReq.Method = "GET"; - - var areq = new AsyncHttpRequest(authReq); - using (var resp = areq.GetResponse()) - { - m_storageUrl = resp.Headers["X-Storage-Url"]; - m_authToken = resp.Headers["X-Auth-Token"]; - } - - if (string.IsNullOrEmpty(m_authToken) || string.IsNullOrEmpty(m_storageUrl)) - throw new Exception(Strings.CloudFiles.UnexpectedResponseError); - }).ConfigureAwait(false); - } - - var req = (HttpWebRequest)HttpWebRequest.Create(m_storageUrl + UrlEncode(m_path + remotename) + query); - req.Headers.Add("X-Auth-Token", UrlEncode(m_authToken!)); - - req.UserAgent = "Duplicati CloudFiles Backend v" + System.Reflection.Assembly.GetExecutingAssembly().GetName().Version; - req.KeepAlive = false; - req.PreAuthenticate = true; - req.AllowWriteStreamBuffering = false; - - return req; - } - - private static string UrlEncode(string value) - { - return Utility.Uri.UrlEncode(value).Replace("+", "%20").Replace("%2f", "/"); - } - } -} diff --git a/Duplicati/Library/Backend/CloudFiles/Duplicati.Library.Backend.CloudFiles.csproj b/Duplicati/Library/Backend/CloudFiles/Duplicati.Library.Backend.CloudFiles.csproj deleted file mode 100644 index 9f355da46..000000000 --- a/Duplicati/Library/Backend/CloudFiles/Duplicati.Library.Backend.CloudFiles.csproj +++ /dev/null @@ -1,25 +0,0 @@ - - - - net8.0 - Library - Duplicati.Library.Backend - Copyright © 2025 Team Duplicati, MIT license - enable - enable - - - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - diff --git a/Duplicati/Library/Backend/CloudFiles/Strings.cs b/Duplicati/Library/Backend/CloudFiles/Strings.cs deleted file mode 100644 index acb5dc405..000000000 --- a/Duplicati/Library/Backend/CloudFiles/Strings.cs +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. -using Duplicati.Library.Localization.Short; -namespace Duplicati.Library.Backend.Strings -{ - internal static class CloudFiles - { - public static string Description_v2 { get { return LC.L(@"This backend can read and write data to CloudFiles. Allowed format is ""cloudfiles://container/folder""."); } } - public static string DisplayName { get { return LC.L(@"Rackspace CloudFiles"); } } - public static string DescriptionAuthenticationURLLong_v2(string optionname) { return LC.L(@"CloudFiles use different servers for authentication based on where the account resides. Use this option to set an alternate authentication URL. This option overrides --{0}.", optionname); } - public static string DescriptionAuthenticationURLShort { get { return LC.L(@"Provide another authentication URL"); } } - public static string DescriptionPasswordLong { get { return LC.L(@"The API Access Key used to authenticate with CloudFiles."); } } - public static string DescriptionPasswordShort { get { return LC.L(@"Supply the access key used to connect to the server"); } } - public static string DescriptionUKAccountLong(string optionname, string optionvalue) { return LC.L(@"Duplicati will assume that the credentials given are for a US account. Use this option if the account is a UK based account. Note that this is equivalent to setting --{0}={1}.", optionname, optionvalue); } - public static string DescriptionUKAccountShort { get { return LC.L(@"Use a UK account"); } } - public static string DescriptionUsernameLong { get { return LC.L(@"The username used to authenticate with CloudFiles."); } } - public static string DescriptionUsernameShort { get { return LC.L(@"Supply the username used to authenticate with CloudFiles"); } } - public static string ETagVerificationError { get { return LC.L(@"MD5 Hash (ETag) verification failed"); } } - public static string FileDeleteError { get { return LC.L(@"Failed to delete file"); } } - public static string FileUploadError { get { return LC.L(@"Failed to upload file"); } } - public static string NoAPIKeyError { get { return LC.L(@"No CloudFiles API Access Key given"); } } - public static string NoUserIDError { get { return LC.L(@"No CloudFiles userID given"); } } - public static string UnexpectedResponseError { get { return LC.L(@"Unexpected CloudFiles response. Perhaps the API has changed?"); } } - public static string DeprecatedFormatWarning(string hostname) { return LC.L(@$"The URL is using the marker hostname {hostname} which is no longer supported. Please use the new format ""cloudfiles://container/folder"" instead."); } - } -} diff --git a/Duplicati/Library/Backends/BackendModules.cs b/Duplicati/Library/Backends/BackendModules.cs index dd66ce6c6..05ef8db4f 100644 --- a/Duplicati/Library/Backends/BackendModules.cs +++ b/Duplicati/Library/Backends/BackendModules.cs @@ -42,7 +42,6 @@ public static class BackendModules new Backend.AzureBlob.AzureBlobBackend(), new Backend.Backblaze.B2(), new Backend.Box.BoxBackend(), - new Backend.CloudFiles(), new Backend.Dropbox(), new Backend.FTP(), new Backend.AlternateFTPBackend(), From 42c3ecd009f4dacde4fd4a678ff0824ced7490eb Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 9 May 2025 13:46:45 +0200 Subject: [PATCH 22/24] Fix throttle stream issues Some providers, B2, Idrivee2, JottaCloud and S3 (AWS version) needs a checksum for the content. To calculate this content the libraries will calculate the hash by reading the stream. But since the stream can be throttled, the hash calculation is throttled. In some cases the contents are buffered, so the transfer is done at full speed, but the hashing at throttled speed. The average speed is the same, but network traffic is bursty, which it should not be with a throttle. Also added option to disable signing with AWS Sigv4 as that interferes with the throttled stream. It is not currently possible to pass the sha-256 hash to the Sigv4 code, so it will always read the entire (potentially throttled) stream before sending the contents, causing unwanted delays. Use the new option `--s3-disable-payload-signing` to disable v4 signing. --- .../Backend/AliyunOSS/AliyunOSSBackend.cs | 7 +- Duplicati/Library/Backend/Backblaze/B2.cs | 43 +---- .../Backend/Idrivee2/Idrivee2Backend.cs | 3 +- .../Library/Backend/Jottacloud/Jottacloud.cs | 155 +++++++----------- Duplicati/Library/Backend/S3/S3AwsClient.cs | 16 +- Duplicati/Library/Backend/S3/S3Backend.cs | 7 +- .../Library/Utility/OverrideableStream.cs | 5 + Duplicati/Library/Utility/Utility.cs | 89 +++++++++- 8 files changed, 180 insertions(+), 145 deletions(-) diff --git a/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs b/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs index d77909b41..72b1aaa9e 100644 --- a/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs +++ b/Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs @@ -198,9 +198,14 @@ namespace Duplicati.Library.Backend.AliyunOSS var client = GetClient(); try { + var metadata = new ObjectMetadata + { + ContentLength = stream.Length + }; + using var timeoutStream = stream.ObserveReadTimeout(_timeouts.ReadWriteTimeout, false); var objectResult = await Task.Factory.FromAsync( - (cb, state) => client.BeginPutObject(bucketName, objectName, timeoutStream, cb, state), + (cb, state) => client.BeginPutObject(bucketName, objectName, timeoutStream, metadata, cb, state), client.EndPutObject, null).ConfigureAwait(false); diff --git a/Duplicati/Library/Backend/Backblaze/B2.cs b/Duplicati/Library/Backend/Backblaze/B2.cs index 63c7a04c0..967c41a68 100644 --- a/Duplicati/Library/Backend/Backblaze/B2.cs +++ b/Duplicati/Library/Backend/Backblaze/B2.cs @@ -310,43 +310,8 @@ public class B2 : IStreamingBackend public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { - TempFile? tmp = null; - - var measure = stream; - while (measure is OverrideableStream os && - os.GetType().GetField("m_basestream", BindingFlags.NonPublic | BindingFlags.Instance)?.GetValue(os) is Stream baseStream) - { - measure = baseStream; - } - - if (measure == null) - throw new Exception($"Unable to unwrap stream from: {stream.GetType()}"); - - string sha1; - if (measure.CanSeek) - { - var p = measure.Position; - - // Compute the hash - using (var hashalg = HashFactory.CreateHasher("SHA1")) - sha1 = Utility.Utility.ByteArrayAsHexString(hashalg.ComputeHash(measure)); - - measure.Position = p; - } - else - { - // No seeking possible, use a temp file - tmp = new TempFile(); - await using (var sr = File.OpenWrite(tmp)) - using (var hasher = HashFactory.CreateHasher("SHA1")) - await using (var hc = new HashCalculatingStream(measure, hasher)) - { - await Utility.Utility.CopyStreamAsync(hc, sr, cancelToken).ConfigureAwait(false); - sha1 = hc.GetFinalHashString(); - } - - stream = File.OpenRead(tmp); - } + (stream, var sha1, var tmp) = await Utility.Utility.CalculateThrottledStreamHash(stream, "SHA1", cancelToken).ConfigureAwait(false); + using var _ = tmp; if (_filecache == null) await RebuildFileCache(cancelToken).ConfigureAwait(false); @@ -400,10 +365,6 @@ public class B2 : IStreamingBackend throw; } - finally - { - tmp?.Dispose(); - } } /// diff --git a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs index d4f5d7809..c8c3f5432 100644 --- a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs +++ b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs @@ -190,9 +190,8 @@ namespace Duplicati.Library.Backend if (m_s3Client == null) { (var accessKeyId, var accessKeySecret) = m_auth.GetCredentials(); - // TODO: Do not make blocking calls in the constructor var host = await GetRegionEndpointAsync("https://api.idrivee2.com/api/service/get_region_end_point/" + accessKeyId, cancellationToken).ConfigureAwait(false); - m_s3Client = new S3AwsClient(accessKeyId, accessKeySecret, null, host, null, true, false, m_timeouts, m_options); + m_s3Client = new S3AwsClient(accessKeyId, accessKeySecret, null, host, null, true, false, false, m_timeouts, m_options); } return m_s3Client; diff --git a/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs b/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs index ec4e5f30f..805544398 100644 --- a/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs +++ b/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs @@ -489,108 +489,71 @@ namespace Duplicati.Library.Backend // we specify the file size in a custom header. And if the stream is not seek-able we are not able // to use stream.Length, so we are back at square one. - // TODO: This unwrapping code is shared with CloudFiles - TempFile? tmpFile = null; - Stream? baseStream = stream; - while (baseStream is OverrideableStream) - baseStream = typeof(OverrideableStream).GetField("m_basestream", System.Reflection.BindingFlags.DeclaredOnly | System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic)?.GetValue(baseStream) as System.IO.Stream; - if (baseStream == null) - throw new Exception(string.Format("Unable to unwrap stream from: {0}", stream.GetType())); - string md5Hash; - if (baseStream.CanSeek) - { - var originalPosition = baseStream.Position; - using (var md5 = MD5.Create()) - md5Hash = Utility.Utility.ByteArrayAsHexString(md5.ComputeHash(baseStream)); - baseStream.Position = originalPosition; - } - else - { - // No seeking possible, use a temp file - tmpFile = new TempFile(); - using (var os = File.OpenWrite(tmpFile)) - using (var hasher = MD5.Create()) - using (var md5 = new HashCalculatingStream(baseStream, hasher)) - { - await Utility.Utility.CopyStreamAsync(md5, os, true, cancelToken).ConfigureAwait(false); - md5Hash = md5.GetFinalHashString(); - } - stream = File.OpenRead(tmpFile); - } - try - { - // Create request, with query parameter, and a few custom headers. - // NB: If we wanted to we could send the same POST request as below but without the file contents - // and with "cphash=[md5Hash]" as the only query parameter. Then we will get an HTTP 200 (OK) response - // if an identical file already exists, and we can skip uploading the new file. We will get - // HTTP 404 (Not Found) if file does not exists or it exists with a different hash, in which - // case we must send a new request to upload the new content. - var fileSize = stream.Length; - var req = CreateRequest(WebRequestMethods.Http.Post, remotename, "umode=nomultipart", true); - req.Headers.Add("JMd5", md5Hash); // Not required, but it will make the server verify the content and mark the file as corrupt if there is a mismatch. - req.Headers.Add("JSize", fileSize.ToString()); // Required, and used to mark file as incomplete if we upload something be the total size of the original file! - // File time stamp headers: Since we are working with a stream here we do not know the local file's timestamps, - // and then we can just omit the JCreated and JModified and let the server automatically set the current time. - //req.Headers.Add("JCreated", timeCreated); - //req.Headers.Add("JModified", timeModified); - req.ContentType = "application/octet-stream"; - req.ContentLength = fileSize; + (stream, var md5Hash, var tmp) = await Utility.Utility.CalculateThrottledStreamHash(stream, "MD5", cancelToken).ConfigureAwait(false); + using var _ = tmp; - // Write post data request - var areq = new AsyncHttpRequest(req); - using (var rs = areq.GetRequestStream()) - using (var ts = rs.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout)) - await Utility.Utility.CopyStreamAsync(stream, rs, true, cancelToken).ConfigureAwait(false); - // Send request, and check response - using (var resp = (HttpWebResponse)areq.GetResponse()) - { - if (resp.StatusCode != HttpStatusCode.Created) - throw new WebException(Strings.Jottacloud.FileUploadError, null, WebExceptionStatus.ProtocolError, resp); + // Create request, with query parameter, and a few custom headers. + // NB: If we wanted to we could send the same POST request as below but without the file contents + // and with "cphash=[md5Hash]" as the only query parameter. Then we will get an HTTP 200 (OK) response + // if an identical file already exists, and we can skip uploading the new file. We will get + // HTTP 404 (Not Found) if file does not exists or it exists with a different hash, in which + // case we must send a new request to upload the new content. + var fileSize = stream.Length; + var req = CreateRequest(WebRequestMethods.Http.Post, remotename, "umode=nomultipart", true); + req.Headers.Add("JMd5", md5Hash); // Not required, but it will make the server verify the content and mark the file as corrupt if there is a mismatch. + req.Headers.Add("JSize", fileSize.ToString()); // Required, and used to mark file as incomplete if we upload something be the total size of the original file! + // File time stamp headers: Since we are working with a stream here we do not know the local file's timestamps, + // and then we can just omit the JCreated and JModified and let the server automatically set the current time. + //req.Headers.Add("JCreated", timeCreated); + //req.Headers.Add("JModified", timeModified); + req.ContentType = "application/octet-stream"; + req.ContentLength = fileSize; - // Request seems to be successful, but we must verify the response XML content to be sure that the file - // was correctly uploaded: The server will verify the JSize header and mark the file as incomplete if - // there was mismatch, and it will verify the JMd5 header and mark the file as corrupt if there was a hash - // mismatch. The returned XML contains a file element, and if upload was error free it contains a single - // child element "currentRevision", which has a "state" child element with the string "COMPLETED". - // If there was a problem we should have a "latestRevision" child element, and this will have state with - // value "INCOMPLETE" or "CORRUPT". If the file was new or had no previous complete versions the latestRevision - // will be the only child, but if not there may also be a "currentRevision" representing the previous - // complete version - and then we need to detect the case where our upload failed but there was an existing - // complete version! - using (var rs = areq.GetResponseStream()) + // Write post data request + var areq = new AsyncHttpRequest(req); + using (var rs = areq.GetRequestStream()) + using (var ts = rs.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout)) + await Utility.Utility.CopyStreamAsync(stream, rs, true, cancelToken).ConfigureAwait(false); + // Send request, and check response + using (var resp = (HttpWebResponse)areq.GetResponse()) + { + if (resp.StatusCode != HttpStatusCode.Created) + throw new WebException(Strings.Jottacloud.FileUploadError, null, WebExceptionStatus.ProtocolError, resp); + + // Request seems to be successful, but we must verify the response XML content to be sure that the file + // was correctly uploaded: The server will verify the JSize header and mark the file as incomplete if + // there was mismatch, and it will verify the JMd5 header and mark the file as corrupt if there was a hash + // mismatch. The returned XML contains a file element, and if upload was error free it contains a single + // child element "currentRevision", which has a "state" child element with the string "COMPLETED". + // If there was a problem we should have a "latestRevision" child element, and this will have state with + // value "INCOMPLETE" or "CORRUPT". If the file was new or had no previous complete versions the latestRevision + // will be the only child, but if not there may also be a "currentRevision" representing the previous + // complete version - and then we need to detect the case where our upload failed but there was an existing + // complete version! + using (var rs = areq.GetResponseStream()) + { + var doc = new System.Xml.XmlDocument(); + try { doc.Load(rs); } + catch (System.Xml.XmlException) { - var doc = new System.Xml.XmlDocument(); - try { doc.Load(rs); } - catch (System.Xml.XmlException) - { - throw new WebException(Strings.Jottacloud.FileUploadError, WebExceptionStatus.ProtocolError); - } - bool uploadCompletedSuccessfully = false; - var xFile = doc["file"]; - if (xFile != null) - { - var xRevState = xFile.SelectSingleNode("latestRevision"); - if (xRevState == null) - { - xRevState = xFile.SelectSingleNode("currentRevision/state"); - if (xRevState != null) - uploadCompletedSuccessfully = xRevState.InnerText == "COMPLETED"; // Success: There is no "latestRevision", only a "currentRevision" (and it specifies the file is complete, but I think it always will). - } - } - if (!uploadCompletedSuccessfully) // Report error (and we just let the incomplete/corrupt file revision stay on the server..) - throw new WebException(Strings.Jottacloud.FileUploadError, WebExceptionStatus.ProtocolError); + throw new WebException(Strings.Jottacloud.FileUploadError, WebExceptionStatus.ProtocolError); } + bool uploadCompletedSuccessfully = false; + var xFile = doc["file"]; + if (xFile != null) + { + var xRevState = xFile.SelectSingleNode("latestRevision"); + if (xRevState == null) + { + xRevState = xFile.SelectSingleNode("currentRevision/state"); + if (xRevState != null) + uploadCompletedSuccessfully = xRevState.InnerText == "COMPLETED"; // Success: There is no "latestRevision", only a "currentRevision" (and it specifies the file is complete, but I think it always will). + } + } + if (!uploadCompletedSuccessfully) // Report error (and we just let the incomplete/corrupt file revision stay on the server..) + throw new WebException(Strings.Jottacloud.FileUploadError, WebExceptionStatus.ProtocolError); } } - finally - { - try - { - if (tmpFile != null) - tmpFile.Dispose(); - } - catch { } - } } } } diff --git a/Duplicati/Library/Backend/S3/S3AwsClient.cs b/Duplicati/Library/Backend/S3/S3AwsClient.cs index d21ea417e..ae2c5957a 100644 --- a/Duplicati/Library/Backend/S3/S3AwsClient.cs +++ b/Duplicati/Library/Backend/S3/S3AwsClient.cs @@ -61,6 +61,10 @@ namespace Duplicati.Library.Backend /// The option to specify if chunk encoding should be used /// private readonly bool m_useChunkEncoding; + /// + /// The option to specify if payload signing should be disabled + /// + private readonly bool m_disablePayloadSigning; /// /// The DNS host of the S3 server @@ -93,7 +97,7 @@ namespace Duplicati.Library.Backend ]); public S3AwsClient(string awsID, string awsKey, string? locationConstraint, string servername, - string? storageClass, bool useSSL, bool disableChunkEncoding, TimeoutOptionsHelper.Timeouts timeouts, Dictionary options) + string? storageClass, bool useSSL, bool disableChunkEncoding, bool disablePayloadSigning, TimeoutOptionsHelper.Timeouts timeouts, Dictionary options) { var cfg = GetDefaultAmazonS3Config(); cfg.UseHttp = !useSSL; @@ -109,6 +113,7 @@ namespace Duplicati.Library.Backend m_storageClass = storageClass; m_dnsHost = string.IsNullOrWhiteSpace(cfg.ServiceURL) ? null : new System.Uri(cfg.ServiceURL).Host; m_useChunkEncoding = !disableChunkEncoding; + m_disablePayloadSigning = disablePayloadSigning; m_archiveClasses = ParseStorageClasses(options.GetValueOrDefault(S3_ARCHIVE_CLASSES_OPTION)); } @@ -217,13 +222,20 @@ namespace Duplicati.Library.Backend public virtual async Task AddFileStreamAsync(string bucketName, string keyName, Stream source, CancellationToken cancelToken) { + (source, var md5, var tmp) = await Utility.Utility.CalculateThrottledStreamHash(source, "MD5", cancelToken).ConfigureAwait(false); + using var _ = tmp; + + md5 = Convert.ToBase64String(Utility.Utility.HexStringAsByteArray(md5)); + using var ts = source.ObserveReadTimeout(m_timeouts.ReadWriteTimeout, false); var objectAddRequest = new PutObjectRequest { BucketName = bucketName, Key = keyName, InputStream = ts, - UseChunkEncoding = m_useChunkEncoding + UseChunkEncoding = m_useChunkEncoding, + MD5Digest = md5, + DisablePayloadSigning = m_disablePayloadSigning }; if (!string.IsNullOrWhiteSpace(m_storageClass)) objectAddRequest.StorageClass = new S3StorageClass(m_storageClass); diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index 8d4a2be0e..b6b21c166 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -41,6 +41,7 @@ namespace Duplicati.Library.Backend private const string SSL_OPTION = "use-ssl"; private const string S3_CLIENT_OPTION = "s3-client"; private const string S3_DISABLE_CHUNK_ENCODING_OPTION = "s3-disable-chunk-encoding"; + private const string S3_DISABLE_PAYLOAD_SIGNING_OPTION = "s3-disable-payload-signing"; private const string S3_LIST_API_VERSION_OPTION = "s3-list-api-version"; private const string S3_RECURSIVE_LIST = "s3-recursive-list"; @@ -242,14 +243,15 @@ namespace Duplicati.Library.Backend if (!options.ContainsKey("s3-ext-forcepathstyle") && !hostname.EndsWith(".amazonaws.com", StringComparison.OrdinalIgnoreCase)) options["s3-ext-forcepathstyle"] = "true"; - var disableChunkEncoding = Utility.Utility.ParseBoolOption(options, S3_DISABLE_CHUNK_ENCODING_OPTION); var s3ClientOptionValue = options.GetValueOrDefault(S3_CLIENT_OPTION); (var awsID, var awsKey) = auth.GetCredentials(); if (string.IsNullOrWhiteSpace(s3ClientOptionValue) || string.Equals(s3ClientOptionValue, "aws", StringComparison.OrdinalIgnoreCase)) { - m_s3Client = new S3AwsClient(awsID, awsKey, locationConstraint, hostname, storageClass, useSSL, disableChunkEncoding, timeout, options); + var disableChunkEncoding = Utility.Utility.ParseBoolOption(options, S3_DISABLE_CHUNK_ENCODING_OPTION); + var disablePayloadSigning = Utility.Utility.ParseBoolOption(options, S3_DISABLE_PAYLOAD_SIGNING_OPTION); + m_s3Client = new S3AwsClient(awsID, awsKey, locationConstraint, hostname, storageClass, useSSL, disableChunkEncoding, disablePayloadSigning, timeout, options); } else if (string.Equals(s3ClientOptionValue, "minio", StringComparison.OrdinalIgnoreCase)) { @@ -336,6 +338,7 @@ namespace Duplicati.Library.Backend new CommandLineArgument(SSL_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionUseSSLShort, Strings.S3Backend.DescriptionUseSSLLong), new CommandLineArgument(S3_CLIENT_OPTION, CommandLineArgument.ArgumentType.Enumeration, Strings.S3Backend.S3ClientDescriptionShort, Strings.S3Backend.S3ClientDescriptionLong, "aws", null, new string[] { "aws", "minio" }), new CommandLineArgument(S3_DISABLE_CHUNK_ENCODING_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionDisableChunkEncodingShort, Strings.S3Backend.DescriptionDisableChunkEncodingLong, "false"), + new CommandLineArgument(S3_DISABLE_PAYLOAD_SIGNING_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionDisablePayloadSigningShort, Strings.S3Backend.DescriptionDisablePayloadSigningLong, "false"), new CommandLineArgument(S3AwsClient.S3_ARCHIVE_CLASSES_OPTION, CommandLineArgument.ArgumentType.Flags, Strings.S3Backend.S3ArchiveClassesDescriptionShort, Strings.S3Backend.S3ArchiveClassesDescriptionLong, string.Join(",", S3AwsClient.DEFAULT_ARCHIVE_CLASSES.Select(x => x.Value)), null, KNOWN_S3_STORAGE_CLASSES.Select(x => x.Value).WhereNotNullOrWhiteSpace().ToArray()), new CommandLineArgument(S3_LIST_API_VERSION_OPTION, CommandLineArgument.ArgumentType.Enumeration, Strings.S3Backend.DescriptionListApiVersionShort, Strings.S3Backend.DescriptionListApiVersionLong, "v1", null, ["v1", "v2"]), new CommandLineArgument(S3_RECURSIVE_LIST, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionRecursiveListShort, Strings.S3Backend.DescriptionRecursiveListLong, "false"), diff --git a/Duplicati/Library/Utility/OverrideableStream.cs b/Duplicati/Library/Utility/OverrideableStream.cs index 35e8a5273..b331c0b2f 100644 --- a/Duplicati/Library/Utility/OverrideableStream.cs +++ b/Duplicati/Library/Utility/OverrideableStream.cs @@ -37,6 +37,11 @@ namespace Duplicati.Library.Utility /// protected Stream m_basestream; + /// + /// The base stream that is wrapped + /// + public Stream BaseStream => m_basestream; + /// /// Creates a new instance /// diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 7f54fb5a4..d29f548ab 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -23,6 +23,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.IO; using System.Linq; @@ -1397,7 +1398,8 @@ namespace Duplicati.Library.Utility /// The wrapped commandline element. /// The argument to wrap. /// A flag indicating if environment variables are allowed to be expanded - public static string WrapCommandLineElement(string? arg, bool allowEnvExpansion) + [return: NotNullIfNotNull("arg")] + public static string? WrapCommandLineElement(string? arg, bool allowEnvExpansion) { if (string.IsNullOrWhiteSpace(arg)) return arg; @@ -1847,5 +1849,90 @@ namespace Duplicati.Library.Utility : Timeout.Infinite }; } + + /// + /// The types of streams that are considered basic (i.e. not wrapped) + /// + private static readonly IReadOnlySet _basicStreamTypes = new HashSet + { + typeof(FileStream), + typeof(MemoryStream), + typeof(NetworkStream), + typeof(BufferedStream), + typeof(System.IO.Compression.DeflateStream), + typeof(System.IO.Compression.GZipStream), + typeof(System.IO.Compression.ZLibStream) + }; + + /// + /// Unwraps a stream from layers of wrapping streams + /// + /// The stream to unwrap + /// The unwrapped stream + public static Stream UnwrapThrottledStream(this Stream stream) + { + if (stream == null) + throw new ArgumentNullException(nameof(stream)); + + var previousStream = stream; + + do + { + previousStream = stream; + + while (stream is WrappingStream wrappingStream) + stream = wrappingStream.BaseStream; + while (stream is OverrideableStream overrideableStream) + stream = overrideableStream.BaseStream; + + } while (stream != previousStream); + +#if DEBUG + if (!_basicStreamTypes.Contains(stream.GetType())) + throw new InvalidOperationException($"The unwrapped stream is not a basic stream, but a {stream.GetType()}"); +#endif + + return stream; + } + + /// + /// Calculates the hash of a throttled stream and returns the stream to read + /// + /// The source stream + /// The hash algorithm to use + /// The cancellation token to observe + /// A tuple with the stream, the hash and a temporary file if used + public static async Task<(Stream content, string hash, TempFile? tmpfile)> CalculateThrottledStreamHash(Stream stream, string hashalgorithm, CancellationToken cancelToken) + { + TempFile? tmp = null; + string contentHash; + var measure = stream.UnwrapThrottledStream(); + if (measure.CanSeek) + { + var p = measure.Position; + + // Compute the hash + using (var hashalg = HashFactory.CreateHasher(hashalgorithm)) + contentHash = ByteArrayAsHexString(hashalg.ComputeHash(measure)); + + measure.Position = p; + } + else + { + // No seeking possible, use a temp file + tmp = new TempFile(); + await using (var sr = File.OpenWrite(tmp)) + using (var hasher = HashFactory.CreateHasher(hashalgorithm)) + await using (var hc = new HashCalculatingStream(measure, hasher)) + { + await CopyStreamAsync(hc, sr, cancelToken).ConfigureAwait(false); + contentHash = hc.GetFinalHashString(); + } + + stream = File.OpenRead(tmp); + } + + return (stream, contentHash, tmp); + } } } From b6decf41ad4d721e42f50f5ec9ffbcc7cb829ba7 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 9 May 2025 13:49:11 +0200 Subject: [PATCH 23/24] Added missing commit --- Duplicati/Library/Backend/S3/Strings.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Duplicati/Library/Backend/S3/Strings.cs b/Duplicati/Library/Backend/S3/Strings.cs index 016f5c896..fc77f6ec6 100644 --- a/Duplicati/Library/Backend/S3/Strings.cs +++ b/Duplicati/Library/Backend/S3/Strings.cs @@ -47,6 +47,8 @@ namespace Duplicati.Library.Backend.Strings public static string DescriptionUseSSLShort { get { return LC.L(@"Instruct Duplicati to use an SSL (https) connection"); } } public static string DescriptionDisableChunkEncodingLong { get { return LC.L(@"This disables chunk encoding for the aws client, which is not supported by all S3 providers."); } } public static string DescriptionDisableChunkEncodingShort { get { return LC.L(@"Disable chunk encoding (aws client only)"); } } + public static string DescriptionDisablePayloadSigningLong { get { return LC.L(@"This disables payload signing for the aws client, which is not supported by all S3 providers."); } } + public static string DescriptionDisablePayloadSigningShort { get { return LC.L(@"Disable payload signing (aws client only)"); } } public static string S3StorageclassDescriptionLong { get { return LC.L(@"Use this option to specify a storage class. If this option is not used, the server will choose a default storage class."); } } public static string S3StorageclassDescriptionShort { get { return LC.L(@"Specify storage class"); } } public static string S3ArchiveClassesDescriptionShort { get { return LC.L(@"Specify archive storage class"); } } From c3d022aa087cfe3ccdf59c04baf6d4784a552748 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 9 May 2025 15:18:33 +0200 Subject: [PATCH 24/24] Fixed a bug with disposing an active task --- Duplicati/Library/Backend/Rclone/Rclone.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Backend/Rclone/Rclone.cs b/Duplicati/Library/Backend/Rclone/Rclone.cs index 99f1873ff..bb78b96c5 100644 --- a/Duplicati/Library/Backend/Rclone/Rclone.cs +++ b/Duplicati/Library/Backend/Rclone/Rclone.cs @@ -158,9 +158,9 @@ namespace Duplicati.Library.Backend process.BeginOutputReadLine(); process.BeginErrorReadLine(); - using var timer = timeout == Timeout.InfiniteTimeSpan + var timer = timeout == Timeout.InfiniteTimeSpan ? new TaskCompletionSource().Task - : Task.Delay(timeout); + : Task.Delay(timeout, cancelToken); var killed = false; while (!process.HasExited)