diff --git a/.github/workflows/backendtests.yml b/.github/workflows/backendtests.yml index 9b3148002..9cd5239df 100644 --- a/.github/workflows/backendtests.yml +++ b/.github/workflows/backendtests.yml @@ -14,6 +14,7 @@ jobs: google_secrets_available: "${{ steps.check_google.outputs.available }}" pcloud_secrets_available: "${{ steps.check_pcloud.outputs.available }}" b2_secrets_available: "${{ steps.check_b2.outputs.available }}" + box_secrets_available: "${{ steps.check_box.outputs.available }}" steps: - id: check_pcloud name: Check pCloud secrets @@ -120,6 +121,20 @@ jobs: echo "available=false" >> $GITHUB_OUTPUT fi echo "B2 check completed" + - id: check_box + name: Check Box.com secrets + shell: bash + run: | + echo "Starting Box.com secrets check..." + if [[ -n "${{ secrets.TESTCREDENTIAL_BOX_FOLDER }}" ]] && \ + [[ -n "${{ secrets.TESTCREDENTIAL_BOX_AUTHID }}" ]]; then + echo "All Box.com secrets found" + echo "available=true" >> $GITHUB_OUTPUT + else + echo "Missing some Box.com secrets" + echo "available=false" >> $GITHUB_OUTPUT + fi + echo "Box.com check completed" - name: Debug Output shell: bash run: | @@ -130,6 +145,7 @@ jobs: echo "pCloud Available: ${{ steps.check_pcloud.outputs.available }}" echo "Azure Available: ${{ steps.check_azure.outputs.available }}" echo "B2 Available: ${{ steps.check_b2.outputs.available }}" + echo "Box.com Available: ${{ steps.check_box.outputs.available }}" test_ftp: needs: check_secrets if: needs.check_secrets.outputs.testcontainers_secrets_available == 'true' @@ -468,40 +484,76 @@ jobs: --logger:"console;verbosity=detailed" LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln test_b2: - needs: check_secrets - if: needs.check_secrets.outputs.b2_secrets_available == 'true' - name: Backblaze B2 Tests - runs-on: "${{ matrix.os }}" - strategy: - max-parallel: 1 - fail-fast: false - matrix: - os: - - ubuntu-latest - - windows-latest - - macos-latest - steps: - - name: Set up .NET - uses: actions/setup-dotnet@v4 - with: - dotnet-version: 8.x - - name: Checkout source - uses: actions/checkout@v4 - - name: Restore NuGet dependencies - run: >- - dotnet restore - LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln - - name: Build project - run: >- - dotnet build --no-restore - LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln - - name: Run B2 tests - env: - TESTCREDENTIAL_B2_BUCKET: "${{ secrets.TESTCREDENTIAL_B2_BUCKET }}" - TESTCREDENTIAL_B2_FOLDER: "${{ secrets.TESTCREDENTIAL_B2_FOLDER }}" - TESTCREDENTIAL_B2_USERNAME: "${{ secrets.TESTCREDENTIAL_B2_USERNAME }}" - TESTCREDENTIAL_B2_PASSWORD: "${{ secrets.TESTCREDENTIAL_B2_PASSWORD }}" - run: >- - dotnet test --no-build --filter="ClassName~B2Tests" - --logger:"console;verbosity=detailed" - LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln \ No newline at end of file + needs: check_secrets + if: needs.check_secrets.outputs.b2_secrets_available == 'true' + name: Backblaze B2 Tests + runs-on: "${{ matrix.os }}" + strategy: + max-parallel: 1 + fail-fast: false + matrix: + os: + - ubuntu-latest + - windows-latest + - macos-latest + steps: + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.x + - name: Checkout source + uses: actions/checkout@v4 + - name: Restore NuGet dependencies + run: >- + dotnet restore + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + - name: Build project + run: >- + dotnet build --no-restore + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + - name: Run B2 tests + env: + TESTCREDENTIAL_B2_BUCKET: "${{ secrets.TESTCREDENTIAL_B2_BUCKET }}" + TESTCREDENTIAL_B2_FOLDER: "${{ secrets.TESTCREDENTIAL_B2_FOLDER }}" + TESTCREDENTIAL_B2_USERNAME: "${{ secrets.TESTCREDENTIAL_B2_USERNAME }}" + TESTCREDENTIAL_B2_PASSWORD: "${{ secrets.TESTCREDENTIAL_B2_PASSWORD }}" + run: >- + dotnet test --no-build --filter="ClassName~B2Tests" + --logger:"console;verbosity=detailed" + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + test_box: + needs: check_secrets + if: needs.check_secrets.outputs.box_secrets_available == 'true' + name: Box.com Tests + runs-on: "${{ matrix.os }}" + strategy: + max-parallel: 1 + fail-fast: false + matrix: + os: + - ubuntu-latest + - windows-latest + - macos-latest + steps: + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: 8.x + - name: Checkout source + uses: actions/checkout@v4 + - name: Restore NuGet dependencies + run: >- + dotnet restore + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + - name: Build project + run: >- + dotnet build --no-restore + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln + - name: Run Box.com tests + env: + TESTCREDENTIAL_BOX_FOLDER: "${{ secrets.TESTCREDENTIAL_BOX_FOLDER }}" + TESTCREDENTIAL_BOX_AUTHID: "${{ secrets.TESTCREDENTIAL_BOX_AUTHID }}" + run: >- + dotnet test --no-build --filter="ClassName~Box.BoxTests" + --logger:"console;verbosity=detailed" + LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln diff --git a/Duplicati/Agent/Program.cs b/Duplicati/Agent/Program.cs index 12451ea3d..c02ecbdd0 100644 --- a/Duplicati/Agent/Program.cs +++ b/Duplicati/Agent/Program.cs @@ -34,7 +34,7 @@ using Duplicati.Library.RemoteControl; using Duplicati.Library.RestAPI; using Duplicati.Library.Utility; using Duplicati.Server; -using Duplicati.WebserverCore.Abstractions; +using Duplicati.WebserverCore.Services; using Microsoft.Extensions.DependencyInjection; namespace Duplicati.Agent; @@ -46,12 +46,6 @@ public static class Program /// private static readonly string LogTag = Log.LogTagFromType(typeof(Program)); - /// - /// Create a random pre-shared key for the agent - /// - private static string? PreSharedKey = null; - - /// /// Commandline arguments for the agent /// @@ -275,7 +269,7 @@ public static class Program // Set the pre-shared key for the agent if (!agentConfig.DisablePreSharedKey) - WebserverCore.Middlewares.PreSharedKeyFilter.PreSharedKey = PreSharedKey = RandomNumberGenerator.GetHexString(128); + WebserverCore.Middlewares.PreSharedKeyFilter.PreSharedKey = RandomNumberGenerator.GetHexString(128); using var cts = new CancellationTokenSource(); @@ -313,8 +307,10 @@ public static class Program settings.CertificateUrl, settings.ServerCertificates, cts.Token, - c => ReKey(agentConfig, c), - (m) => OnMessage(m, agentConfig) + OnConnect, + m => ReKey(m, agentConfig), + OnControl, + OnMessage ) ); @@ -324,24 +320,19 @@ public static class Program } } - private static async Task OnMessage(KeepRemoteConnection.CommandMessage message, CommandLineArguments agentConfig) - { - Log.WriteMessage(LogMessageType.Verbose, LogTag, "OnMessage", "Received message: {0}", message); - - var provider = FIXMEGlobal.Provider.GetRequiredService(); - var token = provider.CreateAccessToken("agent", provider.TemporaryFamilyId, TimeSpan.FromMinutes(2)); - using var httpClient = FIXMEGlobal.Provider.GetRequiredService().CreateClient(); - httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token); - httpClient.BaseAddress = new System.Uri($"http://127.0.0.1:{agentConfig.WebservicePort}"); - - if (!string.IsNullOrWhiteSpace(WebserverCore.Middlewares.PreSharedKeyFilter.PreSharedKey)) - httpClient.DefaultRequestHeaders.Add(WebserverCore.Middlewares.PreSharedKeyFilter.HeaderName, PreSharedKey); - - await message.Handle(httpClient); - } - - private static Task ReKey(CommandLineArguments agentConfig, ClaimedClientData keydata) + private static Task> OnConnect(Dictionary metadata) + => FIXMEGlobal.Provider.GetRequiredService().OnConnect(metadata); + + private static Task OnControl(KeepRemoteConnection.ControlMessage message) + => FIXMEGlobal.Provider.GetRequiredService().OnControl(message); + + private static Task OnMessage(KeepRemoteConnection.CommandMessage message) + => FIXMEGlobal.Provider.GetRequiredService().OnMessage(message); + + + private static Task ReKey(ClaimedClientData keydata, CommandLineArguments agentConfig) { + // ReKey is handled here because we store the config outside of the database Log.WriteMessage(LogMessageType.Verbose, LogTag, "ReKey", "Rekeying the settings"); var settings = Settings.Load(agentConfig.AgentSettingsFile.FullName, agentConfig.AgentSettingsFilePassphrase); if (!string.IsNullOrWhiteSpace(keydata.JWT) && settings.JWT != keydata.JWT) diff --git a/Duplicati/CommandLine/CLI/Commands.cs b/Duplicati/CommandLine/CLI/Commands.cs index 76425af0c..9aa3a8e99 100644 --- a/Duplicati/CommandLine/CLI/Commands.cs +++ b/Duplicati/CommandLine/CLI/Commands.cs @@ -668,6 +668,7 @@ namespace Duplicati.CommandLine if (parsedStats.KnownFileCount > 0) { output.MessageEvent(string.Format(" Remote files: {0}", parsedStats.KnownFileCount)); + output.MessageEvent(string.Format(" Remote filesets: {0}", parsedStats.KnownFilesets)); output.MessageEvent(string.Format(" Remote size: {0}", Library.Utility.Utility.FormatSizeString(parsedStats.KnownFileSize))); } diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs index d9e924630..2529f79f4 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs @@ -224,7 +224,7 @@ No password provided, unable to connect to server, exiting"); if (hosted != null && Server.Program.ApplicationInstance != null) Server.Program.ApplicationInstance.SecondInstanceDetected += new Server.SingleInstance.SecondInstanceDelegate( - x => { tk.ShowUrlInWindow(serverURL.ToString()); }); + x => tk.ShowStatusWindow()); // TODO: If we change to hosted browser this should be a callback if (openui) diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs index 1b769718f..d6815b21f 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs @@ -180,7 +180,7 @@ namespace Duplicati.GUI.TrayIcon }); } - protected void ShowStatusWindow() + public void ShowStatusWindow() { Program.Connection.GetStatusWindowURLAsync().ContinueWith(t => { diff --git a/Duplicati/Library/Backend/Backblaze/B2.cs b/Duplicati/Library/Backend/Backblaze/B2.cs index 8d72ed7c0..4ed35e91b 100644 --- a/Duplicati/Library/Backend/Backblaze/B2.cs +++ b/Duplicati/Library/Backend/Backblaze/B2.cs @@ -272,7 +272,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend _uploadUrl = await _b2AuthHelper.PostAndGetJsonDataAsync( $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_get_upload_url", - new UploadUrlRequest { BucketID = GetBucketAsync(cancellationToken).Await().BucketID }, + new UploadUrlRequest { BucketID = (await GetBucketAsync(cancellationToken)).BucketID }, combinedCancellationToken.Token ).ConfigureAwait(false); @@ -487,7 +487,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_file_versions", new ListFilesRequest { - BucketID = GetBucketAsync(cancellationToken).Await().BucketID, + BucketID = (await GetBucketAsync(cancellationToken)).BucketID, MaxFileCount = _pageSize, Prefix = _prefix, StartFileID = nextFileId, diff --git a/Duplicati/Library/Interface/ResultInterfaces.cs b/Duplicati/Library/Interface/ResultInterfaces.cs index 1848e98ca..0c886c43b 100644 --- a/Duplicati/Library/Interface/ResultInterfaces.cs +++ b/Duplicati/Library/Interface/ResultInterfaces.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -38,7 +38,7 @@ namespace Duplicati.Library.Interface DateTime BeginTime { get; } DateTime EndTime { get; } TimeSpan Duration { get; } - + IEnumerable Errors { get; } IEnumerable Warnings { get; } IEnumerable Messages { get; } @@ -64,6 +64,7 @@ namespace Duplicati.Library.Interface long UnknownFileCount { get; set; } long KnownFileCount { get; set; } long KnownFileSize { get; set; } + long KnownFilesets { get; set; } DateTime LastBackupDate { get; set; } long BackupListCount { get; set; } long TotalQuotaSpace { get; set; } @@ -116,14 +117,14 @@ namespace Duplicati.Library.Interface IEnumerable LogMessages { get; } IEnumerable RemoteVolumes { get; } } - + public interface IDeleteResults : IBasicResults { IEnumerable> DeletedSets { get; } ICompactResults CompactResults { get; } bool Dryrun { get; } } - + public interface IBackupResults : IBasicResults, IBackendStatsticsReporter { long DeletedFiles { get; } @@ -143,16 +144,16 @@ namespace Duplicati.Library.Interface long ModifiedFolders { get; } long ModifiedSymlinks { get; } long AddedSymlinks { get; } - long DeletedSymlinks { get; } + long DeletedSymlinks { get; } bool PartialBackup { get; } bool Dryrun { get; } - + ICompactResults CompactResults { get; } IVacuumResults VacuumResults { get; } IDeleteResults DeleteResults { get; } IRepairResults RepairResults { get; } } - + public interface IRestoreResults : IBasicResults { long RestoredFiles { get; } @@ -163,10 +164,10 @@ namespace Duplicati.Library.Interface long DeletedFiles { get; } long DeletedFolders { get; } long DeletedSymlinks { get; } - + IRecreateDatabaseResults RecreateDatabaseResults { get; } } - + public interface IRecreateDatabaseResults : IBasicResults { } @@ -188,23 +189,23 @@ namespace Duplicati.Library.Interface IVacuumResults VacuumResults { get; } } - + public interface ICreateLogDatabaseResults : IBasicResults { string TargetPath { get; } } - + public interface IRestoreControlFilesResults : IBasicResults { IEnumerable Files { get; } } - + public interface IRepairResults : IBasicResults { IRecreateDatabaseResults RecreateDatabaseResults { get; } } - - + + /// /// The possible change types for an entry /// @@ -222,8 +223,8 @@ namespace Duplicati.Library.Interface /// The element was modified /// Modified - } - + } + /// /// The possible entry types /// @@ -242,16 +243,16 @@ namespace Duplicati.Library.Interface /// File } - + public interface IListChangesResults : IBasicResults { DateTime BaseVersionTimestamp { get; } DateTime CompareVersionTimestamp { get; } long BaseVersionIndex { get; } long CompareVersionIndex { get; } - - IEnumerable> ChangeDetails { get; } - + + IEnumerable> ChangeDetails { get; } + long AddedFolders { get; } long AddedSymlinks { get; } long AddedFiles { get; } @@ -263,14 +264,14 @@ namespace Duplicati.Library.Interface long ModifiedFolders { get; } long ModifiedSymlinks { get; } long ModifiedFiles { get; } - + long PreviousSize { get; } long CurrentSize { get; } long AddedSize { get; } long DeletedSize { get; } } - + /// /// The status of a change in a test entry /// @@ -298,7 +299,7 @@ namespace Duplicati.Library.Interface { IEnumerable>>> Verifications { get; } } - + public interface ITestFilterResults : IBasicResults { long FileSize { get; set; } diff --git a/Duplicati/Library/Localization/ILocalizationService.cs b/Duplicati/Library/Localization/ILocalizationService.cs index c0c35f262..f9eeb825f 100644 --- a/Duplicati/Library/Localization/ILocalizationService.cs +++ b/Duplicati/Library/Localization/ILocalizationService.cs @@ -1,31 +1,35 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; -using System.Collections.Generic; +using System.Globalization; namespace Duplicati.Library.Localization { public interface ILocalizationService { + /// + /// The culture of the localization service + /// + public CultureInfo Culture { get; } + /// /// Localizes the string similar to how string.Format works /// diff --git a/Duplicati/Library/Localization/LocalizationContext.cs b/Duplicati/Library/Localization/LocalizationContext.cs index d2162bd05..9fed23cbb 100644 --- a/Duplicati/Library/Localization/LocalizationContext.cs +++ b/Duplicati/Library/Localization/LocalizationContext.cs @@ -1,26 +1,26 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using Duplicati.Library.Logging; using System; +using System.Threading; namespace Duplicati.Library.Localization { @@ -29,10 +29,14 @@ namespace Duplicati.Library.Localization /// internal class LocalizationContext : IDisposable { + /// + /// The current culture, using an async local to keep the call context + /// + private static readonly AsyncLocal _currentCulture = new(); /// /// The previous context /// - private readonly object m_prev; + private readonly string m_prev; /// /// Flag to prevent double dispose @@ -45,12 +49,17 @@ namespace Duplicati.Library.Localization /// The localization to use. public LocalizationContext(System.Globalization.CultureInfo ci) { - m_prev = CallContext.GetData(LocalizationService.LOGICAL_CONTEXT_KEY) as string; - CallContext.SetData(LocalizationService.LOGICAL_CONTEXT_KEY, ci.Name); + m_prev = _currentCulture.Value; + _currentCulture.Value = ci.Name; m_isDisposed = false; } + /// + /// Returns the current culture value for the call context + /// + public static string Current => _currentCulture.Value; + /// /// Releases all resource used by the object. /// @@ -64,7 +73,7 @@ namespace Duplicati.Library.Localization { if (!m_isDisposed) { - CallContext.SetData(LocalizationService.LOGICAL_CONTEXT_KEY, m_prev); + _currentCulture.Value = m_prev; m_isDisposed = true; } } diff --git a/Duplicati/Library/Localization/LocalizationService.cs b/Duplicati/Library/Localization/LocalizationService.cs index b86b67182..636a75217 100644 --- a/Duplicati/Library/Localization/LocalizationService.cs +++ b/Duplicati/Library/Localization/LocalizationService.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -24,7 +24,6 @@ using System.Linq; using System.Globalization; using System.Collections.Generic; using System.Text.RegularExpressions; -using Duplicati.Library.Localization.Short; using Duplicati.Library.Logging; namespace Duplicati.Library.Localization @@ -39,11 +38,6 @@ namespace Duplicati.Library.Localization /// private static readonly Dictionary Services = new Dictionary(); - /// - /// The key for accessing logical context - /// - internal const string LOGICAL_CONTEXT_KEY = "DUPLICATI_LOCALIZATION_CULTURE_CONTEXT"; - /// /// Regular expression to match a locale /// @@ -97,13 +91,27 @@ namespace Duplicati.Library.Localization { get { - var lc = CallContext.GetData(LOGICAL_CONTEXT_KEY) as string; + var lc = LocalizationContext.Current; if (!string.IsNullOrWhiteSpace(lc)) return Get(new CultureInfo(lc)); return Get(CultureInfo.CurrentCulture); } } + /// + /// Gets a localization provider with the current UI language + /// + public static ILocalizationService CurrentUI + { + get + { + var lc = LocalizationContext.Current; + if (!string.IsNullOrWhiteSpace(lc)) + return Get(new CultureInfo(lc)); + return Get(CultureInfo.CurrentUICulture); + } + } + /// /// Gets a localization provider with the OS install language /// @@ -147,7 +155,7 @@ namespace Duplicati.Library.Localization /// /// Returns true if the culture has localization support /// - public static Boolean isCultureSupported(CultureInfo culture) + public static bool IsCultureSupported(CultureInfo culture) { return MoLocalizationService.isCultureSupported(culture); } diff --git a/Duplicati/Library/Localization/MoLocalizationService.cs b/Duplicati/Library/Localization/MoLocalizationService.cs index e0a81fbbb..a5983cf77 100644 --- a/Duplicati/Library/Localization/MoLocalizationService.cs +++ b/Duplicati/Library/Localization/MoLocalizationService.cs @@ -75,6 +75,7 @@ namespace Duplicati.Library.Localization /// The culture to find. public MoLocalizationService(CultureInfo ci) { + Culture = ci; var filenames = new string[] { // Load the specialized version first string.Format("localization-{0}.mo", ci.Name.Replace('-', '_')), @@ -117,6 +118,11 @@ namespace Duplicati.Library.Localization } } + /// + /// Gets the culture of the localization + /// + public CultureInfo Culture { get; } + /// /// Pre-processes the message to use Linux line endings /// diff --git a/Duplicati/Library/Localization/MockLocalizationService.cs b/Duplicati/Library/Localization/MockLocalizationService.cs index d0e894cf9..1e2ee86e9 100644 --- a/Duplicati/Library/Localization/MockLocalizationService.cs +++ b/Duplicati/Library/Localization/MockLocalizationService.cs @@ -1,31 +1,35 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; -using System.Collections.Generic; +using System.Globalization; namespace Duplicati.Library.Localization { public class MockLocalizationService : ILocalizationService { + /// + /// The culture of the localization service + /// + public CultureInfo Culture => CultureInfo.InvariantCulture; + /// /// Localizes the string similar to how string.Format works /// diff --git a/Duplicati/Library/Logging/CallContext.cs b/Duplicati/Library/Logging/CallContext.cs deleted file mode 100644 index dfb6e8f36..000000000 --- a/Duplicati/Library/Logging/CallContext.cs +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. -using System.Collections.Concurrent; -using System.Threading; - -namespace Duplicati.Library.Logging -{ - /// - /// Provides a way to set contextual data that flows with the call and - /// async context of a test or invocation. - /// - public static class CallContext - { - static ConcurrentDictionary> state = new ConcurrentDictionary>(); - - /// - /// Stores a given object and associates it with the specified name. - /// - /// The name with which to associate the new item in the call context. - /// The object to store in the call context. - public static void SetData(string name, object data) => - state.GetOrAdd(name, _ => new AsyncLocal()).Value = data; - - /// - /// Retrieves an object with the specified name from the . - /// - /// The name of the item in the call context. - /// The object in the call context associated with the specified name, or if not found. - public static object GetData(string name) => - state.TryGetValue(name, out AsyncLocal data) ? data.Value : null; - } -} \ No newline at end of file diff --git a/Duplicati/Library/Logging/Log.cs b/Duplicati/Library/Logging/Log.cs index bea2236ab..d53a9de3f 100644 --- a/Duplicati/Library/Logging/Log.cs +++ b/Duplicati/Library/Logging/Log.cs @@ -1,27 +1,27 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; -using System.Text; +using System.Threading; namespace Duplicati.Library.Logging { @@ -72,18 +72,13 @@ namespace Duplicati.Library.Logging /// /// The key used to assign the current scope into the current call-context /// - private const string LOGICAL_CONTEXT_KEY = "Duplicati:LoggingEntry"; + private static readonly AsyncLocal _currentScope = new AsyncLocal(); /// /// The root scope /// private static readonly LogScope m_root = new LogScope(null, new LogTagFilter(LogMessageType.Error, null, null), null, true); - /// - /// The stored log instances - /// - private static readonly Dictionary m_log_instances = new Dictionary(); - /// /// Static lock object to provide thread safe logging /// @@ -216,15 +211,15 @@ namespace Duplicati.Library.Logging WriteMessage(LogMessageType.Verbose, tag, id, null, message, arguments); } - /// + /// /// Writes a verbose message to the current log destination /// /// The message to write /// The tag-type for this message /// The message id - /// The exception to log + /// The exception to log /// The message format arguments - public static void WriteVerboseMessage(string tag, string id, Exception ex, string message, params object[] arguments) + public static void WriteVerboseMessage(string tag, string id, Exception ex, string message, params object[] arguments) { WriteMessage(LogMessageType.Verbose, tag, id, ex, message, arguments); } @@ -428,11 +423,10 @@ namespace Duplicati.Library.Logging /// The scope to finish. internal static void CloseScope(LogScope scope) { - lock(m_lock) + lock (m_lock) { if (CurrentScope == scope && scope != m_root) CurrentScope = scope.Parent; - m_log_instances.Remove(scope.InstanceID); } } @@ -444,32 +438,12 @@ namespace Duplicati.Library.Logging get { lock (m_lock) - { - var cur = CallContext.GetData(LOGICAL_CONTEXT_KEY) as string; - if (cur == null || cur == m_root.InstanceID) - return m_root; - - LogScope sc; - if (!m_log_instances.TryGetValue(cur, out sc)) - throw new Exception("Unable to find log in lookup table, this may be caused by attempting to transport call contexts between AppDomains (e.g. with remoting calls)"); - - return sc; - } + return _currentScope.Value ?? m_root; } private set { lock (m_lock) - { - if (value != null) - { - m_log_instances[value.InstanceID] = value; - CallContext.SetData(LOGICAL_CONTEXT_KEY, value.InstanceID); - } - else - { - CallContext.SetData(LOGICAL_CONTEXT_KEY, null); - } - } + _currentScope.Value = value; } } } diff --git a/Duplicati/Library/Main/Backend/BackendManager.Handler.cs b/Duplicati/Library/Main/Backend/BackendManager.Handler.cs index 62c5fd4eb..c43895250 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.Handler.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.Handler.cs @@ -415,7 +415,7 @@ partial class BackendManager } } - context.Statwriter.SendEvent(op.Operation, retries < maxRetries ? BackendEventType.Retrying : BackendEventType.Failed, op.RemoteFilename, op.Size); + context.Statwriter.SendEvent(op.Operation, retries <= maxRetries ? BackendEventType.Retrying : BackendEventType.Failed, op.RemoteFilename, op.Size); // Check if we can recover from the error var recovered = false; @@ -428,14 +428,14 @@ partial class BackendManager } // We did not recover, so wait or give up - if (!recovered && retries < maxRetries && retryDelay.Ticks != 0) + if (!recovered && retries <= maxRetries && retryDelay.Ticks != 0) { var delay = Library.Utility.Utility.GetRetryDelay(retryDelay, retries, retryWithExponentialBackoff); await Task.Delay(delay, context.TaskReader.ProgressToken).ConfigureAwait(false); } } - } while (retries < maxRetries); + } while (retries <= maxRetries); // If we have a last exception, we failed if (lastException != null) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 93a8e3222..ba4165515 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -530,6 +530,7 @@ namespace Duplicati.Library.Main.Operation if (result.BackendWriter != null) { result.BackendWriter.KnownFileCount = database.GetRemoteVolumes().Count(); + result.BackendWriter.KnownFilesets = database.GetRemoteVolumes().Count(x => x.Type == RemoteVolumeType.Files); result.BackendWriter.KnownFileSize = database.GetRemoteVolumes().Select(x => Math.Max(0, x.Size)).Sum(); result.BackendWriter.UnknownFileCount = 0; diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index f1d10fdfb..37cbdbd88 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -234,6 +234,7 @@ namespace Duplicati.Library.Main.Operation select n).ToList(); log.KnownFileCount = remotelist.Count; + log.KnownFilesets = filesets.Count; long knownFileSize = remotelist.Select(x => Math.Max(0, x.File.Size)).Sum(); log.KnownFileSize = knownFileSize; log.UnknownFileCount = unknownlist.Count; diff --git a/Duplicati/Library/Main/ResultClasses.cs b/Duplicati/Library/Main/ResultClasses.cs index 846332274..25123a65d 100644 --- a/Duplicati/Library/Main/ResultClasses.cs +++ b/Duplicati/Library/Main/ResultClasses.cs @@ -99,6 +99,7 @@ namespace Duplicati.Library.Main public long UnknownFileCount { get; set; } public long KnownFileCount { get; set; } public long KnownFileSize { get; set; } + public long KnownFilesets { get; set; } public DateTime LastBackupDate { get; set; } public long BackupListCount { get; set; } public long TotalQuotaSpace { get; set; } diff --git a/Duplicati/Library/RemoteControl/ExchangeDataTypes.cs b/Duplicati/Library/RemoteControl/ExchangeDataTypes.cs index bb4003848..6b6cf7fa4 100644 --- a/Duplicati/Library/RemoteControl/ExchangeDataTypes.cs +++ b/Duplicati/Library/RemoteControl/ExchangeDataTypes.cs @@ -45,6 +45,10 @@ internal enum MessageType /// Command, /// + /// A control message + /// + Control, + /// /// An unknown command /// Unknown @@ -56,7 +60,9 @@ internal enum MessageType /// The client token /// The client public key /// The version of the client -public record AuthMessage(string Token, string PublicKey, string ClientVersion, int ProtocolVersion); +/// The protocol version of the client +/// The optional metadata to send +public record AuthMessage(string Token, string PublicKey, string ClientVersion, int ProtocolVersion, Dictionary? Metadata); /// /// A message authentication response @@ -75,6 +81,35 @@ internal sealed record AuthResultMessage(bool? Accepted, bool? WillReplaceToken, /// The version of the server public sealed record WelcomeMessage(string PublicKeyHash, string MachineName, string ServerVersion, IEnumerable SupportedProtocolVersions); +/// +/// A message with a control request +/// +/// The control command being requested +/// The parameters for the command +public sealed record ControlRequestMessage(string Command, Dictionary Parameters) +{ + /// + /// The configure report url command + /// + public const string ConfigureReportUrlSet = "configure-report-url:set"; + /// + /// A control message to return if the report url is configured + /// + public const string ConfigureReportUrlGet = "configure-report-url:get"; + /// + /// The url parameter for the configure report url command + /// + public const string ConfigureReportUrlParameter = "url"; + +} +/// +/// A message with a control response +/// +/// Whether the command was successful +/// The output of the command +/// The error message if the command failed +public sealed record ControlResponseMessage(bool Success, Dictionary? Output, string? ErrorMessage); + /// /// A message to send a command /// @@ -180,6 +215,7 @@ internal sealed record EnvelopedMessage "auth" => MessageType.Auth, "pong" => MessageType.Pong, "command" => MessageType.Command, + "control" => MessageType.Control, _ => MessageType.Unknown }; } diff --git a/Duplicati/Library/RemoteControl/KeepRemoteConnection.cs b/Duplicati/Library/RemoteControl/KeepRemoteConnection.cs index c2ad64e63..a2d25790f 100644 --- a/Duplicati/Library/RemoteControl/KeepRemoteConnection.cs +++ b/Duplicati/Library/RemoteControl/KeepRemoteConnection.cs @@ -19,10 +19,12 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +using System.Diagnostics; using System.Net; using System.Net.Http.Headers; using System.Security.Cryptography; using System.Text.Json; +using Duplicati.Library.AutoUpdater; using Duplicati.Library.Logging; namespace Duplicati.Library.RemoteControl; @@ -133,12 +135,20 @@ public class KeepRemoteConnection : IDisposable /// private RSA? _serverPublicKey; + /// + /// The callback to call when connecting + /// + private readonly Func, Task>> _onConnect; /// /// The callback to call when rekeying /// private readonly Func _onReKey; /// - /// The callback to call when a message is received + /// The callback to call when a control message is received + /// + private readonly Func _onControl; + /// + /// The callback to call when a command message is received /// private readonly Func _onMessage; @@ -168,16 +178,32 @@ public class KeepRemoteConnection : IDisposable /// /// The url to use /// The JWT token to use + /// The certificate url to use /// The server keys to use /// The token to cancel the connection - private KeepRemoteConnection(string serverUrl, string JWT, string certificateUrl, IEnumerable serverKeys, CancellationToken cancellationToken, Func onReKey, Func onMessage) + /// The callback to call when connecting + /// The callback to call when rekeying + /// The callback to call when a control message is received + /// The callback to call when a command message is received + private KeepRemoteConnection( + string serverUrl, + string JWT, + string certificateUrl, + IEnumerable serverKeys, + CancellationToken cancellationToken, + Func, Task>> onConnect, + Func onReKey, + Func onControl, + Func onMessage) { _serverUrl = serverUrl; _certificateUrl = certificateUrl; _token = JWT; _serverKeys = serverKeys; _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + _onConnect = onConnect; _onReKey = onReKey; + _onControl = onControl; _onMessage = onMessage; _client = new Websocket.Client.WebsocketClient(new Uri(serverUrl)); @@ -300,13 +326,27 @@ public class KeepRemoteConnection : IDisposable } _state = ConnectionState.WelcomeReceived; + + // Prepare basic metadata and allow additional metadata to be added + var metadata = await _onConnect(new Dictionary() { + { "client-version", UpdaterManager.SelfVersion?.Version ?? "0.0.0.0" }, + { "client-id", ClientId }, + { "client-uptime", (DateTime.Now - Process.GetCurrentProcess().StartTime).ToString() }, + { "machine-name", DataFolderManager.MachineName }, + { "machine-id", DataFolderManager.MachineID }, + { "install-id", DataFolderManager.InstallID }, + { "machine-os", UpdaterManager.OperatingSystemName }, + { "package-id", UpdaterManager.PackageTypeId }, + }); + SendEnvelope( welcomeEnvelope.RespondWith( new AuthMessage( _token, ClientKey.ExportRSAPublicKeyPem(), - AutoUpdater.UpdaterManager.SelfVersion?.Version ?? "0.0.0", - PROTOCOL_VERSION + UpdaterManager.SelfVersion?.Version ?? "0.0.0.0", + PROTOCOL_VERSION, + metadata ), "auth" ), @@ -352,6 +392,13 @@ public class KeepRemoteConnection : IDisposable )); break; + case MessageType.Control: + await _onControl(new ControlMessage( + envelope.GetPayload(), + response => SendEnvelope(envelope.RespondWith(response)) + )); + break; + default: throw new ProtocolViolationException("Unexpected message"); } @@ -403,13 +450,24 @@ public class KeepRemoteConnection : IDisposable /// The certificate url to use /// The server keys to use /// The token to cancel the connection + /// The callback to call when connecting /// The callback to call when rekeying - /// The callback to call when a message is received + /// The callback to call when a control message is received + /// The callback to call when a command message is received /// - public static Task Start(string serverUrl, string JWT, string certificateUrl, IEnumerable serverKeys, CancellationToken cancellationToken, Func onReKey, Func onMessage) + public static Task Start( + string serverUrl, + string JWT, + string certificateUrl, + IEnumerable serverKeys, + CancellationToken cancellationToken, + Func, Task>> onConnect, + Func onReKey, + Func onControl, + Func onMessage) => Task.Run(async () => { - using var connection = new KeepRemoteConnection(serverUrl, JWT, certificateUrl, serverKeys, cancellationToken, onReKey, onMessage); + using var connection = new KeepRemoteConnection(serverUrl, JWT, certificateUrl, serverKeys, cancellationToken, onConnect, onReKey, onControl, onMessage); await connection._runnerTask; }); @@ -477,14 +535,24 @@ public class KeepRemoteConnection : IDisposable /// /// The url to use /// The JWT token to use - /// + /// The certificate url to use /// The server keys to use + /// The callback to call when connecting /// The callback to call when rekeying + /// The callback to call when a control message is received /// The callback to call when a message is received - /// The token to cancel the connection /// The connection object - public static KeepRemoteConnection CreateRemoteListener(string serverUrl, string JWT, string certificateUrl, IEnumerable serverKeys, CancellationToken cancellationToken, Func onReKey, Func onMessage) - => new KeepRemoteConnection(serverUrl, JWT, certificateUrl, serverKeys, cancellationToken, onReKey, onMessage); + public static KeepRemoteConnection CreateRemoteListener( + string serverUrl, + string JWT, + string certificateUrl, + IEnumerable serverKeys, + CancellationToken cancellationToken, + Func, Task>> onConnect, + Func onReKey, + Func onControl, + Func onMessage) + => new KeepRemoteConnection(serverUrl, JWT, certificateUrl, serverKeys, cancellationToken, onConnect, onReKey, onControl, onMessage); /// /// Requests a certificate refresh @@ -587,4 +655,38 @@ public class KeepRemoteConnection : IDisposable } } } + + /// + /// A wrapper for allowing external code to handle a control message + /// + public sealed class ControlMessage + { + /// + /// The callback method that will receive the response + /// + private readonly Func _respondCommand; + /// + /// The command request message + /// + public ControlRequestMessage ControlRequestMessage { get; } + + /// + /// Creates a new command message + /// + /// The command request message + /// The callback method that will receive the response + public ControlMessage(ControlRequestMessage controlRequestMessage, Func respondCommand) + { + ControlRequestMessage = controlRequestMessage; + _respondCommand = respondCommand; + } + + /// + /// Responds to the command message + /// + /// The response to send + /// True if the response was sent + public bool Respond(ControlResponseMessage response) + => _respondCommand(response); + } } diff --git a/Duplicati/Library/RestAPI/Database/Connection.cs b/Duplicati/Library/RestAPI/Database/Connection.cs index f9aadf1cd..ea15e5188 100644 --- a/Duplicati/Library/RestAPI/Database/Connection.cs +++ b/Duplicati/Library/RestAPI/Database/Connection.cs @@ -38,8 +38,8 @@ namespace Duplicati.Server.Database { public class Connection : IDisposable { - private readonly System.Data.IDbConnection m_connection; - private readonly System.Data.IDbCommand m_errorcmd; + private readonly IDbConnection m_connection; + private readonly IDbCommand m_errorcmd; public readonly object m_lock = new object(); public const int ANY_BACKUP_ID = -1; public const int SERVER_SETTINGS_ID = -2; @@ -65,7 +65,7 @@ namespace Duplicati.Server.Database ]) .ToHashSet(StringComparer.OrdinalIgnoreCase); - public Connection(System.Data.IDbConnection connection, bool disableFieldEncryption, EncryptedFieldHelper.KeyInstance? key) + public Connection(IDbConnection connection, bool disableFieldEncryption, EncryptedFieldHelper.KeyInstance? key) { m_encryptSensitiveFields = !disableFieldEncryption; m_key = key; @@ -141,7 +141,7 @@ namespace Duplicati.Server.Database } } - public void ExecuteWithCommand(Action f) + public void ExecuteWithCommand(Action f) { lock (m_lock) using (var cmd = m_connection.CreateCommand()) @@ -214,7 +214,7 @@ namespace Duplicati.Server.Database .ToDictionary((k) => k.Key, (k) => k.Value); } - internal void SetMetadata(IDictionary values, long id, System.Data.IDbTransaction? transaction) + internal void SetMetadata(IDictionary values, long id, IDbTransaction? transaction) { lock (m_lock) using (var tr = transaction == null ? m_connection.BeginTransaction() : null) @@ -246,7 +246,7 @@ namespace Duplicati.Server.Database .ToArray(); } - internal void SetFilters(IEnumerable values, long id, System.Data.IDbTransaction? transaction = null) + internal void SetFilters(IEnumerable values, long id, IDbTransaction? transaction = null) { lock (m_lock) using (var tr = transaction == null ? m_connection.BeginTransaction() : null) @@ -279,7 +279,7 @@ namespace Duplicati.Server.Database .ToArray(); } - internal void SetSettings(IEnumerable values, long id, System.Data.IDbTransaction? transaction = null) + internal void SetSettings(IEnumerable values, long id, IDbTransaction? transaction = null) { lock (m_lock) using (var tr = transaction == null ? m_connection.BeginTransaction() : null) @@ -319,7 +319,7 @@ namespace Duplicati.Server.Database .ToArray(); } - internal void SetSources(IEnumerable values, long id, System.Data.IDbTransaction transaction) + internal void SetSources(IEnumerable values, long id, IDbTransaction transaction) { lock (m_lock) using (var tr = transaction == null ? m_connection.BeginTransaction() : null) @@ -715,7 +715,7 @@ namespace Duplicati.Server.Database } } - private void AddOrUpdateSchedule(ISchedule item, System.Data.IDbTransaction tr) + private void AddOrUpdateSchedule(ISchedule item, IDbTransaction tr) { lock (m_lock) { @@ -981,7 +981,7 @@ namespace Duplicati.Server.Database .ToDictionary(x => x.Key, x => x.Last().Value); } - public void SetUISettings(string scheme, IDictionary values, System.Data.IDbTransaction? transaction = null) + public void SetUISettings(string scheme, IDictionary values, IDbTransaction? transaction = null) { lock (m_lock) using (var tr = transaction == null ? m_connection.BeginTransaction() : null) @@ -1002,7 +1002,7 @@ namespace Duplicati.Server.Database } } - public void UpdateUISettings(string scheme, IDictionary values, System.Data.IDbTransaction? transaction = null) + public void UpdateUISettings(string scheme, IDictionary values, IDbTransaction? transaction = null) { lock (m_lock) using (var tr = transaction == null ? m_connection.BeginTransaction() : null) @@ -1066,24 +1066,24 @@ namespace Duplicati.Server.Database } } - private static DateTime ConvertToDateTime(System.Data.IDataReader rd, int index) + private static DateTime ConvertToDateTime(IDataReader rd, int index) { var unixTime = ConvertToInt64(rd, index); return unixTime == 0 ? new DateTime(0) : Library.Utility.Utility.EPOCH.AddSeconds(unixTime); } - private static bool ConvertToBoolean(System.Data.IDataReader rd, int index) + private static bool ConvertToBoolean(IDataReader rd, int index) { return ConvertToInt64(rd, index) == 1; } - private static string? ConvertToString(System.Data.IDataReader rd, int index) + private static string? ConvertToString(IDataReader rd, int index) { var r = rd.GetValue(index); return r == null || r == DBNull.Value ? null : r.ToString(); } - private static long ConvertToInt64(System.Data.IDataReader rd, int index) + private static long ConvertToInt64(IDataReader rd, int index) { try { @@ -1097,20 +1097,20 @@ namespace Duplicati.Server.Database return -1; } - private static long ExecuteScalarInt64(System.Data.IDbCommand cmd, long defaultValue = -1) + private static long ExecuteScalarInt64(IDbCommand cmd, long defaultValue = -1) { using (var rd = cmd.ExecuteReader()) return rd.Read() ? ConvertToInt64(rd, 0) : defaultValue; } - private static string? ExecuteScalarString(System.Data.IDbCommand cmd) + private static string? ExecuteScalarString(IDbCommand cmd) { using (var rd = cmd.ExecuteReader()) return rd.Read() ? ConvertToString(rd, 0) : null; } - private object? ConvertToEnum(Type enumType, System.Data.IDataReader rd, int index, object? @default) + private object? ConvertToEnum(Type enumType, IDataReader rd, int index, object? @default) { try { @@ -1124,13 +1124,13 @@ namespace Duplicati.Server.Database } // Overloaded function for legacy functionality - private bool DeleteFromDb(string tablename, long id, System.Data.IDbTransaction? transaction = null) + private bool DeleteFromDb(string tablename, long id, IDbTransaction? transaction = null) { return DeleteFromDb(tablename, id, "ID", transaction); } // New function that allows to delete rows from tables with arbitrary identifier values (e.g. ID or BackupID) - private bool DeleteFromDb(string tablename, long id, string identifier, System.Data.IDbTransaction? transaction = null) + private bool DeleteFromDb(string tablename, long id, string identifier, IDbTransaction? transaction = null) { if (transaction == null) { @@ -1158,14 +1158,14 @@ namespace Duplicati.Server.Database } } - private static IEnumerable Read(System.Data.IDbCommand cmd, Func f) + private static IEnumerable Read(IDbCommand cmd, Func f) { using (var rd = cmd.ExecuteReader()) while (rd.Read()) yield return f(rd); } - private static IEnumerable Read(System.Data.IDataReader rd, Func f) + private static IEnumerable Read(IDataReader rd, Func f) { while (rd.Read()) yield return f(); @@ -1226,7 +1226,7 @@ namespace Duplicati.Server.Database }, sql, args); } - private void OverwriteAndUpdateDb(System.Data.IDbTransaction? transaction, string? deleteSql, object[]? deleteArgs, IEnumerable values, bool updateExisting) + private void OverwriteAndUpdateDb(IDbTransaction? transaction, string? deleteSql, object[]? deleteArgs, IEnumerable values, bool updateExisting) { var properties = GetORMFields(); var idfield = properties.FirstOrDefault(x => x.Name == "ID") @@ -1286,7 +1286,7 @@ namespace Duplicati.Server.Database } } - private IEnumerable ReadFromDb(Func f, string sql, params object?[]? args) + private IEnumerable ReadFromDb(Func f, string sql, params object?[]? args) { using (var cmd = m_connection.CreateCommand()) { @@ -1297,7 +1297,7 @@ namespace Duplicati.Server.Database } } - private void OverwriteAndUpdateDb(System.Data.IDbTransaction? transaction, string? deleteSql, object[]? deleteArgs, IEnumerable values, string insertSql, Func f) + private void OverwriteAndUpdateDb(IDbTransaction? transaction, string? deleteSql, object[]? deleteArgs, IEnumerable values, string insertSql, Func f) { using (var cmd = m_connection.CreateCommand()) { diff --git a/Duplicati/Library/RestAPI/Database/ServerSettings.cs b/Duplicati/Library/RestAPI/Database/ServerSettings.cs index 770c5dd39..d90d463b5 100644 --- a/Duplicati/Library/RestAPI/Database/ServerSettings.cs +++ b/Duplicati/Library/RestAPI/Database/ServerSettings.cs @@ -71,6 +71,7 @@ namespace Duplicati.Server.Database public const string TIMEZONE_OPTION = "server-timezone"; public const string PAUSED_UNTIL = "paused-until"; public const string LAST_UPDATE_CHECK_VERSION = "last-update-check-version"; + public const string ADDITIONAL_REPORT_URL = "additional-report-url"; } private readonly Dictionary settings; @@ -847,6 +848,20 @@ namespace Duplicati.Server.Database SaveSettings(); } } + + public string? AdditionalReportUrl + { + get + { + return settings[CONST.ADDITIONAL_REPORT_URL]; + } + set + { + lock (databaseConnection.m_lock) + settings[CONST.ADDITIONAL_REPORT_URL] = value; + SaveSettings(); + } + } } } diff --git a/Duplicati/Library/RestAPI/Runner.cs b/Duplicati/Library/RestAPI/Runner.cs index 2014358c8..6a8c63ffc 100644 --- a/Duplicati/Library/RestAPI/Runner.cs +++ b/Duplicati/Library/RestAPI/Runner.cs @@ -736,6 +736,7 @@ namespace Duplicati.Server backup.Metadata["TargetFilesSize"] = r.KnownFileSize.ToString(); backup.Metadata["TargetFilesCount"] = r.KnownFileCount.ToString(); + backup.Metadata["TargetFilesetsCount"] = r.KnownFilesets.ToString(); backup.Metadata["TargetSizeString"] = Duplicati.Library.Utility.Utility.FormatSizeString(r.KnownFileSize); } } @@ -919,6 +920,18 @@ namespace Duplicati.Server // The server hangs if the module is enabled as there is no console attached DisableModule("console-password-input", options); + // Patch in additional report urls + var additionalReportUrl = FIXMEGlobal.DataConnection.ApplicationSettings.AdditionalReportUrl; + if (!string.IsNullOrWhiteSpace(additionalReportUrl)) + { + options["send-http-json-urls"] = string.Join(";", + new[] { + options.GetValueOrDefault("send-http-json-urls"), + additionalReportUrl + }.Where(x => !string.IsNullOrWhiteSpace(x)) + ); + } + return options; } diff --git a/Duplicati/Library/RestAPI/Scheduler.cs b/Duplicati/Library/RestAPI/Scheduler.cs index 037af95cf..8940691fd 100644 --- a/Duplicati/Library/RestAPI/Scheduler.cs +++ b/Duplicati/Library/RestAPI/Scheduler.cs @@ -143,7 +143,7 @@ namespace Duplicati.Server get { lock (m_lock) - return m_schedule.ToList(); + return (m_schedule ?? []).ToList(); } } diff --git a/Duplicati/Library/Utility/CallContextSettings.cs b/Duplicati/Library/Utility/CallContextSettings.cs index c78bea518..66a1824cf 100644 --- a/Duplicati/Library/Utility/CallContextSettings.cs +++ b/Duplicati/Library/Utility/CallContextSettings.cs @@ -1,32 +1,32 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Net.Security; using System.Security.Cryptography.X509Certificates; - +using System.Threading; using Duplicati.Library.Interface; -using Duplicati.Library.Logging; namespace Duplicati.Library.Utility { @@ -81,12 +81,9 @@ namespace Duplicati.Library.Utility } set { - lock (CallContextSettings._lock) - { - var st = CallContextSettings.Settings; - st.Tempdir = value; - CallContextSettings.Settings = st; - } + var st = CallContextSettings.Settings; + st.Tempdir = value; + CallContextSettings.Settings = st; } } @@ -228,20 +225,58 @@ namespace Duplicati.Library.Utility public static class CallContextSettings { /// - /// The settings that are stored in the call context, which are not serializable + /// The call context settings for OAuth /// - private static readonly Dictionary _settings = new Dictionary(); + private static readonly AsyncLocal _settings = new AsyncLocal(); /// - /// The context setting types that are used and need their private setting key - /// to prevent overwriting each others settings. + /// The instances that are currently active /// - private static string contextSettingsType = null; + private static readonly ConcurrentDictionary _instances = new ConcurrentDictionary(); /// - /// Lock for protecting the dictionary + /// Disposable class for setting call context settings /// - public static readonly object _lock = new object(); + /// The type of the settings + private sealed class Disposer : IDisposable + { + /// + /// A flag indicating if the object has been disposed + /// + private bool m_disposed = false; + /// + /// The previous value + /// + private T m_prev; + /// + /// The current value + /// + private T m_current; + + /// + /// Initializes a new instance of the class. + /// + /// The current server URL + /// The previous server URL + public Disposer(T current, T prev) + { + _settings.Value = m_current = current; + this.m_prev = prev; + _instances.TryAdd(current, null); + } + + /// + public void Dispose() + { + if (m_disposed) + return; + GC.SuppressFinalize(this); + m_disposed = true; + if (_settings.Value.Equals(m_current)) + _settings.Value = m_prev; + _instances.TryRemove(m_current, out _); + } + } /// /// Gets or sets the values in the current call context @@ -249,40 +284,8 @@ namespace Duplicati.Library.Utility /// The settings. public static T Settings { - get - { - var key = ContextID; - if (string.IsNullOrWhiteSpace(key)) - return default(T); - - - if (!_settings.TryGetValue(key, out T res)) - lock (_lock) // if not present but context id is not null, wait - if (!_settings.TryGetValue(key, out res)) - return default(T); - - return res; - } - set - { - var key = ContextID; - if (!string.IsNullOrWhiteSpace(key)) - lock (_lock) - _settings[key] = value; - } - } - - /// - /// Starts the context wit a default value. - /// - /// The disposable handle for the context. - /// The initial value. - public static IDisposable StartContext(string settingsKey, T initial = default(T)) - { - contextSettingsType = settingsKey; - var res = new ContextGuard(); - Settings = initial; - return res; + get => _settings.Value; + set => _settings.Value = value; } /// @@ -291,72 +294,10 @@ namespace Duplicati.Library.Utility /// The disposable handle for the context. /// The initial value. public static IDisposable StartContext(T initial = default(T)) - { - return StartContext(typeof(T).ToString(), initial); - } + => new Disposer(initial, _settings.Value); - /// - /// Gets all instances currently registered - /// - internal static T[] GetAllInstances() - { - lock (_lock) - return _settings.Values.ToArray(); - } + public static IEnumerable GetAllInstances() + => _instances.Keys.ToList(); - /// - /// Help class for providing a way to release resources associated with a call context - /// - private class ContextGuard : IDisposable - { - /// - /// Randomly generated identifier - /// - private readonly string ID = Guid.NewGuid().ToString(); - - /// - /// Initializes a new instance of the - /// class, - /// and sets up the call context - /// - public ContextGuard() - { - CallContext.SetData(contextSettingsType, ID); - } - - /// - /// Releases all resource used by the - /// object. - /// - /// Call when you are finished using the - /// . The - /// method leaves the in an - /// unusable state. After calling , you must release all references to the - /// so the garbage collector - /// can reclaim the memory that the - /// was occupying. - public void Dispose() - { - lock (_lock) - { - // Release the resources, if any - _settings.Remove(ID); - } - - CallContext.SetData(contextSettingsType, null); - } - } - - /// - /// Gets the context ID for this call. - /// - /// The context identifier. - private static string ContextID - { - get - { - return contextSettingsType != null ? CallContext.GetData(contextSettingsType) as string : null; - } - } } } diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 038d3adad..5843328db 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -774,6 +774,18 @@ namespace Duplicati.Library.Utility return (T)(object)flags; } + + /// + /// Parses an option with int value, returning the default value if the option is not found or cannot be parsed + /// + /// The set of options to look for the setting in + /// The value to look for in the settings + /// default value + /// + public static int ParseIntOption(IReadOnlyDictionary options, string value, int @default) + { + return options.TryGetValue(value, out var opt) && int.TryParse(opt ?? string.Empty, out var result) ? result : @default; + } /// /// Converts a sequence of bytes to a hex string diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs index 16fac2c5d..6e27637bf 100644 --- a/Duplicati/UnitTest/DisruptionTests.cs +++ b/Duplicati/UnitTest/DisruptionTests.cs @@ -25,6 +25,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Interface; +using Duplicati.Library.Logging; using Duplicati.Library.Main; using Duplicati.Library.Main.Database; using Duplicati.Library.Main.Volumes; @@ -50,11 +51,12 @@ namespace Duplicati.UnitTest } } - private async Task RunPartialBackup(Controller controller) + private async Task<(IBackupResults Result, List ModifiedFiles)> RunPartialBackup(Controller controller) { this.ModifySourceFiles(); var stopped = new TaskCompletionSource(); + var modifiedFiles = new List(); controller.OnOperationStarted = r => { var pv = r as ITaskControlProvider; @@ -63,7 +65,8 @@ namespace Duplicati.UnitTest #if DEBUG pv.TaskControl.TestMethodCallback = (path) => { - if (path.EndsWith(this.fileSizes[2] + "MB")) + modifiedFiles.Add(path); + if (modifiedFiles.Count == 3) { Thread.Sleep(500); controller.Stop(); @@ -80,7 +83,7 @@ namespace Duplicati.UnitTest if (t != stopped.Task) throw new Exception("Backup task completed before we could stop it"); - return await backupTask.ConfigureAwait(false); + return (await backupTask.ConfigureAwait(false), modifiedFiles); } [SetUp] @@ -121,7 +124,7 @@ namespace Duplicati.UnitTest // Run a partial backup. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false); + (var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false); Assert.AreEqual(0, backupResults.Errors.Count()); Assert.AreEqual(1, backupResults.Warnings.Count()); } @@ -240,7 +243,7 @@ namespace Duplicati.UnitTest DateTime thirdBackupTime; using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false); + (var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false); Assert.AreEqual(0, backupResults.Errors.Count()); Assert.AreEqual(1, backupResults.Warnings.Count()); thirdBackupTime = c.List().Filesets.First().Time; @@ -267,7 +270,7 @@ namespace Duplicati.UnitTest // even when all the "recent" backups are partial. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false); + (var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false); Assert.AreEqual(0, backupResults.Errors.Count()); Assert.AreEqual(1, backupResults.Warnings.Count()); DateTime fourthBackupTime = c.List().Filesets.First().Time; @@ -319,7 +322,7 @@ namespace Duplicati.UnitTest // Run a partial backup. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false); + (var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false); Assert.AreEqual(0, backupResults.Errors.Count()); Assert.AreEqual(1, backupResults.Warnings.Count()); } @@ -327,7 +330,7 @@ namespace Duplicati.UnitTest // Run a partial backup. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false); + (var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false); Assert.AreEqual(0, backupResults.Errors.Count()); Assert.AreEqual(1, backupResults.Warnings.Count()); } @@ -347,7 +350,7 @@ namespace Duplicati.UnitTest using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { options["keep-versions"] = "2"; - IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false); + (var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false); Assert.AreEqual(0, backupResults.Errors.Count()); Assert.AreEqual(1, backupResults.Warnings.Count()); DateTime fifthBackupTime = c.List().Filesets.First().Time; @@ -413,7 +416,7 @@ namespace Duplicati.UnitTest // Run a partial backup. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false); + (var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false); Assert.AreEqual(0, backupResults.Errors.Count()); Assert.AreEqual(1, backupResults.Warnings.Count()); @@ -477,7 +480,7 @@ namespace Duplicati.UnitTest DateTime thirdBackupTime; using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false); + (var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false); Assert.AreEqual(0, backupResults.Errors.Count()); Assert.AreEqual(1, backupResults.Warnings.Count()); thirdBackupTime = c.List().Filesets.First().Time; @@ -596,9 +599,11 @@ namespace Duplicati.UnitTest // Run a partial backup. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false); + (var backupResults, var modified) = await this.RunPartialBackup(c).ConfigureAwait(false); Assert.AreEqual(0, backupResults.Errors.Count()); Assert.AreEqual(1, backupResults.Warnings.Count()); + if (backupResults.ModifiedFiles == 0) + throw new Exception($"No files were modified, likely the stop was issued too early, list is: {string.Join(", ", modified)}"); Assert.That(backupResults.ModifiedFiles, Is.GreaterThan(0), "No files were added, likely the stop was issued too early"); Assert.That(backupResults.ModifiedFiles, Is.LessThan(fileSizes.Length), "All files were added, likely the stop was issued too late"); @@ -852,5 +857,318 @@ namespace Duplicati.UnitTest Assert.AreEqual(3, listResults.Filesets.Count()); } } + + private class LogSink : IMessageSink + { + public List StartedFiles { get; } = new List(); + public List CompletedFiles { get; } = new List(); + public void BackendEvent(BackendActionType action, BackendEventType type, string path, long size) + { + if (action == BackendActionType.Put && type == BackendEventType.Started) + { + this.StartedFiles.Add(path); + } + if (action == BackendActionType.Put && type == BackendEventType.Completed) + { + this.CompletedFiles.Add(path); + } + } + + public void SetBackendProgress(IBackendProgress progress) + { + } + + public void SetOperationProgress(IOperationProgress progress) + { + } + + public void WriteMessage(LogEntry entry) + { + } + } + + [Test] + [Category("Disruption")] + public void TestFailedDblockUploadWithOperationCancellation() + { + var testopts = TestOptions; + testopts["number-of-retries"] = "1"; + testopts["dblock-size"] = "10mb"; + + // Make a base backup + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + } + + // Make a new backup that fails uploading a dblock file + ModifySourceFiles(); + + // Deterministic error backend + Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend()); + var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; + var uploadCount = 0; + var failedFile = string.Empty; + + // Fail the dlist upload + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => + { + if (action == DeterministicErrorBackend.BackendAction.PutBefore && remotename.Contains(".dblock.")) + { + // Fail once with a cancellation + if (Interlocked.Increment(ref uploadCount) == 1) + { + failedFile = remotename; + throw new OperationCanceledException(); + } + } + + return false; + }; + + using (var c = new Library.Main.Controller(failtarget, testopts, null)) + { + var sink = new LogSink(); + c.AppendSink(sink); + var res = c.Backup(new string[] { DATAFOLDER }); + if (uploadCount == 0) + Assert.Fail("Upload count was not incremented"); + + Assert.AreEqual(2, c.List().Filesets.Count()); + Assert.AreEqual(7 * 2 + 1, sink.CompletedFiles.Count); // 7 completed (dblock + dindex) + 1 dlist + Assert.AreEqual(7 * 2 + 2, sink.StartedFiles.Count); // 1 retry + Assert.AreEqual(1, res.BackendStatistics.RetryAttempts); + } + + ModifySourceFiles(); + // Create a regular backup + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + } + + // Verify that all is in order + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null)) + { + var r = c.Test(long.MaxValue); + Assert.AreEqual(0, r.Errors.Count()); + Assert.AreEqual(0, r.Warnings.Count()); + Assert.IsFalse(r.Verifications.Any(p => p.Value.Any())); + } + + // Test that we can recreate + var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite"); + if (File.Exists(recreatedDatabaseFile)) + File.Delete(recreatedDatabaseFile); + + testopts["dbpath"] = recreatedDatabaseFile; + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IRepairResults repairResults = c.Repair(); + Assert.AreEqual(0, repairResults.Errors.Count()); + Assert.AreEqual(0, repairResults.Warnings.Count()); + } + + // Check that we have 3 versions + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IListResults listResults = c.List(); + Assert.AreEqual(0, listResults.Errors.Count()); + Assert.AreEqual(0, listResults.Warnings.Count()); + Assert.AreEqual(3, listResults.Filesets.Count()); + } + } + + [Test] + [Category("Disruption")] + public void TestFailedDindexUploadWithOperationCancellation() + { + var testopts = TestOptions; + testopts["number-of-retries"] = "1"; + testopts["dblock-size"] = "10mb"; + + // Make a base backup + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + } + + // Make a new backup that fails uploading a dblock file + ModifySourceFiles(); + + // Deterministic error backend + Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend()); + var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; + var uploadCount = 0; + var failedFile = string.Empty; + + // Fail the dlist upload + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => + { + if (action == DeterministicErrorBackend.BackendAction.PutBefore && remotename.Contains(".dindex.")) + { + // Fail once with a cancellation + if (Interlocked.Increment(ref uploadCount) == 1) + { + failedFile = remotename; + throw new OperationCanceledException(); + } + } + + return false; + }; + + using (var c = new Library.Main.Controller(failtarget, testopts, null)) + { + var sink = new LogSink(); + c.AppendSink(sink); + var res = c.Backup(new string[] { DATAFOLDER }); + if (uploadCount == 0) + Assert.Fail("Upload count was not incremented"); + + Assert.AreEqual(2, c.List().Filesets.Count()); + Assert.AreEqual(7 * 2 + 1, sink.CompletedFiles.Count); // 7 completed (dblock + dindex) + 1 dlist + Assert.AreEqual(7 * 2 + 2, sink.StartedFiles.Count); // 1 retry + Assert.AreEqual(sink.CompletedFiles.Count(x => x.Contains(".dblock.")), sink.CompletedFiles.Count(x => x.Contains(".dindex."))); + Assert.AreEqual(1, res.BackendStatistics.RetryAttempts); + } + + ModifySourceFiles(); + // Create a regular backup + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + } + + // Verify that all is in order + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null)) + { + var r = c.Test(long.MaxValue); + Assert.AreEqual(0, r.Errors.Count()); + Assert.AreEqual(0, r.Warnings.Count()); + Assert.IsFalse(r.Verifications.Any(p => p.Value.Any())); + } + + // Test that we can recreate + var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite"); + if (File.Exists(recreatedDatabaseFile)) + File.Delete(recreatedDatabaseFile); + + testopts["dbpath"] = recreatedDatabaseFile; + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IRepairResults repairResults = c.Repair(); + Assert.AreEqual(0, repairResults.Errors.Count()); + Assert.AreEqual(0, repairResults.Warnings.Count()); + } + + // Check that we have 3 versions + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IListResults listResults = c.List(); + Assert.AreEqual(0, listResults.Errors.Count()); + Assert.AreEqual(0, listResults.Warnings.Count()); + Assert.AreEqual(3, listResults.Filesets.Count()); + } + } + + [Test] + [Category("Disruption")] + public void TestFailedDlistUploadWithOperationCancellation() + { + var testopts = TestOptions; + testopts["number-of-retries"] = "1"; + testopts["dblock-size"] = "10mb"; + + // Make a base backup + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + } + + // Make a new backup that fails uploading a dblock file + ModifySourceFiles(); + + // Deterministic error backend + Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend()); + var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; + var uploadCount = 0; + + // Fail the dlist upload + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => + { + if (action == DeterministicErrorBackend.BackendAction.PutBefore && remotename.Contains(".dlist.")) + { + if (Interlocked.Increment(ref uploadCount) == 1) + throw new OperationCanceledException(); + } + + return false; + }; + + using (var c = new Library.Main.Controller(failtarget, testopts, null)) + { + var res = c.Backup(new string[] { DATAFOLDER }); + var dlistCount = Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).Count(); + if (uploadCount == 0) + Assert.Fail("Upload count was not incremented"); + Assert.AreEqual(2, c.List().Filesets.Count()); + Assert.AreEqual(2, dlistCount); + Assert.AreEqual(1, res.BackendStatistics.RetryAttempts); + } + + ModifySourceFiles(); + // Create a regular backup + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + } + + // Verify that all is in order + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null)) + { + var r = c.Test(long.MaxValue); + Assert.AreEqual(0, r.Errors.Count()); + Assert.AreEqual(0, r.Warnings.Count()); + Assert.IsFalse(r.Verifications.Any(p => p.Value.Any())); + } + + // Test that we can recreate + var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite"); + if (File.Exists(recreatedDatabaseFile)) + File.Delete(recreatedDatabaseFile); + + testopts["dbpath"] = recreatedDatabaseFile; + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IRepairResults repairResults = c.Repair(); + Assert.AreEqual(0, repairResults.Errors.Count()); + Assert.AreEqual(0, repairResults.Warnings.Count()); + } + + // Check that we have 3 versions + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IListResults listResults = c.List(); + Assert.AreEqual(0, listResults.Errors.Count()); + Assert.AreEqual(0, listResults.Warnings.Count()); + Assert.AreEqual(3, listResults.Filesets.Count()); + } + } } } \ No newline at end of file diff --git a/Duplicati/WebserverCore/DayOfWeekStringEnumConverter.cs b/Duplicati/WebserverCore/DayOfWeekStringEnumConverter.cs index 27ebec38a..c0e93e3f6 100644 --- a/Duplicati/WebserverCore/DayOfWeekStringEnumConverter.cs +++ b/Duplicati/WebserverCore/DayOfWeekStringEnumConverter.cs @@ -32,16 +32,18 @@ public class DayOfWeekStringEnumConverter : JsonConverterFactory if (reader.TokenType != JsonTokenType.String) throw new JsonException(); - return reader.GetString()?.Trim()?.ToLowerInvariant() switch + var value = reader.GetString()?.Trim()?.ToLowerInvariant(); + + return value switch { - "sun" => DayOfWeek.Sunday, - "mon" => DayOfWeek.Monday, - "tue" => DayOfWeek.Tuesday, - "wed" => DayOfWeek.Wednesday, - "thu" => DayOfWeek.Thursday, - "fri" => DayOfWeek.Friday, - "sat" => DayOfWeek.Saturday, - _ => throw new JsonException() + "sun" or "sunday" => DayOfWeek.Sunday, + "mon" or "monday" => DayOfWeek.Monday, + "tue" or "tuesday" => DayOfWeek.Tuesday, + "wed" or "wednesday" => DayOfWeek.Wednesday, + "thu" or "thursday" => DayOfWeek.Thursday, + "fri" or "friday" => DayOfWeek.Friday, + "sat" or "saturday" => DayOfWeek.Saturday, + _ => throw new JsonException($"Invalid day of week: {value}"), }; } @@ -71,7 +73,7 @@ public class DayOfWeekStringEnumConverter : JsonConverterFactory writer.WriteStringValue("sat"); break; default: - throw new JsonException(); + throw new JsonException($"Invalid day of week: {value}"); } writer.WriteStringValue(value.ToString()); } diff --git a/Duplicati/WebserverCore/Duplicati.WebserverCore.csproj b/Duplicati/WebserverCore/Duplicati.WebserverCore.csproj index be4dbc9f8..93da37923 100644 --- a/Duplicati/WebserverCore/Duplicati.WebserverCore.csproj +++ b/Duplicati/WebserverCore/Duplicati.WebserverCore.csproj @@ -17,8 +17,6 @@ - - diff --git a/Duplicati/WebserverCore/Endpoints/V1/ServerSetting.cs b/Duplicati/WebserverCore/Endpoints/V1/ServerSetting.cs index 464652b09..b43f1d46c 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/ServerSetting.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/ServerSetting.cs @@ -19,7 +19,6 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using Duplicati.Server; using Duplicati.Server.Database; using Duplicati.WebserverCore.Abstractions; using Duplicati.WebserverCore.Exceptions; diff --git a/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs b/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs index df6c64b0c..7f3e80ad3 100644 --- a/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs +++ b/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs @@ -66,6 +66,7 @@ public static class ServiceCollectionsExtensions .AddTransient() .AddTransient() .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton(); diff --git a/Duplicati/WebserverCore/Middlewares/LanguageFilter.cs b/Duplicati/WebserverCore/Middlewares/LanguageFilter.cs index 353720ebf..a376baee5 100644 --- a/Duplicati/WebserverCore/Middlewares/LanguageFilter.cs +++ b/Duplicati/WebserverCore/Middlewares/LanguageFilter.cs @@ -18,6 +18,7 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +using System.Globalization; using Duplicati.Library.Localization; namespace Duplicati.WebserverCore.Middlewares; @@ -30,7 +31,7 @@ public class LanguageFilter : IEndpointFilter ? LocalizationService.ParseCulture(locale) : null; - using (ci == null ? null : LocalizationService.TemporaryContext(ci)) + using (LocalizationService.TemporaryContext(ci ?? CultureInfo.CurrentUICulture)) return await next(context); } diff --git a/Duplicati/WebserverCore/Services/IRemoteControllerHandler.cs b/Duplicati/WebserverCore/Services/IRemoteControllerHandler.cs new file mode 100644 index 000000000..338b098cd --- /dev/null +++ b/Duplicati/WebserverCore/Services/IRemoteControllerHandler.cs @@ -0,0 +1,58 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using Duplicati.Library.RemoteControl; + +namespace Duplicati.WebserverCore.Services; + +/// +/// Interface for handling remote control messages. +/// +public interface IRemoteControllerHandler +{ + /// + /// Prepares metadata for a remote control connection. + /// + /// The initial metadata + /// The metadata to use + Task> OnConnect(Dictionary metadata); + + /// + /// Re-keys the remote control connection. + /// + /// The data to re-key with + /// An awaitable task + Task ReKey(ClaimedClientData data); + + /// + /// Handles a control message. + /// + /// The control message to handle + /// An awaitable task + Task OnControl(KeepRemoteConnection.ControlMessage message); + + /// + /// Handles a command message. + /// + /// The command message to handle + /// An awaitable task + Task OnMessage(KeepRemoteConnection.CommandMessage commandMessage); +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Services/RemoteControllerHandler.cs b/Duplicati/WebserverCore/Services/RemoteControllerHandler.cs new file mode 100644 index 000000000..6664e2918 --- /dev/null +++ b/Duplicati/WebserverCore/Services/RemoteControllerHandler.cs @@ -0,0 +1,120 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System.Net.Http.Headers; +using Duplicati.Library.Logging; +using Duplicati.Library.RemoteControl; +using Duplicati.Library.RestAPI; +using Duplicati.Server.Database; +using Duplicati.WebserverCore.Abstractions; +using Newtonsoft.Json; + +namespace Duplicati.WebserverCore.Services; + +/// +/// Controller for handling remote control messages. +/// +/// The connection to the database +/// The HTTP client factory +/// The JWT token provider +public class RemoteControllerHandler(Connection connection, IHttpClientFactory httpClientFactory, IJWTTokenProvider jwtTokenProvider) : IRemoteControllerHandler +{ + /// + /// The log tag for this class. + /// + private static readonly string LOGTAG = Log.LogTagFromType(); + + /// + public Task> OnConnect(Dictionary metadata) + { + metadata["feature:additional-report-url"] = connection.ApplicationSettings.AdditionalReportUrl; + return Task.FromResult(metadata); + } + + /// + public Task ReKey(ClaimedClientData data) + { + var oldCerts = connection.ApplicationSettings.RemoteControlConfig == null + ? null + : JsonConvert.DeserializeObject(connection.ApplicationSettings.RemoteControlConfig)?.ServerCertificates; + + connection.ApplicationSettings.RemoteControlConfig = JsonConvert.SerializeObject(new RemoteControlConfig + { + Token = data.JWT, + ServerCertificates = data.ServerCertificates ?? oldCerts ?? [], + ServerUrl = data.ServerUrl, + CertificateUrl = data.CertificateUrl + }); + + if (!FIXMEGlobal.SettingsEncryptionKeyProvidedExternally) + { + // TODO: Implement changing the encryption key + // if (!string.IsNullOrWhiteSpace(data.LocalEncryptionKey) && data.LocalEncryptionKey != connection.ApplicationSettings.SettingsEncryptionKey) + // connection.ChangeDbKey(keydata.LocalEncryptionKey); + } + + return Task.CompletedTask; + } + + /// + public async Task OnControl(KeepRemoteConnection.ControlMessage message) + { + // Make method async + await Task.CompletedTask; + + Dictionary? result = null; + + Log.WriteMessage(LogMessageType.Verbose, LOGTAG, "OnControl", "Received control message: {0}", message); + try + { + if (string.Equals(message.ControlRequestMessage.Command, ControlRequestMessage.ConfigureReportUrlSet, StringComparison.OrdinalIgnoreCase)) + FIXMEGlobal.DataConnection.ApplicationSettings.AdditionalReportUrl = message.ControlRequestMessage.Parameters.GetValueOrDefault(ControlRequestMessage.ConfigureReportUrlParameter); + + if (string.Equals(message.ControlRequestMessage.Command, ControlRequestMessage.ConfigureReportUrlGet, StringComparison.OrdinalIgnoreCase)) + result = new Dictionary { { ControlRequestMessage.ConfigureReportUrlParameter, FIXMEGlobal.DataConnection.ApplicationSettings.AdditionalReportUrl } }; + } + catch (Exception ex) + { + Log.WriteMessage(LogMessageType.Error, LOGTAG, "OnControl", ex, "Failed to handle control message: {0}", message); + message.Respond(new ControlResponseMessage(false, null, ex.Message)); + return; + } + + message.Respond(new ControlResponseMessage(true, result, null)); + } + + + /// + public async Task OnMessage(KeepRemoteConnection.CommandMessage commandMessage) + { + using var httpClient = httpClientFactory.CreateClient(); + var token = jwtTokenProvider.CreateAccessToken("remote-control", jwtTokenProvider.TemporaryFamilyId, TimeSpan.FromMinutes(2)); + + httpClient.BaseAddress = new Uri($"{(connection.ApplicationSettings.UseHTTPS ? "https" : "http")}://127.0.0.1:{connection.ApplicationSettings.LastWebserverPort}"); + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token); + + var psk = Middlewares.PreSharedKeyFilter.PreSharedKey; + if (!string.IsNullOrWhiteSpace(psk)) + httpClient.DefaultRequestHeaders.Add(Middlewares.PreSharedKeyFilter.HeaderName, psk); + + await commandMessage.Handle(httpClient); + } +} diff --git a/Duplicati/WebserverCore/Services/RemoteControllerService.cs b/Duplicati/WebserverCore/Services/RemoteControllerService.cs index decefb361..9286f87ff 100644 --- a/Duplicati/WebserverCore/Services/RemoteControllerService.cs +++ b/Duplicati/WebserverCore/Services/RemoteControllerService.cs @@ -19,9 +19,7 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System.Net.Http.Headers; using Duplicati.Library.RemoteControl; -using Duplicati.Library.RestAPI; using Duplicati.Server.Database; using Duplicati.WebserverCore.Abstractions; using Newtonsoft.Json; @@ -32,9 +30,8 @@ namespace Duplicati.WebserverCore.Services; /// Controller for toggling remote control. /// /// The connection to the database -/// The HTTP client factory -/// The JWT token provider -public class RemoteControllerService(Connection connection, IHttpClientFactory httpClientFactory, IJWTTokenProvider jwtTokenProvider) : IRemoteController +/// The remote controller handler +public class RemoteControllerService(Connection connection, IRemoteControllerHandler controllerHandler) : IRemoteController { /// /// Gets a value indicating whether remote control is enabled. @@ -83,58 +80,15 @@ public class RemoteControllerService(Connection connection, IHttpClientFactory h config.CertificateUrl, config.ServerCertificates, CancellationToken.None, - ReKey, - OnMessage + controllerHandler.OnConnect, + controllerHandler.ReKey, + controllerHandler.OnControl, + controllerHandler.OnMessage ); connection.ApplicationSettings.RemoteControlEnabled = true; } - /// - /// Re-keys the remote control connection. - /// - /// The data to re-key with - /// An awaitable task - private Task ReKey(ClaimedClientData data) - { - var oldCerts = connection.ApplicationSettings.RemoteControlConfig == null - ? null - : JsonConvert.DeserializeObject(connection.ApplicationSettings.RemoteControlConfig)?.ServerCertificates; - - connection.ApplicationSettings.RemoteControlConfig = JsonConvert.SerializeObject(new RemoteControlConfig - { - Token = data.JWT, - ServerCertificates = data.ServerCertificates ?? oldCerts ?? [], - ServerUrl = data.ServerUrl, - CertificateUrl = data.CertificateUrl - }); - - if (!FIXMEGlobal.SettingsEncryptionKeyProvidedExternally) - { - // TODO: Implement changing the encryption key - // if (!string.IsNullOrWhiteSpace(data.LocalEncryptionKey) && data.LocalEncryptionKey != connection.ApplicationSettings.SettingsEncryptionKey) - // connection.ChangeDbKey(keydata.LocalEncryptionKey); - } - - return Task.CompletedTask; - } - - /// - /// Handles a command message. - /// - /// The command message to handle - /// An awaitable task - private async Task OnMessage(KeepRemoteConnection.CommandMessage commandMessage) - { - using var httpClient = httpClientFactory.CreateClient(); - var token = jwtTokenProvider.CreateAccessToken("remote-control", jwtTokenProvider.TemporaryFamilyId, TimeSpan.FromMinutes(2)); - - httpClient.BaseAddress = new Uri($"{(connection.ApplicationSettings.UseHTTPS ? "https" : "http")}://127.0.0.1:{connection.ApplicationSettings.LastWebserverPort}"); - httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token); - - await commandMessage.Handle(httpClient); - } - /// public void Disable() { diff --git a/Duplicati/WebserverCore/Services/SystemInfoProvider.cs b/Duplicati/WebserverCore/Services/SystemInfoProvider.cs index 1662b3352..724732457 100644 --- a/Duplicati/WebserverCore/Services/SystemInfoProvider.cs +++ b/Duplicati/WebserverCore/Services/SystemInfoProvider.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using System.Globalization; using Duplicati.Library.AutoUpdater; +using Duplicati.Library.Localization; using Duplicati.Library.RestAPI; using Duplicati.Server; using Duplicati.Server.Serialization.Interface; @@ -205,62 +206,87 @@ public class SystemInfoProvider : ISystemInfoProvider } /// - /// Cached static system information. + /// Creates a new instance of the class. /// - private Lazy _systemInfoBase = new(() => new StaticSystemInformation - { - APIVersion = 1, - PasswordPlaceholder = FIXMEGlobal.PASSWORD_PLACEHOLDER, - ServerVersion = UpdaterManager.SelfVersion.Version, - ServerVersionName = License.VersionNumbers.VERSION_NAME, - ServerVersionType = UpdaterManager.SelfVersion.ReleaseType, - RemoteControlRegistrationUrl = Library.RemoteControl.RegisterForRemote.DefaultRegisterationUrl, - StartedBy = FIXMEGlobal.Origin, - DefaultUpdateChannel = AutoUpdateSettings.DefaultUpdateChannel.ToString(), - DefaultUsageReportLevel = Library.UsageReporter.Reporter.DefaultReportLevel, - OSType = UpdaterManager.OperatingSystemName, - OSVersion = Library.UsageReporter.OSInfoHelper.PlatformString, - DirectorySeparator = Path.DirectorySeparatorChar, - PathSeparator = Path.PathSeparator, - CaseSensitiveFilesystem = Library.Utility.Utility.IsFSCaseSensitive, - MachineName = Environment.MachineName, - PackageTypeId = UpdaterManager.PackageTypeId, - UserName = OperatingSystem.IsWindows() ? System.Security.Principal.WindowsIdentity.GetCurrent().Name : Environment.UserName, - NewLine = Environment.NewLine, - CLRVersion = Environment.Version.ToString(), - Options = Server.Serializable.ServerSettings.Options, - CompressionModules = Server.Serializable.ServerSettings.CompressionModules, - EncryptionModules = Server.Serializable.ServerSettings.EncryptionModules, - BackendModules = Server.Serializable.ServerSettings.BackendModules, - GenericModules = Server.Serializable.ServerSettings.GenericModules, - WebModules = Server.Serializable.ServerSettings.WebModules, - ConnectionModules = Server.Serializable.ServerSettings.ConnectionModules, - ServerModules = Server.Serializable.ServerSettings.ServerModules, - SecretProviderModules = Server.Serializable.ServerSettings.SecretProviderModules, - UsingAlternateUpdateURLs = AutoUpdateSettings.UsesAlternateURLs, - LogLevels = Enum.GetNames(typeof(Library.Logging.LogMessageType)), - SpecialFolders = SpecialFolders.Nodes.Select(n => new Dto.SystemInfoDto.SpecialFolderDto { ID = n.id, Path = n.resolvedpath }).ToArray(), - SupportedLocales = Library.Localization.LocalizationService.SupportedCultures + /// The new instance. + private static StaticSystemInformation CreateStaticSystemInformation() + => new StaticSystemInformation + { + APIVersion = 1, + PasswordPlaceholder = FIXMEGlobal.PASSWORD_PLACEHOLDER, + ServerVersion = UpdaterManager.SelfVersion.Version, + ServerVersionName = License.VersionNumbers.VERSION_NAME, + ServerVersionType = UpdaterManager.SelfVersion.ReleaseType, + RemoteControlRegistrationUrl = Library.RemoteControl.RegisterForRemote.DefaultRegisterationUrl, + StartedBy = FIXMEGlobal.Origin, + DefaultUpdateChannel = AutoUpdateSettings.DefaultUpdateChannel.ToString(), + DefaultUsageReportLevel = Library.UsageReporter.Reporter.DefaultReportLevel, + OSType = UpdaterManager.OperatingSystemName, + OSVersion = Library.UsageReporter.OSInfoHelper.PlatformString, + DirectorySeparator = Path.DirectorySeparatorChar, + PathSeparator = Path.PathSeparator, + CaseSensitiveFilesystem = Library.Utility.Utility.IsFSCaseSensitive, + MachineName = Environment.MachineName, + PackageTypeId = UpdaterManager.PackageTypeId, + UserName = OperatingSystem.IsWindows() ? System.Security.Principal.WindowsIdentity.GetCurrent().Name : Environment.UserName, + NewLine = Environment.NewLine, + CLRVersion = Environment.Version.ToString(), + Options = Server.Serializable.ServerSettings.Options, + CompressionModules = Server.Serializable.ServerSettings.CompressionModules, + EncryptionModules = Server.Serializable.ServerSettings.EncryptionModules, + BackendModules = Server.Serializable.ServerSettings.BackendModules, + GenericModules = Server.Serializable.ServerSettings.GenericModules, + WebModules = Server.Serializable.ServerSettings.WebModules, + ConnectionModules = Server.Serializable.ServerSettings.ConnectionModules, + ServerModules = Server.Serializable.ServerSettings.ServerModules, + SecretProviderModules = Server.Serializable.ServerSettings.SecretProviderModules, + UsingAlternateUpdateURLs = AutoUpdateSettings.UsesAlternateURLs, + LogLevels = Enum.GetNames(typeof(Library.Logging.LogMessageType)), + SpecialFolders = SpecialFolders.Nodes.Select(n => new Dto.SystemInfoDto.SpecialFolderDto { ID = n.id, Path = n.resolvedpath }).ToArray(), + SupportedLocales = Library.Localization.LocalizationService.SupportedCultures .Select(x => new Dto.SystemInfoDto.LocaleDto { Code = x, EnglishName = new CultureInfo(x).EnglishName, DisplayName = new CultureInfo(x).NativeName }).ToArray(), - TimeZones = Library.Utility.TimeZoneHelper.GetTimeZones() + TimeZones = Library.Utility.TimeZoneHelper.GetTimeZones() .Select(x => new Dto.SystemInfoDto.TimeZoneDto { ID = x.Id, DisplayName = x.DisplayName, CurrentUTCOffset = x.CurrentUtcOffset.ToString() }), - }); + }; + + /// + /// The lock guarding the _systemInfoBases cache. + /// + private readonly object _lock = new(); + + /// + /// Cached static system information, the key is the language code. + /// + private Dictionary _systemInfoBases = new(StringComparer.OrdinalIgnoreCase); /// public SystemInfoDto GetSystemInfo(CultureInfo? browserlanguage) { browserlanguage ??= CultureInfo.InvariantCulture; - var systeminfo = _systemInfoBase.Value; + var currentLanguage = LocalizationService.CurrentUI?.Culture.Name; + if (string.IsNullOrWhiteSpace(currentLanguage)) + currentLanguage = CultureInfo.CurrentUICulture.Name; + + StaticSystemInformation? systeminfo; + + lock (_lock) + { + if (!_systemInfoBases.TryGetValue(currentLanguage, out systeminfo)) + { + systeminfo = CreateStaticSystemInformation(); + _systemInfoBases[currentLanguage] = systeminfo; + } + } // Return the system information, patch in dynamic values return new SystemInfoDto() @@ -305,7 +331,7 @@ public class SystemInfoProvider : ISystemInfoProvider DisplayName = browserlanguage.NativeName }, SupportedLocales = systeminfo.SupportedLocales, - BrowserLocaleSupported = Library.Localization.LocalizationService.isCultureSupported(browserlanguage), + BrowserLocaleSupported = Library.Localization.LocalizationService.IsCultureSupported(browserlanguage), TimeZones = systeminfo.TimeZones, }; } diff --git a/LiveTests/Duplicati.Backend.Tests/Box/BoxTests.cs b/LiveTests/Duplicati.Backend.Tests/Box/BoxTests.cs new file mode 100644 index 000000000..d393a2118 --- /dev/null +++ b/LiveTests/Duplicati.Backend.Tests/Box/BoxTests.cs @@ -0,0 +1,46 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +namespace Duplicati.Backend.Tests.Box; + +/// +/// Box.com Tests +/// +[TestClass] +public sealed class BoxTests : BaseTest +{ + [TestMethod] + public Task TestBox() + { + CheckRequiredEnvironment(["TESTCREDENTIAL_BOX_FOLDER", "TESTCREDENTIAL_BOX_AUTHID"]); + + var exitCode = CommandLine.BackendTester.Program.Main( + new[] + { + $"box://{Environment.GetEnvironmentVariable("TESTCREDENTIAL_BOX_FOLDER")}/?authid={Uri.EscapeDataString(Environment.GetEnvironmentVariable("TESTCREDENTIAL_BOX_AUTHID")!)}", + + }.Concat(Parameters.GlobalTestParameters).ToArray()); + + if (exitCode != 0) Assert.Fail("BackendTester is returning non-zero exit code, check logs for details"); + + return Task.CompletedTask; + } +} \ No newline at end of file diff --git a/LiveTests/Duplicati.Backend.Tests/README.md b/LiveTests/Duplicati.Backend.Tests/README.md index 8427d1ffd..c15ba6fcb 100644 --- a/LiveTests/Duplicati.Backend.Tests/README.md +++ b/LiveTests/Duplicati.Backend.Tests/README.md @@ -27,6 +27,7 @@ NUMBER_OF_FILES default is 20 | AzureBlob | Planned | Live | | CIFS | Planned | Live | | Google Drive | Planned | Live | +| Box | Planned | Live | | TahoeLAFS | Pending Decision | | | TescentCOS | Pending Decision | | | Storj | Pending Decision | | @@ -39,7 +40,6 @@ NUMBER_OF_FILES default is 20 | Jottacloud | Pending Decision | | | IDrive2 | Planned | | | File | Planned | | -| Box | Pending Decision | | | CloudFiles | Pending Decision | | | backBlaze | Planned | Live | | AliyunOSS | Pending Decision | | @@ -56,6 +56,13 @@ Please note that TestContainers token has to be configured in secrets/Github act ## Backends that require Environment variables +## Box.com + +``` +TESTCREDENTIAL_BOX_FOLDER +TESTCREDENTIAL_BOX_AUTHID +``` + ## Backblaze B2 Backblaze B2 credentials are mapped to the following environment variables: