haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into bugfix/restore-deadlock

Browse Source
This commit is contained in:
Carl Johnsen
2025-03-10 05:50:26 +01:00
parent 83dc334069 9a027e9264
commit f77fcc186c
38 changed files with 1249 additions and 597 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/backendtests.yml
+89 -37
View File
@@ -14,6 +14,7 @@ jobs:
google_secrets_available: "${{ steps.check_google.outputs.available }}"
pcloud_secrets_available: "${{ steps.check_pcloud.outputs.available }}"
b2_secrets_available: "${{ steps.check_b2.outputs.available }}"
box_secrets_available: "${{ steps.check_box.outputs.available }}"
steps:
- id: check_pcloud
name: Check pCloud secrets
@@ -120,6 +121,20 @@ jobs:
echo "available=false" >> $GITHUB_OUTPUT
fi
echo "B2 check completed"
- id: check_box
name: Check Box.com secrets
shell: bash
run: |
echo "Starting Box.com secrets check..."
if [[ -n "${{ secrets.TESTCREDENTIAL_BOX_FOLDER }}" ]] && \
[[ -n "${{ secrets.TESTCREDENTIAL_BOX_AUTHID }}" ]]; then
echo "All Box.com secrets found"
echo "available=true" >> $GITHUB_OUTPUT
else
echo "Missing some Box.com secrets"
echo "available=false" >> $GITHUB_OUTPUT
fi
echo "Box.com check completed"
- name: Debug Output
shell: bash
run: |
@@ -130,6 +145,7 @@ jobs:
echo "pCloud Available: ${{ steps.check_pcloud.outputs.available }}"
echo "Azure Available: ${{ steps.check_azure.outputs.available }}"
echo "B2 Available: ${{ steps.check_b2.outputs.available }}"
echo "Box.com Available: ${{ steps.check_box.outputs.available }}"
test_ftp:
needs: check_secrets
if: needs.check_secrets.outputs.testcontainers_secrets_available == 'true'
@@ -468,40 +484,76 @@ jobs:
--logger:"console;verbosity=detailed"
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
test_b2:
needs: check_secrets
if: needs.check_secrets.outputs.b2_secrets_available == 'true'
name: Backblaze B2 Tests
runs-on: "${{ matrix.os }}"
strategy:
max-parallel: 1
fail-fast: false
matrix:
os:
- ubuntu-latest
- windows-latest
- macos-latest
steps:
- name: Set up .NET
uses: actions/setup-dotnet@v4
with:
dotnet-version: 8.x
- name: Checkout source
uses: actions/checkout@v4
- name: Restore NuGet dependencies
run: >-
dotnet restore
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
- name: Build project
run: >-
dotnet build --no-restore
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
- name: Run B2 tests
env:
TESTCREDENTIAL_B2_BUCKET: "${{ secrets.TESTCREDENTIAL_B2_BUCKET }}"
TESTCREDENTIAL_B2_FOLDER: "${{ secrets.TESTCREDENTIAL_B2_FOLDER }}"
TESTCREDENTIAL_B2_USERNAME: "${{ secrets.TESTCREDENTIAL_B2_USERNAME }}"
TESTCREDENTIAL_B2_PASSWORD: "${{ secrets.TESTCREDENTIAL_B2_PASSWORD }}"
run: >-
dotnet test --no-build --filter="ClassName~B2Tests"
--logger:"console;verbosity=detailed"
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
needs: check_secrets
if: needs.check_secrets.outputs.b2_secrets_available == 'true'
name: Backblaze B2 Tests
runs-on: "${{ matrix.os }}"
strategy:
max-parallel: 1
fail-fast: false
matrix:
os:
- ubuntu-latest
- windows-latest
- macos-latest
steps:
- name: Set up .NET
uses: actions/setup-dotnet@v4
with:
dotnet-version: 8.x
- name: Checkout source
uses: actions/checkout@v4
- name: Restore NuGet dependencies
run: >-
dotnet restore
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
- name: Build project
run: >-
dotnet build --no-restore
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
- name: Run B2 tests
env:
TESTCREDENTIAL_B2_BUCKET: "${{ secrets.TESTCREDENTIAL_B2_BUCKET }}"
TESTCREDENTIAL_B2_FOLDER: "${{ secrets.TESTCREDENTIAL_B2_FOLDER }}"
TESTCREDENTIAL_B2_USERNAME: "${{ secrets.TESTCREDENTIAL_B2_USERNAME }}"
TESTCREDENTIAL_B2_PASSWORD: "${{ secrets.TESTCREDENTIAL_B2_PASSWORD }}"
run: >-
dotnet test --no-build --filter="ClassName~B2Tests"
--logger:"console;verbosity=detailed"
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
test_box:
needs: check_secrets
if: needs.check_secrets.outputs.box_secrets_available == 'true'
name: Box.com Tests
runs-on: "${{ matrix.os }}"
strategy:
max-parallel: 1
fail-fast: false
matrix:
os:
- ubuntu-latest
- windows-latest
- macos-latest
steps:
- name: Set up .NET
uses: actions/setup-dotnet@v4
with:
dotnet-version: 8.x
- name: Checkout source
uses: actions/checkout@v4
- name: Restore NuGet dependencies
run: >-
dotnet restore
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
- name: Build project
run: >-
dotnet build --no-restore
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
- name: Run Box.com tests
env:
TESTCREDENTIAL_BOX_FOLDER: "${{ secrets.TESTCREDENTIAL_BOX_FOLDER }}"
TESTCREDENTIAL_BOX_AUTHID: "${{ secrets.TESTCREDENTIAL_BOX_AUTHID }}"
run: >-
dotnet test --no-build --filter="ClassName~Box.BoxTests"
--logger:"console;verbosity=detailed"
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
Duplicati/Agent/Program.cs
+18 -27
View File
@@ -34,7 +34,7 @@ using Duplicati.Library.RemoteControl;
using Duplicati.Library.RestAPI;
using Duplicati.Library.Utility;
using Duplicati.Server;
using Duplicati.WebserverCore.Abstractions;
using Duplicati.WebserverCore.Services;
using Microsoft.Extensions.DependencyInjection;
namespace Duplicati.Agent;
@@ -46,12 +46,6 @@ public static class Program
/// </summary>
private static readonly string LogTag = Log.LogTagFromType(typeof(Program));
/// <summary>
/// Create a random pre-shared key for the agent
/// </summary>
private static string? PreSharedKey = null;
/// <summary>
/// Commandline arguments for the agent
/// </summary>
@@ -275,7 +269,7 @@ public static class Program
// Set the pre-shared key for the agent
if (!agentConfig.DisablePreSharedKey)
WebserverCore.Middlewares.PreSharedKeyFilter.PreSharedKey = PreSharedKey = RandomNumberGenerator.GetHexString(128);
WebserverCore.Middlewares.PreSharedKeyFilter.PreSharedKey = RandomNumberGenerator.GetHexString(128);
using var cts = new CancellationTokenSource();
@@ -313,8 +307,10 @@ public static class Program
settings.CertificateUrl,
settings.ServerCertificates,
cts.Token,
c => ReKey(agentConfig, c),
(m) => OnMessage(m, agentConfig)
OnConnect,
m => ReKey(m, agentConfig),
OnControl,
OnMessage
)
);
@@ -324,24 +320,19 @@ public static class Program
}
}
private static async Task OnMessage(KeepRemoteConnection.CommandMessage message, CommandLineArguments agentConfig)
{
Log.WriteMessage(LogMessageType.Verbose, LogTag, "OnMessage", "Received message: {0}", message);
var provider = FIXMEGlobal.Provider.GetRequiredService<IJWTTokenProvider>();
var token = provider.CreateAccessToken("agent", provider.TemporaryFamilyId, TimeSpan.FromMinutes(2));
using var httpClient = FIXMEGlobal.Provider.GetRequiredService<IHttpClientFactory>().CreateClient();
httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token);
httpClient.BaseAddress = new System.Uri($"http://127.0.0.1:{agentConfig.WebservicePort}");
if (!string.IsNullOrWhiteSpace(WebserverCore.Middlewares.PreSharedKeyFilter.PreSharedKey))
httpClient.DefaultRequestHeaders.Add(WebserverCore.Middlewares.PreSharedKeyFilter.HeaderName, PreSharedKey);
await message.Handle(httpClient);
}
private static Task ReKey(CommandLineArguments agentConfig, ClaimedClientData keydata)
private static Task<Dictionary<string, string?>> OnConnect(Dictionary<string, string?> metadata)
=> FIXMEGlobal.Provider.GetRequiredService<IRemoteControllerHandler>().OnConnect(metadata);
private static Task OnControl(KeepRemoteConnection.ControlMessage message)
=> FIXMEGlobal.Provider.GetRequiredService<IRemoteControllerHandler>().OnControl(message);
private static Task OnMessage(KeepRemoteConnection.CommandMessage message)
=> FIXMEGlobal.Provider.GetRequiredService<IRemoteControllerHandler>().OnMessage(message);
private static Task ReKey(ClaimedClientData keydata, CommandLineArguments agentConfig)
{
// ReKey is handled here because we store the config outside of the database
Log.WriteMessage(LogMessageType.Verbose, LogTag, "ReKey", "Rekeying the settings");
var settings = Settings.Load(agentConfig.AgentSettingsFile.FullName, agentConfig.AgentSettingsFilePassphrase);
if (!string.IsNullOrWhiteSpace(keydata.JWT) && settings.JWT != keydata.JWT)
Duplicati/CommandLine/CLI/Commands.cs
+1
View File
@@ -668,6 +668,7 @@ namespace Duplicati.CommandLine
if (parsedStats.KnownFileCount > 0)
{
output.MessageEvent(string.Format(" Remote files: {0}", parsedStats.KnownFileCount));
output.MessageEvent(string.Format(" Remote filesets: {0}", parsedStats.KnownFilesets));
output.MessageEvent(string.Format(" Remote size: {0}", Library.Utility.Utility.FormatSizeString(parsedStats.KnownFileSize)));
}
Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs
+1 -1
View File
@@ -224,7 +224,7 @@ No password provided, unable to connect to server, exiting");
if (hosted != null && Server.Program.ApplicationInstance != null)
Server.Program.ApplicationInstance.SecondInstanceDetected +=
new Server.SingleInstance.SecondInstanceDelegate(
x => { tk.ShowUrlInWindow(serverURL.ToString()); });
x => tk.ShowStatusWindow());
// TODO: If we change to hosted browser this should be a callback
if (openui)
Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs
+1 -1
View File
@@ -180,7 +180,7 @@ namespace Duplicati.GUI.TrayIcon
});
}
protected void ShowStatusWindow()
public void ShowStatusWindow()
{
Program.Connection.GetStatusWindowURLAsync().ContinueWith(t =>
{
Duplicati/Library/Backend/Backblaze/B2.cs
+2 -2
View File
@@ -272,7 +272,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
_uploadUrl = await _b2AuthHelper.PostAndGetJsonDataAsync<UploadUrlResponse>(
$"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_get_upload_url",
new UploadUrlRequest { BucketID = GetBucketAsync(cancellationToken).Await().BucketID },
new UploadUrlRequest { BucketID = (await GetBucketAsync(cancellationToken)).BucketID },
combinedCancellationToken.Token
).ConfigureAwait(false);
@@ -487,7 +487,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
$"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_file_versions",
new ListFilesRequest
{
BucketID = GetBucketAsync(cancellationToken).Await().BucketID,
BucketID = (await GetBucketAsync(cancellationToken)).BucketID,
MaxFileCount = _pageSize,
Prefix = _prefix,
StartFileID = nextFileId,
Duplicati/Library/Interface/ResultInterfaces.cs
+42 -41
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
@@ -38,7 +38,7 @@ namespace Duplicati.Library.Interface
DateTime BeginTime { get; }
DateTime EndTime { get; }
TimeSpan Duration { get; }
IEnumerable<string> Errors { get; }
IEnumerable<string> Warnings { get; }
IEnumerable<string> Messages { get; }
@@ -64,6 +64,7 @@ namespace Duplicati.Library.Interface
long UnknownFileCount { get; set; }
long KnownFileCount { get; set; }
long KnownFileSize { get; set; }
long KnownFilesets { get; set; }
DateTime LastBackupDate { get; set; }
long BackupListCount { get; set; }
long TotalQuotaSpace { get; set; }
@@ -116,14 +117,14 @@ namespace Duplicati.Library.Interface
IEnumerable<IListResultRemoteLog> LogMessages { get; }
IEnumerable<IListResultRemoteVolume> RemoteVolumes { get; }
}
public interface IDeleteResults : IBasicResults
{
IEnumerable<Tuple<long, DateTime>> DeletedSets { get; }
ICompactResults CompactResults { get; }
bool Dryrun { get; }
}
public interface IBackupResults : IBasicResults, IBackendStatsticsReporter
{
long DeletedFiles { get; }
@@ -143,16 +144,16 @@ namespace Duplicati.Library.Interface
long ModifiedFolders { get; }
long ModifiedSymlinks { get; }
long AddedSymlinks { get; }
long DeletedSymlinks { get; }
long DeletedSymlinks { get; }
bool PartialBackup { get; }
bool Dryrun { get; }
ICompactResults CompactResults { get; }
IVacuumResults VacuumResults { get; }
IDeleteResults DeleteResults { get; }
IRepairResults RepairResults { get; }
}
public interface IRestoreResults : IBasicResults
{
long RestoredFiles { get; }
@@ -163,10 +164,10 @@ namespace Duplicati.Library.Interface
long DeletedFiles { get; }
long DeletedFolders { get; }
long DeletedSymlinks { get; }
IRecreateDatabaseResults RecreateDatabaseResults { get; }
}
public interface IRecreateDatabaseResults : IBasicResults
{
}
@@ -188,23 +189,23 @@ namespace Duplicati.Library.Interface
IVacuumResults VacuumResults { get; }
}
public interface ICreateLogDatabaseResults : IBasicResults
{
string TargetPath { get; }
}
public interface IRestoreControlFilesResults : IBasicResults
{
IEnumerable<string> Files { get; }
}
public interface IRepairResults : IBasicResults
{
IRecreateDatabaseResults RecreateDatabaseResults { get; }
}
/// <summary>
/// The possible change types for an entry
/// </summary>
@@ -222,8 +223,8 @@ namespace Duplicati.Library.Interface
/// The element was modified
/// </summary>
Modified
}
}
/// <summary>
/// The possible entry types
/// </summary>
@@ -242,16 +243,16 @@ namespace Duplicati.Library.Interface
/// </summary>
File
}
public interface IListChangesResults : IBasicResults
{
DateTime BaseVersionTimestamp { get; }
DateTime CompareVersionTimestamp { get; }
long BaseVersionIndex { get; }
long CompareVersionIndex { get; }
IEnumerable<Tuple<ListChangesChangeType, ListChangesElementType, string>> ChangeDetails { get; }
IEnumerable<Tuple<ListChangesChangeType, ListChangesElementType, string>> ChangeDetails { get; }
long AddedFolders { get; }
long AddedSymlinks { get; }
long AddedFiles { get; }
@@ -263,14 +264,14 @@ namespace Duplicati.Library.Interface
long ModifiedFolders { get; }
long ModifiedSymlinks { get; }
long ModifiedFiles { get; }
long PreviousSize { get; }
long CurrentSize { get; }
long AddedSize { get; }
long DeletedSize { get; }
}
/// <summary>
/// The status of a change in a test entry
/// </summary>
@@ -298,7 +299,7 @@ namespace Duplicati.Library.Interface
{
IEnumerable<KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>> Verifications { get; }
}
public interface ITestFilterResults : IBasicResults
{
long FileSize { get; set; }
Duplicati/Library/Localization/ILocalizationService.cs
+25 -21
View File
@@ -1,31 +1,35 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.Globalization;
namespace Duplicati.Library.Localization
{
public interface ILocalizationService
{
/// <summary>
/// The culture of the localization service
/// </summary>
public CultureInfo Culture { get; }
/// <summary>
/// Localizes the string similar to how string.Format works
/// </summary>
Duplicati/Library/Localization/LocalizationContext.cs
+33 -24
View File
@@ -1,26 +1,26 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Logging;
using System;
using System.Threading;
namespace Duplicati.Library.Localization
{
@@ -29,10 +29,14 @@ namespace Duplicati.Library.Localization
/// </summary>
internal class LocalizationContext : IDisposable
{
/// <summary>
/// The current culture, using an async local to keep the call context
/// </summary>
private static readonly AsyncLocal<string> _currentCulture = new();
/// <summary>
/// The previous context
/// </summary>
private readonly object m_prev;
private readonly string m_prev;
/// <summary>
/// Flag to prevent double dispose
@@ -45,12 +49,17 @@ namespace Duplicati.Library.Localization
/// <param name="ci">The localization to use.</param>
public LocalizationContext(System.Globalization.CultureInfo ci)
{
m_prev = CallContext.GetData(LocalizationService.LOGICAL_CONTEXT_KEY) as string;
CallContext.SetData(LocalizationService.LOGICAL_CONTEXT_KEY, ci.Name);
m_prev = _currentCulture.Value;
_currentCulture.Value = ci.Name;
m_isDisposed = false;
}
/// <summary>
/// Returns the current culture value for the call context
/// </summary>
public static string Current => _currentCulture.Value;
/// <summary>
/// Releases all resource used by the <see cref="T:Duplicati.Library.Localization.LocalizationContext"/> object.
/// </summary>
@@ -64,7 +73,7 @@ namespace Duplicati.Library.Localization
{
if (!m_isDisposed)
{
CallContext.SetData(LocalizationService.LOGICAL_CONTEXT_KEY, m_prev);
_currentCulture.Value = m_prev;
m_isDisposed = true;
}
}
Duplicati/Library/Localization/LocalizationService.cs
+35 -27
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
@@ -24,7 +24,6 @@ using System.Linq;
using System.Globalization;
using System.Collections.Generic;
using System.Text.RegularExpressions;
using Duplicati.Library.Localization.Short;
using Duplicati.Library.Logging;
namespace Duplicati.Library.Localization
@@ -39,11 +38,6 @@ namespace Duplicati.Library.Localization
/// </summary>
private static readonly Dictionary<CultureInfo, ILocalizationService> Services = new Dictionary<CultureInfo, ILocalizationService>();
/// <summary>
/// The key for accessing logical context
/// </summary>
internal const string LOGICAL_CONTEXT_KEY = "DUPLICATI_LOCALIZATION_CULTURE_CONTEXT";
/// <summary>
/// Regular expression to match a locale
/// </summary>
@@ -97,13 +91,27 @@ namespace Duplicati.Library.Localization
{
get
{
var lc = CallContext.GetData(LOGICAL_CONTEXT_KEY) as string;
var lc = LocalizationContext.Current;
if (!string.IsNullOrWhiteSpace(lc))
return Get(new CultureInfo(lc));
return Get(CultureInfo.CurrentCulture);
}
}
/// <summary>
/// Gets a localization provider with the current UI language
/// </summary>
public static ILocalizationService CurrentUI
{
get
{
var lc = LocalizationContext.Current;
if (!string.IsNullOrWhiteSpace(lc))
return Get(new CultureInfo(lc));
return Get(CultureInfo.CurrentUICulture);
}
}
/// <summary>
/// Gets a localization provider with the OS install language
/// </summary>
@@ -147,7 +155,7 @@ namespace Duplicati.Library.Localization
/// <summary>
/// Returns true if the culture has localization support
/// </summary>
public static Boolean isCultureSupported(CultureInfo culture)
public static bool IsCultureSupported(CultureInfo culture)
{
return MoLocalizationService.isCultureSupported(culture);
}
Duplicati/Library/Localization/MoLocalizationService.cs
+6
View File
@@ -75,6 +75,7 @@ namespace Duplicati.Library.Localization
/// <param name="ci">The culture to find.</param>
public MoLocalizationService(CultureInfo ci)
{
Culture = ci;
var filenames = new string[] {
// Load the specialized version first
string.Format("localization-{0}.mo", ci.Name.Replace('-', '_')),
@@ -117,6 +118,11 @@ namespace Duplicati.Library.Localization
}
}
/// <summary>
/// Gets the culture of the localization
/// </summary>
public CultureInfo Culture { get; }
/// <summary>
/// Pre-processes the message to use Linux line endings
/// </summary>
Duplicati/Library/Localization/MockLocalizationService.cs
+25 -21
View File
@@ -1,31 +1,35 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.Globalization;
namespace Duplicati.Library.Localization
{
public class MockLocalizationService : ILocalizationService
{
/// <summary>
/// The culture of the localization service
/// </summary>
public CultureInfo Culture => CultureInfo.InvariantCulture;
/// <summary>
/// Localizes the string similar to how string.Format works
/// </summary>
Duplicati/Library/Logging/CallContext.cs
-50
View File
@@ -1,50 +0,0 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.Collections.Concurrent;
using System.Threading;
namespace Duplicati.Library.Logging
{
/// <summary>
/// Provides a way to set contextual data that flows with the call and
/// async context of a test or invocation.
/// </summary>
public static class CallContext
{
static ConcurrentDictionary<string, AsyncLocal<object>> state = new ConcurrentDictionary<string, AsyncLocal<object>>();
/// <summary>
/// Stores a given object and associates it with the specified name.
/// </summary>
/// <param name="name">The name with which to associate the new item in the call context.</param>
/// <param name="data">The object to store in the call context.</param>
public static void SetData(string name, object data) =>
state.GetOrAdd(name, _ => new AsyncLocal<object>()).Value = data;
/// <summary>
/// Retrieves an object with the specified name from the <see cref="CallContext"/>.
/// </summary>
/// <param name="name">The name of the item in the call context.</param>
/// <returns>The object in the call context associated with the specified name, or <see langword="null"/> if not found.</returns>
public static object GetData(string name) =>
state.TryGetValue(name, out AsyncLocal<object> data) ? data.Value : null;
}
}
Duplicati/Library/Logging/Log.cs
+27 -53
View File
@@ -1,27 +1,27 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading;
namespace Duplicati.Library.Logging
{
@@ -72,18 +72,13 @@ namespace Duplicati.Library.Logging
/// <summary>
/// The key used to assign the current scope into the current call-context
/// </summary>
private const string LOGICAL_CONTEXT_KEY = "Duplicati:LoggingEntry";
private static readonly AsyncLocal<LogScope> _currentScope = new AsyncLocal<LogScope>();
/// <summary>
/// The root scope
/// </summary>
private static readonly LogScope m_root = new LogScope(null, new LogTagFilter(LogMessageType.Error, null, null), null, true);
/// <summary>
/// The stored log instances
/// </summary>
private static readonly Dictionary<string, LogScope> m_log_instances = new Dictionary<string, LogScope>();
/// <summary>
/// Static lock object to provide thread safe logging
/// </summary>
@@ -216,15 +211,15 @@ namespace Duplicati.Library.Logging
WriteMessage(LogMessageType.Verbose, tag, id, null, message, arguments);
}
/// <summary>
/// <summary>
/// Writes a verbose message to the current log destination
/// </summary>
/// <param name="message">The message to write</param>
/// <param name="tag">The tag-type for this message</param>
/// <param name="id">The message id</param>
/// <param name="ex">The exception to log</param>
/// <param name="ex">The exception to log</param>
/// <param name="arguments">The message format arguments</param>
public static void WriteVerboseMessage(string tag, string id, Exception ex, string message, params object[] arguments)
public static void WriteVerboseMessage(string tag, string id, Exception ex, string message, params object[] arguments)
{
WriteMessage(LogMessageType.Verbose, tag, id, ex, message, arguments);
}
@@ -428,11 +423,10 @@ namespace Duplicati.Library.Logging
/// <param name="scope">The scope to finish.</param>
internal static void CloseScope(LogScope scope)
{
lock(m_lock)
lock (m_lock)
{
if (CurrentScope == scope && scope != m_root)
CurrentScope = scope.Parent;
m_log_instances.Remove(scope.InstanceID);
}
}
@@ -444,32 +438,12 @@ namespace Duplicati.Library.Logging
get
{
lock (m_lock)
{
var cur = CallContext.GetData(LOGICAL_CONTEXT_KEY) as string;
if (cur == null || cur == m_root.InstanceID)
return m_root;
LogScope sc;
if (!m_log_instances.TryGetValue(cur, out sc))
throw new Exception("Unable to find log in lookup table, this may be caused by attempting to transport call contexts between AppDomains (e.g. with remoting calls)");
return sc;
}
return _currentScope.Value ?? m_root;
}
private set
{
lock (m_lock)
{
if (value != null)
{
m_log_instances[value.InstanceID] = value;
CallContext.SetData(LOGICAL_CONTEXT_KEY, value.InstanceID);
}
else
{
CallContext.SetData(LOGICAL_CONTEXT_KEY, null);
}
}
_currentScope.Value = value;
}
}
}
Duplicati/Library/Main/Backend/BackendManager.Handler.cs
+3 -3
View File
@@ -415,7 +415,7 @@ partial class BackendManager
}
}
context.Statwriter.SendEvent(op.Operation, retries < maxRetries ? BackendEventType.Retrying : BackendEventType.Failed, op.RemoteFilename, op.Size);
context.Statwriter.SendEvent(op.Operation, retries <= maxRetries ? BackendEventType.Retrying : BackendEventType.Failed, op.RemoteFilename, op.Size);
// Check if we can recover from the error
var recovered = false;
@@ -428,14 +428,14 @@ partial class BackendManager
}
// We did not recover, so wait or give up
if (!recovered && retries < maxRetries && retryDelay.Ticks != 0)
if (!recovered && retries <= maxRetries && retryDelay.Ticks != 0)
{
var delay = Library.Utility.Utility.GetRetryDelay(retryDelay, retries, retryWithExponentialBackoff);
await Task.Delay(delay, context.TaskReader.ProgressToken).ConfigureAwait(false);
}
}
} while (retries < maxRetries);
} while (retries <= maxRetries);
// If we have a last exception, we failed
if (lastException != null)
Duplicati/Library/Main/Operation/BackupHandler.cs
+1
View File
@@ -530,6 +530,7 @@ namespace Duplicati.Library.Main.Operation
if (result.BackendWriter != null)
{
result.BackendWriter.KnownFileCount = database.GetRemoteVolumes().Count();
result.BackendWriter.KnownFilesets = database.GetRemoteVolumes().Count(x => x.Type == RemoteVolumeType.Files);
result.BackendWriter.KnownFileSize = database.GetRemoteVolumes().Select(x => Math.Max(0, x.Size)).Sum();
result.BackendWriter.UnknownFileCount = 0;
Duplicati/Library/Main/Operation/FilelistProcessor.cs
+1
View File
@@ -234,6 +234,7 @@ namespace Duplicati.Library.Main.Operation
select n).ToList();
log.KnownFileCount = remotelist.Count;
log.KnownFilesets = filesets.Count;
long knownFileSize = remotelist.Select(x => Math.Max(0, x.File.Size)).Sum();
log.KnownFileSize = knownFileSize;
log.UnknownFileCount = unknownlist.Count;
Duplicati/Library/Main/ResultClasses.cs
+1
View File
@@ -99,6 +99,7 @@ namespace Duplicati.Library.Main
public long UnknownFileCount { get; set; }
public long KnownFileCount { get; set; }
public long KnownFileSize { get; set; }
public long KnownFilesets { get; set; }
public DateTime LastBackupDate { get; set; }
public long BackupListCount { get; set; }
public long TotalQuotaSpace { get; set; }
Duplicati/Library/RemoteControl/ExchangeDataTypes.cs
+37 -1
View File
@@ -45,6 +45,10 @@ internal enum MessageType
/// </summary>
Command,
/// <summary>
/// A control message
/// </summary>
Control,
/// <summary>
/// An unknown command
/// </summary>
Unknown
@@ -56,7 +60,9 @@ internal enum MessageType
/// <param name="Token">The client token</param>
/// <param name="PublicKey">The client public key</param>
/// <param name="Version">The version of the client</param>
public record AuthMessage(string Token, string PublicKey, string ClientVersion, int ProtocolVersion);
/// <param name="ProtocolVersion">The protocol version of the client</param>
/// <param name="Metadata">The optional metadata to send</param>
public record AuthMessage(string Token, string PublicKey, string ClientVersion, int ProtocolVersion, Dictionary<string, string?>? Metadata);
/// <summary>
/// A message authentication response
@@ -75,6 +81,35 @@ internal sealed record AuthResultMessage(bool? Accepted, bool? WillReplaceToken,
/// <param name="ServerVersion">The version of the server</param>
public sealed record WelcomeMessage(string PublicKeyHash, string MachineName, string ServerVersion, IEnumerable<int> SupportedProtocolVersions);
/// <summary>
/// A message with a control request
/// </summary>
/// <param name="Command">The control command being requested</param>
/// <param name="Parameters">The parameters for the command</param>
public sealed record ControlRequestMessage(string Command, Dictionary<string, string?> Parameters)
{
/// <summary>
/// The configure report url command
/// </summary>
public const string ConfigureReportUrlSet = "configure-report-url:set";
/// <summary>
/// A control message to return if the report url is configured
/// </summary>
public const string ConfigureReportUrlGet = "configure-report-url:get";
/// <summary>
/// The url parameter for the configure report url command
/// </summary>
public const string ConfigureReportUrlParameter = "url";
}
/// <summary>
/// A message with a control response
/// </summary>
/// <param name="Success">Whether the command was successful</param>
/// <param name="Output">The output of the command</param>
/// <param name="ErrorMessage">The error message if the command failed</param>
public sealed record ControlResponseMessage(bool Success, Dictionary<string, string?>? Output, string? ErrorMessage);
/// <summary>
/// A message to send a command
/// </summary>
@@ -180,6 +215,7 @@ internal sealed record EnvelopedMessage
"auth" => MessageType.Auth,
"pong" => MessageType.Pong,
"command" => MessageType.Command,
"control" => MessageType.Control,
_ => MessageType.Unknown
};
}
Duplicati/Library/RemoteControl/KeepRemoteConnection.cs
+113 -11
View File
@@ -19,10 +19,12 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.Diagnostics;
using System.Net;
using System.Net.Http.Headers;
using System.Security.Cryptography;
using System.Text.Json;
using Duplicati.Library.AutoUpdater;
using Duplicati.Library.Logging;
namespace Duplicati.Library.RemoteControl;
@@ -133,12 +135,20 @@ public class KeepRemoteConnection : IDisposable
/// </summary>
private RSA? _serverPublicKey;
/// <summary>
/// The callback to call when connecting
/// </summary>
private readonly Func<Dictionary<string, string?>, Task<Dictionary<string, string?>>> _onConnect;
/// <summary>
/// The callback to call when rekeying
/// </summary>
private readonly Func<ClaimedClientData, Task> _onReKey;
/// <summary>
/// The callback to call when a message is received
/// The callback to call when a control message is received
/// </summary>
private readonly Func<ControlMessage, Task> _onControl;
/// <summary>
/// The callback to call when a command message is received
/// </summary>
private readonly Func<CommandMessage, Task> _onMessage;
@@ -168,16 +178,32 @@ public class KeepRemoteConnection : IDisposable
/// </summary>
/// <param name="serverUrl">The url to use</param>
/// <param name="JWT">The JWT token to use</param>
/// <param name="certificateUrl">The certificate url to use</param>
/// <param name="serverKeys">The server keys to use</param>
/// <param name="cancellationToken">The token to cancel the connection</param>
private KeepRemoteConnection(string serverUrl, string JWT, string certificateUrl, IEnumerable<MiniServerCertificate> serverKeys, CancellationToken cancellationToken, Func<ClaimedClientData, Task> onReKey, Func<CommandMessage, Task> onMessage)
/// <param name="onConnect">The callback to call when connecting</param>
/// <param name="onReKey">The callback to call when rekeying</param>
/// <param name="onControl">The callback to call when a control message is received</param>
/// <param name="onMessage">The callback to call when a command message is received</param>
private KeepRemoteConnection(
string serverUrl,
string JWT,
string certificateUrl,
IEnumerable<MiniServerCertificate> serverKeys,
CancellationToken cancellationToken,
Func<Dictionary<string, string?>, Task<Dictionary<string, string?>>> onConnect,
Func<ClaimedClientData, Task> onReKey,
Func<ControlMessage, Task> onControl,
Func<CommandMessage, Task> onMessage)
{
_serverUrl = serverUrl;
_certificateUrl = certificateUrl;
_token = JWT;
_serverKeys = serverKeys;
_cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
_onConnect = onConnect;
_onReKey = onReKey;
_onControl = onControl;
_onMessage = onMessage;
_client = new Websocket.Client.WebsocketClient(new Uri(serverUrl));
@@ -300,13 +326,27 @@ public class KeepRemoteConnection : IDisposable
}
_state = ConnectionState.WelcomeReceived;
// Prepare basic metadata and allow additional metadata to be added
var metadata = await _onConnect(new Dictionary<string, string?>() {
{ "client-version", UpdaterManager.SelfVersion?.Version ?? "0.0.0.0" },
{ "client-id", ClientId },
{ "client-uptime", (DateTime.Now - Process.GetCurrentProcess().StartTime).ToString() },
{ "machine-name", DataFolderManager.MachineName },
{ "machine-id", DataFolderManager.MachineID },
{ "install-id", DataFolderManager.InstallID },
{ "machine-os", UpdaterManager.OperatingSystemName },
{ "package-id", UpdaterManager.PackageTypeId },
});
SendEnvelope(
welcomeEnvelope.RespondWith(
new AuthMessage(
_token,
ClientKey.ExportRSAPublicKeyPem(),
AutoUpdater.UpdaterManager.SelfVersion?.Version ?? "0.0.0",
PROTOCOL_VERSION
UpdaterManager.SelfVersion?.Version ?? "0.0.0.0",
PROTOCOL_VERSION,
metadata
),
"auth"
),
@@ -352,6 +392,13 @@ public class KeepRemoteConnection : IDisposable
));
break;
case MessageType.Control:
await _onControl(new ControlMessage(
envelope.GetPayload<ControlRequestMessage>(),
response => SendEnvelope(envelope.RespondWith(response))
));
break;
default:
throw new ProtocolViolationException("Unexpected message");
}
@@ -403,13 +450,24 @@ public class KeepRemoteConnection : IDisposable
/// <param name="certificateUrl">The certificate url to use</param>
/// <param name="serverKeys">The server keys to use</param>
/// <param name="cancellationToken">The token to cancel the connection</param>
/// <param name="onConnect">The callback to call when connecting</param>
/// <param name="onReKey">The callback to call when rekeying</param>
/// <param name="onMessage">The callback to call when a message is received</param>
/// <param name="onControl">The callback to call when a control message is received</param>
/// <param name="onMessage">The callback to call when a command message is received</param>
/// <returns></returns>
public static Task Start(string serverUrl, string JWT, string certificateUrl, IEnumerable<MiniServerCertificate> serverKeys, CancellationToken cancellationToken, Func<ClaimedClientData, Task> onReKey, Func<CommandMessage, Task> onMessage)
public static Task Start(
string serverUrl,
string JWT,
string certificateUrl,
IEnumerable<MiniServerCertificate> serverKeys,
CancellationToken cancellationToken,
Func<Dictionary<string, string?>, Task<Dictionary<string, string?>>> onConnect,
Func<ClaimedClientData, Task> onReKey,
Func<ControlMessage, Task> onControl,
Func<CommandMessage, Task> onMessage)
=> Task.Run(async () =>
{
using var connection = new KeepRemoteConnection(serverUrl, JWT, certificateUrl, serverKeys, cancellationToken, onReKey, onMessage);
using var connection = new KeepRemoteConnection(serverUrl, JWT, certificateUrl, serverKeys, cancellationToken, onConnect, onReKey, onControl, onMessage);
await connection._runnerTask;
});
@@ -477,14 +535,24 @@ public class KeepRemoteConnection : IDisposable
/// </summary>
/// <param name="serverUrl">The url to use</param>
/// <param name="JWT">The JWT token to use</param>
///
/// <param name="certificateUrl">The certificate url to use</param>
/// <param name="serverKeys">The server keys to use</param>
/// <param name="onConnect">The callback to call when connecting</param>
/// <param name="onReKey">The callback to call when rekeying</param>
/// <param name="onControl">The callback to call when a control message is received</param>
/// <param name="onMessage">The callback to call when a message is received</param>
/// <param name="cancellationToken">The token to cancel the connection</param>
/// <returns>The connection object</returns>
public static KeepRemoteConnection CreateRemoteListener(string serverUrl, string JWT, string certificateUrl, IEnumerable<MiniServerCertificate> serverKeys, CancellationToken cancellationToken, Func<ClaimedClientData, Task> onReKey, Func<CommandMessage, Task> onMessage)
=> new KeepRemoteConnection(serverUrl, JWT, certificateUrl, serverKeys, cancellationToken, onReKey, onMessage);
public static KeepRemoteConnection CreateRemoteListener(
string serverUrl,
string JWT,
string certificateUrl,
IEnumerable<MiniServerCertificate> serverKeys,
CancellationToken cancellationToken,
Func<Dictionary<string, string?>, Task<Dictionary<string, string?>>> onConnect,
Func<ClaimedClientData, Task> onReKey,
Func<ControlMessage, Task> onControl,
Func<CommandMessage, Task> onMessage)
=> new KeepRemoteConnection(serverUrl, JWT, certificateUrl, serverKeys, cancellationToken, onConnect, onReKey, onControl, onMessage);
/// <summary>
/// Requests a certificate refresh
@@ -587,4 +655,38 @@ public class KeepRemoteConnection : IDisposable
}
}
}
/// <summary>
/// A wrapper for allowing external code to handle a control message
/// </summary>
public sealed class ControlMessage
{
/// <summary>
/// The callback method that will receive the response
/// </summary>
private readonly Func<ControlResponseMessage, bool> _respondCommand;
/// <summary>
/// The command request message
/// </summary>
public ControlRequestMessage ControlRequestMessage { get; }
/// <summary>
/// Creates a new command message
/// </summary>
/// <param name="controlRequestMessage">The command request message</param>
/// <param name="respondCommand">The callback method that will receive the response</param>
public ControlMessage(ControlRequestMessage controlRequestMessage, Func<ControlResponseMessage, bool> respondCommand)
{
ControlRequestMessage = controlRequestMessage;
_respondCommand = respondCommand;
}
/// <summary>
/// Responds to the command message
/// </summary>
/// <param name="response">The response to send</param>
/// <returns>True if the response was sent</returns>
public bool Respond(ControlResponseMessage response)
=> _respondCommand(response);
}
}
Duplicati/Library/RestAPI/Database/Connection.cs
+25 -25
View File
@@ -38,8 +38,8 @@ namespace Duplicati.Server.Database
{
public class Connection : IDisposable
{
private readonly System.Data.IDbConnection m_connection;
private readonly System.Data.IDbCommand m_errorcmd;
private readonly IDbConnection m_connection;
private readonly IDbCommand m_errorcmd;
public readonly object m_lock = new object();
public const int ANY_BACKUP_ID = -1;
public const int SERVER_SETTINGS_ID = -2;
@@ -65,7 +65,7 @@ namespace Duplicati.Server.Database
])
.ToHashSet(StringComparer.OrdinalIgnoreCase);
public Connection(System.Data.IDbConnection connection, bool disableFieldEncryption, EncryptedFieldHelper.KeyInstance? key)
public Connection(IDbConnection connection, bool disableFieldEncryption, EncryptedFieldHelper.KeyInstance? key)
{
m_encryptSensitiveFields = !disableFieldEncryption;
m_key = key;
@@ -141,7 +141,7 @@ namespace Duplicati.Server.Database
}
}
public void ExecuteWithCommand(Action<System.Data.IDbCommand> f)
public void ExecuteWithCommand(Action<IDbCommand> f)
{
lock (m_lock)
using (var cmd = m_connection.CreateCommand())
@@ -214,7 +214,7 @@ namespace Duplicati.Server.Database
.ToDictionary((k) => k.Key, (k) => k.Value);
}
internal void SetMetadata(IDictionary<string, string> values, long id, System.Data.IDbTransaction? transaction)
internal void SetMetadata(IDictionary<string, string> values, long id, IDbTransaction? transaction)
{
lock (m_lock)
using (var tr = transaction == null ? m_connection.BeginTransaction() : null)
@@ -246,7 +246,7 @@ namespace Duplicati.Server.Database
.ToArray();
}
internal void SetFilters(IEnumerable<IFilter> values, long id, System.Data.IDbTransaction? transaction = null)
internal void SetFilters(IEnumerable<IFilter> values, long id, IDbTransaction? transaction = null)
{
lock (m_lock)
using (var tr = transaction == null ? m_connection.BeginTransaction() : null)
@@ -279,7 +279,7 @@ namespace Duplicati.Server.Database
.ToArray();
}
internal void SetSettings(IEnumerable<ISetting> values, long id, System.Data.IDbTransaction? transaction = null)
internal void SetSettings(IEnumerable<ISetting> values, long id, IDbTransaction? transaction = null)
{
lock (m_lock)
using (var tr = transaction == null ? m_connection.BeginTransaction() : null)
@@ -319,7 +319,7 @@ namespace Duplicati.Server.Database
.ToArray();
}
internal void SetSources(IEnumerable<string> values, long id, System.Data.IDbTransaction transaction)
internal void SetSources(IEnumerable<string> values, long id, IDbTransaction transaction)
{
lock (m_lock)
using (var tr = transaction == null ? m_connection.BeginTransaction() : null)
@@ -715,7 +715,7 @@ namespace Duplicati.Server.Database
}
}
private void AddOrUpdateSchedule(ISchedule item, System.Data.IDbTransaction tr)
private void AddOrUpdateSchedule(ISchedule item, IDbTransaction tr)
{
lock (m_lock)
{
@@ -981,7 +981,7 @@ namespace Duplicati.Server.Database
.ToDictionary(x => x.Key, x => x.Last().Value);
}
public void SetUISettings(string scheme, IDictionary<string, string?> values, System.Data.IDbTransaction? transaction = null)
public void SetUISettings(string scheme, IDictionary<string, string?> values, IDbTransaction? transaction = null)
{
lock (m_lock)
using (var tr = transaction == null ? m_connection.BeginTransaction() : null)
@@ -1002,7 +1002,7 @@ namespace Duplicati.Server.Database
}
}
public void UpdateUISettings(string scheme, IDictionary<string, string?> values, System.Data.IDbTransaction? transaction = null)
public void UpdateUISettings(string scheme, IDictionary<string, string?> values, IDbTransaction? transaction = null)
{
lock (m_lock)
using (var tr = transaction == null ? m_connection.BeginTransaction() : null)
@@ -1066,24 +1066,24 @@ namespace Duplicati.Server.Database
}
}
private static DateTime ConvertToDateTime(System.Data.IDataReader rd, int index)
private static DateTime ConvertToDateTime(IDataReader rd, int index)
{
var unixTime = ConvertToInt64(rd, index);
return unixTime == 0 ? new DateTime(0) : Library.Utility.Utility.EPOCH.AddSeconds(unixTime);
}
private static bool ConvertToBoolean(System.Data.IDataReader rd, int index)
private static bool ConvertToBoolean(IDataReader rd, int index)
{
return ConvertToInt64(rd, index) == 1;
}
private static string? ConvertToString(System.Data.IDataReader rd, int index)
private static string? ConvertToString(IDataReader rd, int index)
{
var r = rd.GetValue(index);
return r == null || r == DBNull.Value ? null : r.ToString();
}
private static long ConvertToInt64(System.Data.IDataReader rd, int index)
private static long ConvertToInt64(IDataReader rd, int index)
{
try
{
@@ -1097,20 +1097,20 @@ namespace Duplicati.Server.Database
return -1;
}
private static long ExecuteScalarInt64(System.Data.IDbCommand cmd, long defaultValue = -1)
private static long ExecuteScalarInt64(IDbCommand cmd, long defaultValue = -1)
{
using (var rd = cmd.ExecuteReader())
return rd.Read() ? ConvertToInt64(rd, 0) : defaultValue;
}
private static string? ExecuteScalarString(System.Data.IDbCommand cmd)
private static string? ExecuteScalarString(IDbCommand cmd)
{
using (var rd = cmd.ExecuteReader())
return rd.Read() ? ConvertToString(rd, 0) : null;
}
private object? ConvertToEnum(Type enumType, System.Data.IDataReader rd, int index, object? @default)
private object? ConvertToEnum(Type enumType, IDataReader rd, int index, object? @default)
{
try
{
@@ -1124,13 +1124,13 @@ namespace Duplicati.Server.Database
}
// Overloaded function for legacy functionality
private bool DeleteFromDb(string tablename, long id, System.Data.IDbTransaction? transaction = null)
private bool DeleteFromDb(string tablename, long id, IDbTransaction? transaction = null)
{
return DeleteFromDb(tablename, id, "ID", transaction);
}
// New function that allows to delete rows from tables with arbitrary identifier values (e.g. ID or BackupID)
private bool DeleteFromDb(string tablename, long id, string identifier, System.Data.IDbTransaction? transaction = null)
private bool DeleteFromDb(string tablename, long id, string identifier, IDbTransaction? transaction = null)
{
if (transaction == null)
{
@@ -1158,14 +1158,14 @@ namespace Duplicati.Server.Database
}
}
private static IEnumerable<T> Read<T>(System.Data.IDbCommand cmd, Func<System.Data.IDataReader, T> f)
private static IEnumerable<T> Read<T>(IDbCommand cmd, Func<IDataReader, T> f)
{
using (var rd = cmd.ExecuteReader())
while (rd.Read())
yield return f(rd);
}
private static IEnumerable<T> Read<T>(System.Data.IDataReader rd, Func<T> f)
private static IEnumerable<T> Read<T>(IDataReader rd, Func<T> f)
{
while (rd.Read())
yield return f();
@@ -1226,7 +1226,7 @@ namespace Duplicati.Server.Database
}, sql, args);
}
private void OverwriteAndUpdateDb<T>(System.Data.IDbTransaction? transaction, string? deleteSql, object[]? deleteArgs, IEnumerable<T> values, bool updateExisting)
private void OverwriteAndUpdateDb<T>(IDbTransaction? transaction, string? deleteSql, object[]? deleteArgs, IEnumerable<T> values, bool updateExisting)
{
var properties = GetORMFields<T>();
var idfield = properties.FirstOrDefault(x => x.Name == "ID")
@@ -1286,7 +1286,7 @@ namespace Duplicati.Server.Database
}
}
private IEnumerable<T> ReadFromDb<T>(Func<System.Data.IDataReader, T> f, string sql, params object?[]? args)
private IEnumerable<T> ReadFromDb<T>(Func<IDataReader, T> f, string sql, params object?[]? args)
{
using (var cmd = m_connection.CreateCommand())
{
@@ -1297,7 +1297,7 @@ namespace Duplicati.Server.Database
}
}
private void OverwriteAndUpdateDb<T>(System.Data.IDbTransaction? transaction, string? deleteSql, object[]? deleteArgs, IEnumerable<T> values, string insertSql, Func<T, object?[]> f)
private void OverwriteAndUpdateDb<T>(IDbTransaction? transaction, string? deleteSql, object[]? deleteArgs, IEnumerable<T> values, string insertSql, Func<T, object?[]> f)
{
using (var cmd = m_connection.CreateCommand())
{
Duplicati/Library/RestAPI/Database/ServerSettings.cs
+15
View File
@@ -71,6 +71,7 @@ namespace Duplicati.Server.Database
public const string TIMEZONE_OPTION = "server-timezone";
public const string PAUSED_UNTIL = "paused-until";
public const string LAST_UPDATE_CHECK_VERSION = "last-update-check-version";
public const string ADDITIONAL_REPORT_URL = "additional-report-url";
}
private readonly Dictionary<string, string?> settings;
@@ -847,6 +848,20 @@ namespace Duplicati.Server.Database
SaveSettings();
}
}
public string? AdditionalReportUrl
{
get
{
return settings[CONST.ADDITIONAL_REPORT_URL];
}
set
{
lock (databaseConnection.m_lock)
settings[CONST.ADDITIONAL_REPORT_URL] = value;
SaveSettings();
}
}
}
}
Duplicati/Library/RestAPI/Runner.cs
+13
View File
@@ -736,6 +736,7 @@ namespace Duplicati.Server
backup.Metadata["TargetFilesSize"] = r.KnownFileSize.ToString();
backup.Metadata["TargetFilesCount"] = r.KnownFileCount.ToString();
backup.Metadata["TargetFilesetsCount"] = r.KnownFilesets.ToString();
backup.Metadata["TargetSizeString"] = Duplicati.Library.Utility.Utility.FormatSizeString(r.KnownFileSize);
}
}
@@ -919,6 +920,18 @@ namespace Duplicati.Server
// The server hangs if the module is enabled as there is no console attached
DisableModule("console-password-input", options);
// Patch in additional report urls
var additionalReportUrl = FIXMEGlobal.DataConnection.ApplicationSettings.AdditionalReportUrl;
if (!string.IsNullOrWhiteSpace(additionalReportUrl))
{
options["send-http-json-urls"] = string.Join(";",
new[] {
options.GetValueOrDefault("send-http-json-urls"),
additionalReportUrl
}.Where(x => !string.IsNullOrWhiteSpace(x))
);
}
return options;
}
Duplicati/Library/RestAPI/Scheduler.cs
+1 -1
View File
@@ -143,7 +143,7 @@ namespace Duplicati.Server
get
{
lock (m_lock)
return m_schedule.ToList();
return (m_schedule ?? []).ToList();
}
}
Duplicati/Library/Utility/CallContextSettings.cs
+74 -133
View File
@@ -1,32 +1,32 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using System.Threading;
using Duplicati.Library.Interface;
using Duplicati.Library.Logging;
namespace Duplicati.Library.Utility
{
@@ -81,12 +81,9 @@ namespace Duplicati.Library.Utility
}
set
{
lock (CallContextSettings<SystemSettings>._lock)
{
var st = CallContextSettings<SystemSettings>.Settings;
st.Tempdir = value;
CallContextSettings<SystemSettings>.Settings = st;
}
var st = CallContextSettings<SystemSettings>.Settings;
st.Tempdir = value;
CallContextSettings<SystemSettings>.Settings = st;
}
}
@@ -228,20 +225,58 @@ namespace Duplicati.Library.Utility
public static class CallContextSettings<T>
{
/// <summary>
/// The settings that are stored in the call context, which are not serializable
/// The call context settings for OAuth
/// </summary>
private static readonly Dictionary<string, T> _settings = new Dictionary<string, T>();
private static readonly AsyncLocal<T> _settings = new AsyncLocal<T>();
/// <summary>
/// The context setting types that are used and need their private setting key
/// to prevent overwriting each others settings.
/// The instances that are currently active
/// </summary>
private static string contextSettingsType = null;
private static readonly ConcurrentDictionary<T, object> _instances = new ConcurrentDictionary<T, object>();
/// <summary>
/// Lock for protecting the dictionary
/// Disposable class for setting call context settings
/// </summary>
public static readonly object _lock = new object();
/// <typeparam name="T">The type of the settings</typeparam>
private sealed class Disposer : IDisposable
{
/// <summary>
/// A flag indicating if the object has been disposed
/// </summary>
private bool m_disposed = false;
/// <summary>
/// The previous value
/// </summary>
private T m_prev;
/// <summary>
/// The current value
/// </summary>
private T m_current;
/// <summary>
/// Initializes a new instance of the <see cref="Disposer"/> class.
/// </summary>
/// <param name="current">The current server URL</param>
/// <param name="prev">The previous server URL</param>
public Disposer(T current, T prev)
{
_settings.Value = m_current = current;
this.m_prev = prev;
_instances.TryAdd(current, null);
}
/// <inheritdoc/>
public void Dispose()
{
if (m_disposed)
return;
GC.SuppressFinalize(this);
m_disposed = true;
if (_settings.Value.Equals(m_current))
_settings.Value = m_prev;
_instances.TryRemove(m_current, out _);
}
}
/// <summary>
/// Gets or sets the values in the current call context
@@ -249,40 +284,8 @@ namespace Duplicati.Library.Utility
/// <value>The settings.</value>
public static T Settings
{
get
{
var key = ContextID;
if (string.IsNullOrWhiteSpace(key))
return default(T);
if (!_settings.TryGetValue(key, out T res))
lock (_lock) // if not present but context id is not null, wait
if (!_settings.TryGetValue(key, out res))
return default(T);
return res;
}
set
{
var key = ContextID;
if (!string.IsNullOrWhiteSpace(key))
lock (_lock)
_settings[key] = value;
}
}
/// <summary>
/// Starts the context wit a default value.
/// </summary>
/// <returns>The disposable handle for the context.</returns>
/// <param name="initial">The initial value.</param>
public static IDisposable StartContext(string settingsKey, T initial = default(T))
{
contextSettingsType = settingsKey;
var res = new ContextGuard();
Settings = initial;
return res;
get => _settings.Value;
set => _settings.Value = value;
}
/// <summary>
@@ -291,72 +294,10 @@ namespace Duplicati.Library.Utility
/// <returns>The disposable handle for the context.</returns>
/// <param name="initial">The initial value.</param>
public static IDisposable StartContext(T initial = default(T))
{
return StartContext(typeof(T).ToString(), initial);
}
=> new Disposer(initial, _settings.Value);
/// <summary>
/// Gets all instances currently registered
/// </summary>
internal static T[] GetAllInstances()
{
lock (_lock)
return _settings.Values.ToArray();
}
public static IEnumerable<T> GetAllInstances()
=> _instances.Keys.ToList();
/// <summary>
/// Help class for providing a way to release resources associated with a call context
/// </summary>
private class ContextGuard : IDisposable
{
/// <summary>
/// Randomly generated identifier
/// </summary>
private readonly string ID = Guid.NewGuid().ToString();
/// <summary>
/// Initializes a new instance of the
/// <see cref="T:Duplicati.Library.Utility.CallContextSettings`1.ContextGuard"/> class,
/// and sets up the call context
/// </summary>
public ContextGuard()
{
CallContext.SetData(contextSettingsType, ID);
}
/// <summary>
/// Releases all resource used by the
/// <see cref="T:Duplicati.Library.Utility.CallContextSettings`1.ContextGuard"/> object.
/// </summary>
/// <remarks>Call <see cref="Dispose"/> when you are finished using the
/// <see cref="T:Duplicati.Library.Utility.CallContextSettings`1.ContextGuard"/>. The <see cref="Dispose"/>
/// method leaves the <see cref="T:Duplicati.Library.Utility.CallContextSettings`1.ContextGuard"/> in an
/// unusable state. After calling <see cref="Dispose"/>, you must release all references to the
/// <see cref="T:Duplicati.Library.Utility.CallContextSettings`1.ContextGuard"/> so the garbage collector
/// can reclaim the memory that the
/// <see cref="T:Duplicati.Library.Utility.CallContextSettings`1.ContextGuard"/> was occupying.</remarks>
public void Dispose()
{
lock (_lock)
{
// Release the resources, if any
_settings.Remove(ID);
}
CallContext.SetData(contextSettingsType, null);
}
}
/// <summary>
/// Gets the context ID for this call.
/// </summary>
/// <value>The context identifier.</value>
private static string ContextID
{
get
{
return contextSettingsType != null ? CallContext.GetData(contextSettingsType) as string : null;
}
}
}
}
Duplicati/Library/Utility/Utility.cs
+12
View File
@@ -774,6 +774,18 @@ namespace Duplicati.Library.Utility
return (T)(object)flags;
}
/// <summary>
/// Parses an option with int value, returning the default value if the option is not found or cannot be parsed
/// </summary>
/// <param name="options">The set of options to look for the setting in</param>
/// <param name="value">The value to look for in the settings</param>
/// <param name="default">default value</param>
/// <returns></returns>
public static int ParseIntOption(IReadOnlyDictionary<string, string?> options, string value, int @default)
{
return options.TryGetValue(value, out var opt) && int.TryParse(opt ?? string.Empty, out var result) ? result : @default;
}
/// <summary>
/// Converts a sequence of bytes to a hex string
Duplicati/UnitTest/DisruptionTests.cs
+330 -12
View File
@@ -25,6 +25,7 @@ using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Duplicati.Library.Interface;
using Duplicati.Library.Logging;
using Duplicati.Library.Main;
using Duplicati.Library.Main.Database;
using Duplicati.Library.Main.Volumes;
@@ -50,11 +51,12 @@ namespace Duplicati.UnitTest
}
}
private async Task<IBackupResults> RunPartialBackup(Controller controller)
private async Task<(IBackupResults Result, List<string> ModifiedFiles)> RunPartialBackup(Controller controller)
{
this.ModifySourceFiles();
var stopped = new TaskCompletionSource<bool>();
var modifiedFiles = new List<string>();
controller.OnOperationStarted = r =>
{
var pv = r as ITaskControlProvider;
@@ -63,7 +65,8 @@ namespace Duplicati.UnitTest
#if DEBUG
pv.TaskControl.TestMethodCallback = (path) =>
{
if (path.EndsWith(this.fileSizes[2] + "MB"))
modifiedFiles.Add(path);
if (modifiedFiles.Count == 3)
{
Thread.Sleep(500);
controller.Stop();
@@ -80,7 +83,7 @@ namespace Duplicati.UnitTest
if (t != stopped.Task)
throw new Exception("Backup task completed before we could stop it");
return await backupTask.ConfigureAwait(false);
return (await backupTask.ConfigureAwait(false), modifiedFiles);
}
[SetUp]
@@ -121,7 +124,7 @@ namespace Duplicati.UnitTest
// Run a partial backup.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false);
(var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(1, backupResults.Warnings.Count());
}
@@ -240,7 +243,7 @@ namespace Duplicati.UnitTest
DateTime thirdBackupTime;
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false);
(var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(1, backupResults.Warnings.Count());
thirdBackupTime = c.List().Filesets.First().Time;
@@ -267,7 +270,7 @@ namespace Duplicati.UnitTest
// even when all the "recent" backups are partial.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false);
(var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(1, backupResults.Warnings.Count());
DateTime fourthBackupTime = c.List().Filesets.First().Time;
@@ -319,7 +322,7 @@ namespace Duplicati.UnitTest
// Run a partial backup.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false);
(var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(1, backupResults.Warnings.Count());
}
@@ -327,7 +330,7 @@ namespace Duplicati.UnitTest
// Run a partial backup.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false);
(var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(1, backupResults.Warnings.Count());
}
@@ -347,7 +350,7 @@ namespace Duplicati.UnitTest
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
options["keep-versions"] = "2";
IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false);
(var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(1, backupResults.Warnings.Count());
DateTime fifthBackupTime = c.List().Filesets.First().Time;
@@ -413,7 +416,7 @@ namespace Duplicati.UnitTest
// Run a partial backup.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false);
(var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(1, backupResults.Warnings.Count());
@@ -477,7 +480,7 @@ namespace Duplicati.UnitTest
DateTime thirdBackupTime;
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false);
(var backupResults, _) = await this.RunPartialBackup(c).ConfigureAwait(false);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(1, backupResults.Warnings.Count());
thirdBackupTime = c.List().Filesets.First().Time;
@@ -596,9 +599,11 @@ namespace Duplicati.UnitTest
// Run a partial backup.
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
IBackupResults backupResults = await this.RunPartialBackup(c).ConfigureAwait(false);
(var backupResults, var modified) = await this.RunPartialBackup(c).ConfigureAwait(false);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(1, backupResults.Warnings.Count());
if (backupResults.ModifiedFiles == 0)
throw new Exception($"No files were modified, likely the stop was issued too early, list is: {string.Join(", ", modified)}");
Assert.That(backupResults.ModifiedFiles, Is.GreaterThan(0), "No files were added, likely the stop was issued too early");
Assert.That(backupResults.ModifiedFiles, Is.LessThan(fileSizes.Length), "All files were added, likely the stop was issued too late");
@@ -852,5 +857,318 @@ namespace Duplicati.UnitTest
Assert.AreEqual(3, listResults.Filesets.Count());
}
}
private class LogSink : IMessageSink
{
public List<string> StartedFiles { get; } = new List<string>();
public List<string> CompletedFiles { get; } = new List<string>();
public void BackendEvent(BackendActionType action, BackendEventType type, string path, long size)
{
if (action == BackendActionType.Put && type == BackendEventType.Started)
{
this.StartedFiles.Add(path);
}
if (action == BackendActionType.Put && type == BackendEventType.Completed)
{
this.CompletedFiles.Add(path);
}
}
public void SetBackendProgress(IBackendProgress progress)
{
}
public void SetOperationProgress(IOperationProgress progress)
{
}
public void WriteMessage(LogEntry entry)
{
}
}
[Test]
[Category("Disruption")]
public void TestFailedDblockUploadWithOperationCancellation()
{
var testopts = TestOptions;
testopts["number-of-retries"] = "1";
testopts["dblock-size"] = "10mb";
// Make a base backup
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Make a new backup that fails uploading a dblock file
ModifySourceFiles();
// Deterministic error backend
Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend());
var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER;
var uploadCount = 0;
var failedFile = string.Empty;
// Fail the dlist upload
DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) =>
{
if (action == DeterministicErrorBackend.BackendAction.PutBefore && remotename.Contains(".dblock."))
{
// Fail once with a cancellation
if (Interlocked.Increment(ref uploadCount) == 1)
{
failedFile = remotename;
throw new OperationCanceledException();
}
}
return false;
};
using (var c = new Library.Main.Controller(failtarget, testopts, null))
{
var sink = new LogSink();
c.AppendSink(sink);
var res = c.Backup(new string[] { DATAFOLDER });
if (uploadCount == 0)
Assert.Fail("Upload count was not incremented");
Assert.AreEqual(2, c.List().Filesets.Count());
Assert.AreEqual(7 * 2 + 1, sink.CompletedFiles.Count); // 7 completed (dblock + dindex) + 1 dlist
Assert.AreEqual(7 * 2 + 2, sink.StartedFiles.Count); // 1 retry
Assert.AreEqual(1, res.BackendStatistics.RetryAttempts);
}
ModifySourceFiles();
// Create a regular backup
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Verify that all is in order
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null))
{
var r = c.Test(long.MaxValue);
Assert.AreEqual(0, r.Errors.Count());
Assert.AreEqual(0, r.Warnings.Count());
Assert.IsFalse(r.Verifications.Any(p => p.Value.Any()));
}
// Test that we can recreate
var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite");
if (File.Exists(recreatedDatabaseFile))
File.Delete(recreatedDatabaseFile);
testopts["dbpath"] = recreatedDatabaseFile;
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IRepairResults repairResults = c.Repair();
Assert.AreEqual(0, repairResults.Errors.Count());
Assert.AreEqual(0, repairResults.Warnings.Count());
}
// Check that we have 3 versions
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IListResults listResults = c.List();
Assert.AreEqual(0, listResults.Errors.Count());
Assert.AreEqual(0, listResults.Warnings.Count());
Assert.AreEqual(3, listResults.Filesets.Count());
}
}
[Test]
[Category("Disruption")]
public void TestFailedDindexUploadWithOperationCancellation()
{
var testopts = TestOptions;
testopts["number-of-retries"] = "1";
testopts["dblock-size"] = "10mb";
// Make a base backup
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Make a new backup that fails uploading a dblock file
ModifySourceFiles();
// Deterministic error backend
Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend());
var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER;
var uploadCount = 0;
var failedFile = string.Empty;
// Fail the dlist upload
DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) =>
{
if (action == DeterministicErrorBackend.BackendAction.PutBefore && remotename.Contains(".dindex."))
{
// Fail once with a cancellation
if (Interlocked.Increment(ref uploadCount) == 1)
{
failedFile = remotename;
throw new OperationCanceledException();
}
}
return false;
};
using (var c = new Library.Main.Controller(failtarget, testopts, null))
{
var sink = new LogSink();
c.AppendSink(sink);
var res = c.Backup(new string[] { DATAFOLDER });
if (uploadCount == 0)
Assert.Fail("Upload count was not incremented");
Assert.AreEqual(2, c.List().Filesets.Count());
Assert.AreEqual(7 * 2 + 1, sink.CompletedFiles.Count); // 7 completed (dblock + dindex) + 1 dlist
Assert.AreEqual(7 * 2 + 2, sink.StartedFiles.Count); // 1 retry
Assert.AreEqual(sink.CompletedFiles.Count(x => x.Contains(".dblock.")), sink.CompletedFiles.Count(x => x.Contains(".dindex.")));
Assert.AreEqual(1, res.BackendStatistics.RetryAttempts);
}
ModifySourceFiles();
// Create a regular backup
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Verify that all is in order
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null))
{
var r = c.Test(long.MaxValue);
Assert.AreEqual(0, r.Errors.Count());
Assert.AreEqual(0, r.Warnings.Count());
Assert.IsFalse(r.Verifications.Any(p => p.Value.Any()));
}
// Test that we can recreate
var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite");
if (File.Exists(recreatedDatabaseFile))
File.Delete(recreatedDatabaseFile);
testopts["dbpath"] = recreatedDatabaseFile;
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IRepairResults repairResults = c.Repair();
Assert.AreEqual(0, repairResults.Errors.Count());
Assert.AreEqual(0, repairResults.Warnings.Count());
}
// Check that we have 3 versions
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IListResults listResults = c.List();
Assert.AreEqual(0, listResults.Errors.Count());
Assert.AreEqual(0, listResults.Warnings.Count());
Assert.AreEqual(3, listResults.Filesets.Count());
}
}
[Test]
[Category("Disruption")]
public void TestFailedDlistUploadWithOperationCancellation()
{
var testopts = TestOptions;
testopts["number-of-retries"] = "1";
testopts["dblock-size"] = "10mb";
// Make a base backup
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Make a new backup that fails uploading a dblock file
ModifySourceFiles();
// Deterministic error backend
Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend());
var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER;
var uploadCount = 0;
// Fail the dlist upload
DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) =>
{
if (action == DeterministicErrorBackend.BackendAction.PutBefore && remotename.Contains(".dlist."))
{
if (Interlocked.Increment(ref uploadCount) == 1)
throw new OperationCanceledException();
}
return false;
};
using (var c = new Library.Main.Controller(failtarget, testopts, null))
{
var res = c.Backup(new string[] { DATAFOLDER });
var dlistCount = Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).Count();
if (uploadCount == 0)
Assert.Fail("Upload count was not incremented");
Assert.AreEqual(2, c.List().Filesets.Count());
Assert.AreEqual(2, dlistCount);
Assert.AreEqual(1, res.BackendStatistics.RetryAttempts);
}
ModifySourceFiles();
// Create a regular backup
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Verify that all is in order
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null))
{
var r = c.Test(long.MaxValue);
Assert.AreEqual(0, r.Errors.Count());
Assert.AreEqual(0, r.Warnings.Count());
Assert.IsFalse(r.Verifications.Any(p => p.Value.Any()));
}
// Test that we can recreate
var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite");
if (File.Exists(recreatedDatabaseFile))
File.Delete(recreatedDatabaseFile);
testopts["dbpath"] = recreatedDatabaseFile;
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IRepairResults repairResults = c.Repair();
Assert.AreEqual(0, repairResults.Errors.Count());
Assert.AreEqual(0, repairResults.Warnings.Count());
}
// Check that we have 3 versions
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IListResults listResults = c.List();
Assert.AreEqual(0, listResults.Errors.Count());
Assert.AreEqual(0, listResults.Warnings.Count());
Assert.AreEqual(3, listResults.Filesets.Count());
}
}
}
}
Duplicati/WebserverCore/DayOfWeekStringEnumConverter.cs
+12 -10
View File
@@ -32,16 +32,18 @@ public class DayOfWeekStringEnumConverter : JsonConverterFactory
if (reader.TokenType != JsonTokenType.String)
throw new JsonException();
return reader.GetString()?.Trim()?.ToLowerInvariant() switch
var value = reader.GetString()?.Trim()?.ToLowerInvariant();
return value switch
{
"sun" => DayOfWeek.Sunday,
"mon" => DayOfWeek.Monday,
"tue" => DayOfWeek.Tuesday,
"wed" => DayOfWeek.Wednesday,
"thu" => DayOfWeek.Thursday,
"fri" => DayOfWeek.Friday,
"sat" => DayOfWeek.Saturday,
_ => throw new JsonException()
"sun" or "sunday" => DayOfWeek.Sunday,
"mon" or "monday" => DayOfWeek.Monday,
"tue" or "tuesday" => DayOfWeek.Tuesday,
"wed" or "wednesday" => DayOfWeek.Wednesday,
"thu" or "thursday" => DayOfWeek.Thursday,
"fri" or "friday" => DayOfWeek.Friday,
"sat" or "saturday" => DayOfWeek.Saturday,
_ => throw new JsonException($"Invalid day of week: {value}"),
};
}
@@ -71,7 +73,7 @@ public class DayOfWeekStringEnumConverter : JsonConverterFactory
writer.WriteStringValue("sat");
break;
default:
throw new JsonException();
throw new JsonException($"Invalid day of week: {value}");
}
writer.WriteStringValue(value.ToString());
}
Duplicati/WebserverCore/Duplicati.WebserverCore.csproj
-2
View File
@@ -17,8 +17,6 @@
<ItemGroup>
<PackageReference Include="Microsoft.AspNetCore.HostFiltering" Version="2.2.0" />
<PackageReference Include="SixLabors.ImageSharp" Version="3.1.5" />
<PackageReference Include="SixLabors.ImageSharp.Drawing" Version="2.1.3" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" />
<PackageReference Include="Microsoft.IdentityModel.Tokens" Version="7.5.0" />
<PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="7.5.0" />
Duplicati/WebserverCore/Endpoints/V1/ServerSetting.cs
-1
View File
@@ -19,7 +19,6 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Server;
using Duplicati.Server.Database;
using Duplicati.WebserverCore.Abstractions;
using Duplicati.WebserverCore.Exceptions;
Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs
+1
View File
@@ -66,6 +66,7 @@ public static class ServiceCollectionsExtensions
.AddTransient<ITokenFamilyStore, TokenFamilyStore>()
.AddTransient<ILoginProvider, LoginProvider>()
.AddSingleton<IRemoteController, RemoteControllerService>()
.AddSingleton<IRemoteControllerHandler, RemoteControllerHandler>()
.AddSingleton<IRemoteControllerRegistration, RemoteControllerRegistrationService>()
.AddSingleton<ISystemInfoProvider, SystemInfoProvider>();
Duplicati/WebserverCore/Middlewares/LanguageFilter.cs
+2 -1
View File
@@ -18,6 +18,7 @@
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.Globalization;
using Duplicati.Library.Localization;
namespace Duplicati.WebserverCore.Middlewares;
@@ -30,7 +31,7 @@ public class LanguageFilter : IEndpointFilter
? LocalizationService.ParseCulture(locale)
: null;
using (ci == null ? null : LocalizationService.TemporaryContext(ci))
using (LocalizationService.TemporaryContext(ci ?? CultureInfo.CurrentUICulture))
return await next(context);
}
Duplicati/WebserverCore/Services/IRemoteControllerHandler.cs
+58
View File
@@ -0,0 +1,58 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.RemoteControl;
namespace Duplicati.WebserverCore.Services;
/// <summary>
/// Interface for handling remote control messages.
/// </summary>
public interface IRemoteControllerHandler
{
/// <summary>
/// Prepares metadata for a remote control connection.
/// </summary>
/// <param name="metadata">The initial metadata</param>
/// <returns>The metadata to use</returns>
Task<Dictionary<string, string?>> OnConnect(Dictionary<string, string?> metadata);
/// <summary>
/// Re-keys the remote control connection.
/// </summary>
/// <param name="data">The data to re-key with</param>
/// <returns>An awaitable task</returns>
Task ReKey(ClaimedClientData data);
/// <summary>
/// Handles a control message.
/// </summary>
/// <param name="message">The control message to handle</param>
/// <returns>An awaitable task</returns>
Task OnControl(KeepRemoteConnection.ControlMessage message);
/// <summary>
/// Handles a command message.
/// </summary>
/// <param name="commandMessage">The command message to handle</param>
/// <returns>An awaitable task</returns>
Task OnMessage(KeepRemoteConnection.CommandMessage commandMessage);
}
Duplicati/WebserverCore/Services/RemoteControllerHandler.cs
+120
View File
@@ -0,0 +1,120 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.Net.Http.Headers;
using Duplicati.Library.Logging;
using Duplicati.Library.RemoteControl;
using Duplicati.Library.RestAPI;
using Duplicati.Server.Database;
using Duplicati.WebserverCore.Abstractions;
using Newtonsoft.Json;
namespace Duplicati.WebserverCore.Services;
/// <summary>
/// Controller for handling remote control messages.
/// </summary>
/// <param name="connection">The connection to the database</param>
/// <param name="httpClientFactory">The HTTP client factory</param>
/// <param name="jwtTokenProvider">The JWT token provider</param>
public class RemoteControllerHandler(Connection connection, IHttpClientFactory httpClientFactory, IJWTTokenProvider jwtTokenProvider) : IRemoteControllerHandler
{
/// <summary>
/// The log tag for this class.
/// </summary>
private static readonly string LOGTAG = Log.LogTagFromType<RemoteControllerHandler>();
/// <inheritdoc/>
public Task<Dictionary<string, string?>> OnConnect(Dictionary<string, string?> metadata)
{
metadata["feature:additional-report-url"] = connection.ApplicationSettings.AdditionalReportUrl;
return Task.FromResult(metadata);
}
/// <inheritdoc/>
public Task ReKey(ClaimedClientData data)
{
var oldCerts = connection.ApplicationSettings.RemoteControlConfig == null
? null
: JsonConvert.DeserializeObject<RemoteControlConfig>(connection.ApplicationSettings.RemoteControlConfig)?.ServerCertificates;
connection.ApplicationSettings.RemoteControlConfig = JsonConvert.SerializeObject(new RemoteControlConfig
{
Token = data.JWT,
ServerCertificates = data.ServerCertificates ?? oldCerts ?? [],
ServerUrl = data.ServerUrl,
CertificateUrl = data.CertificateUrl
});
if (!FIXMEGlobal.SettingsEncryptionKeyProvidedExternally)
{
// TODO: Implement changing the encryption key
// if (!string.IsNullOrWhiteSpace(data.LocalEncryptionKey) && data.LocalEncryptionKey != connection.ApplicationSettings.SettingsEncryptionKey)
// connection.ChangeDbKey(keydata.LocalEncryptionKey);
}
return Task.CompletedTask;
}
/// <inheritdoc/>
public async Task OnControl(KeepRemoteConnection.ControlMessage message)
{
// Make method async
await Task.CompletedTask;
Dictionary<string, string?>? result = null;
Log.WriteMessage(LogMessageType.Verbose, LOGTAG, "OnControl", "Received control message: {0}", message);
try
{
if (string.Equals(message.ControlRequestMessage.Command, ControlRequestMessage.ConfigureReportUrlSet, StringComparison.OrdinalIgnoreCase))
FIXMEGlobal.DataConnection.ApplicationSettings.AdditionalReportUrl = message.ControlRequestMessage.Parameters.GetValueOrDefault(ControlRequestMessage.ConfigureReportUrlParameter);
if (string.Equals(message.ControlRequestMessage.Command, ControlRequestMessage.ConfigureReportUrlGet, StringComparison.OrdinalIgnoreCase))
result = new Dictionary<string, string?> { { ControlRequestMessage.ConfigureReportUrlParameter, FIXMEGlobal.DataConnection.ApplicationSettings.AdditionalReportUrl } };
}
catch (Exception ex)
{
Log.WriteMessage(LogMessageType.Error, LOGTAG, "OnControl", ex, "Failed to handle control message: {0}", message);
message.Respond(new ControlResponseMessage(false, null, ex.Message));
return;
}
message.Respond(new ControlResponseMessage(true, result, null));
}
/// <inheritdoc/>
public async Task OnMessage(KeepRemoteConnection.CommandMessage commandMessage)
{
using var httpClient = httpClientFactory.CreateClient();
var token = jwtTokenProvider.CreateAccessToken("remote-control", jwtTokenProvider.TemporaryFamilyId, TimeSpan.FromMinutes(2));
httpClient.BaseAddress = new Uri($"{(connection.ApplicationSettings.UseHTTPS ? "https" : "http")}://127.0.0.1:{connection.ApplicationSettings.LastWebserverPort}");
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token);
var psk = Middlewares.PreSharedKeyFilter.PreSharedKey;
if (!string.IsNullOrWhiteSpace(psk))
httpClient.DefaultRequestHeaders.Add(Middlewares.PreSharedKeyFilter.HeaderName, psk);
await commandMessage.Handle(httpClient);
}
}
Duplicati/WebserverCore/Services/RemoteControllerService.cs
+6 -52
View File
@@ -19,9 +19,7 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.Net.Http.Headers;
using Duplicati.Library.RemoteControl;
using Duplicati.Library.RestAPI;
using Duplicati.Server.Database;
using Duplicati.WebserverCore.Abstractions;
using Newtonsoft.Json;
@@ -32,9 +30,8 @@ namespace Duplicati.WebserverCore.Services;
/// Controller for toggling remote control.
/// </summary>
/// <param name="connection">The connection to the database</param>
/// <param name="httpClientFactory">The HTTP client factory</param>
/// <param name="jwtTokenProvider">The JWT token provider</param>
public class RemoteControllerService(Connection connection, IHttpClientFactory httpClientFactory, IJWTTokenProvider jwtTokenProvider) : IRemoteController
/// <param name="controllerHandler">The remote controller handler</param>
public class RemoteControllerService(Connection connection, IRemoteControllerHandler controllerHandler) : IRemoteController
{
/// <summary>
/// Gets a value indicating whether remote control is enabled.
@@ -83,58 +80,15 @@ public class RemoteControllerService(Connection connection, IHttpClientFactory h
config.CertificateUrl,
config.ServerCertificates,
CancellationToken.None,
ReKey,
OnMessage
controllerHandler.OnConnect,
controllerHandler.ReKey,
controllerHandler.OnControl,
controllerHandler.OnMessage
);
connection.ApplicationSettings.RemoteControlEnabled = true;
}
/// <summary>
/// Re-keys the remote control connection.
/// </summary>
/// <param name="data">The data to re-key with</param>
/// <returns>An awaitable task</returns>
private Task ReKey(ClaimedClientData data)
{
var oldCerts = connection.ApplicationSettings.RemoteControlConfig == null
? null
: JsonConvert.DeserializeObject<RemoteControlConfig>(connection.ApplicationSettings.RemoteControlConfig)?.ServerCertificates;
connection.ApplicationSettings.RemoteControlConfig = JsonConvert.SerializeObject(new RemoteControlConfig
{
Token = data.JWT,
ServerCertificates = data.ServerCertificates ?? oldCerts ?? [],
ServerUrl = data.ServerUrl,
CertificateUrl = data.CertificateUrl
});
if (!FIXMEGlobal.SettingsEncryptionKeyProvidedExternally)
{
// TODO: Implement changing the encryption key
// if (!string.IsNullOrWhiteSpace(data.LocalEncryptionKey) && data.LocalEncryptionKey != connection.ApplicationSettings.SettingsEncryptionKey)
// connection.ChangeDbKey(keydata.LocalEncryptionKey);
}
return Task.CompletedTask;
}
/// <summary>
/// Handles a command message.
/// </summary>
/// <param name="commandMessage">The command message to handle</param>
/// <returns>An awaitable task</returns>
private async Task OnMessage(KeepRemoteConnection.CommandMessage commandMessage)
{
using var httpClient = httpClientFactory.CreateClient();
var token = jwtTokenProvider.CreateAccessToken("remote-control", jwtTokenProvider.TemporaryFamilyId, TimeSpan.FromMinutes(2));
httpClient.BaseAddress = new Uri($"{(connection.ApplicationSettings.UseHTTPS ? "https" : "http")}://127.0.0.1:{connection.ApplicationSettings.LastWebserverPort}");
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token);
await commandMessage.Handle(httpClient);
}
/// </inheritdoc>
public void Disable()
{
Duplicati/WebserverCore/Services/SystemInfoProvider.cs
+65 -39
View File
@@ -20,6 +20,7 @@
// DEALINGS IN THE SOFTWARE.
using System.Globalization;
using Duplicati.Library.AutoUpdater;
using Duplicati.Library.Localization;
using Duplicati.Library.RestAPI;
using Duplicati.Server;
using Duplicati.Server.Serialization.Interface;
@@ -205,62 +206,87 @@ public class SystemInfoProvider : ISystemInfoProvider
}
/// <summary>
/// Cached static system information.
/// Creates a new instance of the <see cref="StaticSystemInformation"/> class.
/// </summary>
private Lazy<StaticSystemInformation> _systemInfoBase = new(() => new StaticSystemInformation
{
APIVersion = 1,
PasswordPlaceholder = FIXMEGlobal.PASSWORD_PLACEHOLDER,
ServerVersion = UpdaterManager.SelfVersion.Version,
ServerVersionName = License.VersionNumbers.VERSION_NAME,
ServerVersionType = UpdaterManager.SelfVersion.ReleaseType,
RemoteControlRegistrationUrl = Library.RemoteControl.RegisterForRemote.DefaultRegisterationUrl,
StartedBy = FIXMEGlobal.Origin,
DefaultUpdateChannel = AutoUpdateSettings.DefaultUpdateChannel.ToString(),
DefaultUsageReportLevel = Library.UsageReporter.Reporter.DefaultReportLevel,
OSType = UpdaterManager.OperatingSystemName,
OSVersion = Library.UsageReporter.OSInfoHelper.PlatformString,
DirectorySeparator = Path.DirectorySeparatorChar,
PathSeparator = Path.PathSeparator,
CaseSensitiveFilesystem = Library.Utility.Utility.IsFSCaseSensitive,
MachineName = Environment.MachineName,
PackageTypeId = UpdaterManager.PackageTypeId,
UserName = OperatingSystem.IsWindows() ? System.Security.Principal.WindowsIdentity.GetCurrent().Name : Environment.UserName,
NewLine = Environment.NewLine,
CLRVersion = Environment.Version.ToString(),
Options = Server.Serializable.ServerSettings.Options,
CompressionModules = Server.Serializable.ServerSettings.CompressionModules,
EncryptionModules = Server.Serializable.ServerSettings.EncryptionModules,
BackendModules = Server.Serializable.ServerSettings.BackendModules,
GenericModules = Server.Serializable.ServerSettings.GenericModules,
WebModules = Server.Serializable.ServerSettings.WebModules,
ConnectionModules = Server.Serializable.ServerSettings.ConnectionModules,
ServerModules = Server.Serializable.ServerSettings.ServerModules,
SecretProviderModules = Server.Serializable.ServerSettings.SecretProviderModules,
UsingAlternateUpdateURLs = AutoUpdateSettings.UsesAlternateURLs,
LogLevels = Enum.GetNames(typeof(Library.Logging.LogMessageType)),
SpecialFolders = SpecialFolders.Nodes.Select(n => new Dto.SystemInfoDto.SpecialFolderDto { ID = n.id, Path = n.resolvedpath }).ToArray(),
SupportedLocales = Library.Localization.LocalizationService.SupportedCultures
/// <returns>The new instance.</returns>
private static StaticSystemInformation CreateStaticSystemInformation()
=> new StaticSystemInformation
{
APIVersion = 1,
PasswordPlaceholder = FIXMEGlobal.PASSWORD_PLACEHOLDER,
ServerVersion = UpdaterManager.SelfVersion.Version,
ServerVersionName = License.VersionNumbers.VERSION_NAME,
ServerVersionType = UpdaterManager.SelfVersion.ReleaseType,
RemoteControlRegistrationUrl = Library.RemoteControl.RegisterForRemote.DefaultRegisterationUrl,
StartedBy = FIXMEGlobal.Origin,
DefaultUpdateChannel = AutoUpdateSettings.DefaultUpdateChannel.ToString(),
DefaultUsageReportLevel = Library.UsageReporter.Reporter.DefaultReportLevel,
OSType = UpdaterManager.OperatingSystemName,
OSVersion = Library.UsageReporter.OSInfoHelper.PlatformString,
DirectorySeparator = Path.DirectorySeparatorChar,
PathSeparator = Path.PathSeparator,
CaseSensitiveFilesystem = Library.Utility.Utility.IsFSCaseSensitive,
MachineName = Environment.MachineName,
PackageTypeId = UpdaterManager.PackageTypeId,
UserName = OperatingSystem.IsWindows() ? System.Security.Principal.WindowsIdentity.GetCurrent().Name : Environment.UserName,
NewLine = Environment.NewLine,
CLRVersion = Environment.Version.ToString(),
Options = Server.Serializable.ServerSettings.Options,
CompressionModules = Server.Serializable.ServerSettings.CompressionModules,
EncryptionModules = Server.Serializable.ServerSettings.EncryptionModules,
BackendModules = Server.Serializable.ServerSettings.BackendModules,
GenericModules = Server.Serializable.ServerSettings.GenericModules,
WebModules = Server.Serializable.ServerSettings.WebModules,
ConnectionModules = Server.Serializable.ServerSettings.ConnectionModules,
ServerModules = Server.Serializable.ServerSettings.ServerModules,
SecretProviderModules = Server.Serializable.ServerSettings.SecretProviderModules,
UsingAlternateUpdateURLs = AutoUpdateSettings.UsesAlternateURLs,
LogLevels = Enum.GetNames(typeof(Library.Logging.LogMessageType)),
SpecialFolders = SpecialFolders.Nodes.Select(n => new Dto.SystemInfoDto.SpecialFolderDto { ID = n.id, Path = n.resolvedpath }).ToArray(),
SupportedLocales = Library.Localization.LocalizationService.SupportedCultures
.Select(x => new Dto.SystemInfoDto.LocaleDto
{
Code = x,
EnglishName = new CultureInfo(x).EnglishName,
DisplayName = new CultureInfo(x).NativeName
}).ToArray(),
TimeZones = Library.Utility.TimeZoneHelper.GetTimeZones()
TimeZones = Library.Utility.TimeZoneHelper.GetTimeZones()
.Select(x => new Dto.SystemInfoDto.TimeZoneDto
{
ID = x.Id,
DisplayName = x.DisplayName,
CurrentUTCOffset = x.CurrentUtcOffset.ToString()
}),
});
};
/// <summary>
/// The lock guarding the _systemInfoBases cache.
/// </summary>
private readonly object _lock = new();
/// <summary>
/// Cached static system information, the key is the language code.
/// </summary>
private Dictionary<string, StaticSystemInformation> _systemInfoBases = new(StringComparer.OrdinalIgnoreCase);
/// <inheritdoc />
public SystemInfoDto GetSystemInfo(CultureInfo? browserlanguage)
{
browserlanguage ??= CultureInfo.InvariantCulture;
var systeminfo = _systemInfoBase.Value;
var currentLanguage = LocalizationService.CurrentUI?.Culture.Name;
if (string.IsNullOrWhiteSpace(currentLanguage))
currentLanguage = CultureInfo.CurrentUICulture.Name;
StaticSystemInformation? systeminfo;
lock (_lock)
{
if (!_systemInfoBases.TryGetValue(currentLanguage, out systeminfo))
{
systeminfo = CreateStaticSystemInformation();
_systemInfoBases[currentLanguage] = systeminfo;
}
}
// Return the system information, patch in dynamic values
return new SystemInfoDto()
@@ -305,7 +331,7 @@ public class SystemInfoProvider : ISystemInfoProvider
DisplayName = browserlanguage.NativeName
},
SupportedLocales = systeminfo.SupportedLocales,
BrowserLocaleSupported = Library.Localization.LocalizationService.isCultureSupported(browserlanguage),
BrowserLocaleSupported = Library.Localization.LocalizationService.IsCultureSupported(browserlanguage),
TimeZones = systeminfo.TimeZones,
};
}
LiveTests/Duplicati.Backend.Tests/Box/BoxTests.cs
+46
View File
@@ -0,0 +1,46 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
namespace Duplicati.Backend.Tests.Box;
/// <summary>
/// Box.com Tests
/// </summary>
[TestClass]
public sealed class BoxTests : BaseTest
{
[TestMethod]
public Task TestBox()
{
CheckRequiredEnvironment(["TESTCREDENTIAL_BOX_FOLDER", "TESTCREDENTIAL_BOX_AUTHID"]);
var exitCode = CommandLine.BackendTester.Program.Main(
new[]
{
$"box://{Environment.GetEnvironmentVariable("TESTCREDENTIAL_BOX_FOLDER")}/?authid={Uri.EscapeDataString(Environment.GetEnvironmentVariable("TESTCREDENTIAL_BOX_AUTHID")!)}",
}.Concat(Parameters.GlobalTestParameters).ToArray());
if (exitCode != 0) Assert.Fail("BackendTester is returning non-zero exit code, check logs for details");
return Task.CompletedTask;
}
}
LiveTests/Duplicati.Backend.Tests/README.md
+8 -1
View File
@@ -27,6 +27,7 @@ NUMBER_OF_FILES default is 20
| AzureBlob | Planned | Live |
| CIFS | Planned | Live |
| Google Drive | Planned | Live |
| Box | Planned | Live |
| TahoeLAFS | Pending Decision | |
| TescentCOS | Pending Decision | |
| Storj | Pending Decision | |
@@ -39,7 +40,6 @@ NUMBER_OF_FILES default is 20
| Jottacloud | Pending Decision | |
| IDrive2 | Planned | |
| File | Planned | |
| Box | Pending Decision | |
| CloudFiles | Pending Decision | |
| backBlaze | Planned | Live |
| AliyunOSS | Pending Decision | |
@@ -56,6 +56,13 @@ Please note that TestContainers token has to be configured in secrets/Github act
## Backends that require Environment variables
## Box.com
```
TESTCREDENTIAL_BOX_FOLDER
TESTCREDENTIAL_BOX_AUTHID
```
## Backblaze B2
Backblaze B2 credentials are mapped to the following environment variables: