haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/httpclientmigration-idrivee2

Browse Source
This commit is contained in:
Kenneth Skovhede
2025-05-10 10:05:09 +02:00
committed by GitHub
parent b0cdfc4db9 230d1239af
commit 6b4e6cce81
71 changed files with 1903 additions and 2412 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/backendtests.yml
+54 -1
View File
@@ -17,6 +17,7 @@ jobs:
box_secrets_available: "${{ steps.check_box.outputs.available }}"
filen_secrets_available: "${{ steps.check_filen.outputs.available }}"
filejump_secrets_available: "${{ steps.check_filejump.outputs.available }}"
onedrive_secrets_available: "${{ steps.check_onedrive.outputs.available }}"
steps:
- id: check_pcloud
name: Check pCloud secrets
@@ -156,7 +157,7 @@ jobs:
name: Check Filejump secrets
shell: bash
run: |
echo "Starting Filen.io secrets check..."
echo "Starting Filejump secrets check..."
if [[ -n "${{ secrets.TESTCREDENTIAL_FILEJUMP_FOLDER }}" ]] && \
[[ -n "${{ secrets.TESTCREDENTIAL_FILEJUMP_TOKEN }}" ]]; then
echo "All Filejump secrets found"
@@ -166,6 +167,20 @@ jobs:
echo "available=false" >> $GITHUB_OUTPUT
fi
echo "Filejump check completed"
- id: check_onedrive
name: Check OneDrive secrets
shell: bash
run: |
echo "Starting OneDrive secrets check..."
if [[ -n "${{ secrets.TESTCREDENTIAL_ONEDRIVE_FOLDER }}" ]] && \
[[ -n "${{ secrets.TESTCREDENTIAL_ONEDRIVE_AUTHID }}" ]]; then
echo "All OneDrive secrets found"
echo "available=true" >> $GITHUB_OUTPUT
else
echo "Missing some OneDrive secrets"
echo "available=false" >> $GITHUB_OUTPUT
fi
echo "OneDrive check completed"
- name: Debug Output
shell: bash
run: |
@@ -178,6 +193,8 @@ jobs:
echo "B2 Available: ${{ steps.check_b2.outputs.available }}"
echo "Box.com Available: ${{ steps.check_box.outputs.available }}"
echo "Filen.io Available: ${{ steps.check_filen.outputs.available }}"
echo "Filejump Available: ${{ steps.check_filejump.outputs.available }}"
echo "OneDrive Available: ${{ steps.check_onedrive.outputs.available }}"
test_ftp:
needs: check_secrets
if: needs.check_secrets.outputs.testcontainers_secrets_available == 'true'
@@ -662,3 +679,39 @@ jobs:
dotnet test --no-build --filter="ClassName~Filejump.FilejumpTests"
--logger:"console;verbosity=detailed"
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
test_onedrive:
needs: check_secrets
if: needs.check_secrets.outputs.onedrive_secrets_available == 'true'
name: OneDrive Tests
runs-on: "${{ matrix.os }}"
strategy:
max-parallel: 1
fail-fast: false
matrix:
os:
- ubuntu-latest
- windows-latest
- macos-latest
steps:
- name: Set up .NET
uses: actions/setup-dotnet@v4
with:
dotnet-version: 8.x
- name: Checkout source
uses: actions/checkout@v4
- name: Restore NuGet dependencies
run: >-
dotnet restore
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
- name: Build project
run: >-
dotnet build --no-restore
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
- name: Run OneDrive tests
env:
TESTCREDENTIAL_ONEDRIVE_FOLDER: "${{ secrets.TESTCREDENTIAL_ONEDRIVE_FOLDER }}"
TESTCREDENTIAL_ONEDRIVE_AUTHID: "${{ secrets.TESTCREDENTIAL_ONEDRIVE_AUTHID }}"
run: >-
dotnet test --no-build --filter="ClassName~OneDrive.OneDriveTests"
--logger:"console;verbosity=detailed"
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
Duplicati.sln
-7
View File
@@ -21,8 +21,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Duplicati.CommandLine", "Du
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Duplicati.Library.Backend.WEBDAV", "Duplicati\Library\Backend\WEBDAV\Duplicati.Library.Backend.WEBDAV.csproj", "{BAE27510-8B5D-44B2-B33E-372A98908041}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Duplicati.Library.Backend.CloudFiles", "Duplicati\Library\Backend\CloudFiles\Duplicati.Library.Backend.CloudFiles.csproj", "{1BFAE226-8364-4086-825C-BB83F6F3EE4C}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Duplicati.CommandLine.BackendTester", "Duplicati\CommandLine\BackendTester\Duplicati.CommandLine.BackendTester.csproj", "{E7280DCA-7776-4A73-B9B5-41FD77FC8799}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Duplicati.Library.Interface", "Duplicati\Library\Interface\Duplicati.Library.Interface.csproj", "{C5899F45-B0FF-483C-9D38-24A9FCAAB237}"
@@ -234,10 +232,6 @@ Global
{BAE27510-8B5D-44B2-B33E-372A98908041}.Debug|Any CPU.Build.0 = Debug|Any CPU
{BAE27510-8B5D-44B2-B33E-372A98908041}.Release|Any CPU.ActiveCfg = Release|Any CPU
{BAE27510-8B5D-44B2-B33E-372A98908041}.Release|Any CPU.Build.0 = Release|Any CPU
{1BFAE226-8364-4086-825C-BB83F6F3EE4C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{1BFAE226-8364-4086-825C-BB83F6F3EE4C}.Debug|Any CPU.Build.0 = Debug|Any CPU
{1BFAE226-8364-4086-825C-BB83F6F3EE4C}.Release|Any CPU.ActiveCfg = Release|Any CPU
{1BFAE226-8364-4086-825C-BB83F6F3EE4C}.Release|Any CPU.Build.0 = Release|Any CPU
{E7280DCA-7776-4A73-B9B5-41FD77FC8799}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E7280DCA-7776-4A73-B9B5-41FD77FC8799}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E7280DCA-7776-4A73-B9B5-41FD77FC8799}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -553,7 +547,6 @@ Global
{C03F6DFD-805A-4BE0-9338-64870ADDB4A2} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4}
{81765A64-3661-4E3E-B850-2F6F87A51F74} = {D19A38DD-68F1-4EF5-BF5F-8966CE0D9A5B}
{BAE27510-8B5D-44B2-B33E-372A98908041} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4}
{1BFAE226-8364-4086-825C-BB83F6F3EE4C} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4}
{E7280DCA-7776-4A73-B9B5-41FD77FC8799} = {D19A38DD-68F1-4EF5-BF5F-8966CE0D9A5B}
{0CA86ECF-5BEC-4909-B4F6-110A03B30B92} = {566EBBDA-19A4-4056-A615-D901D57D2439}
{C0270709-2A40-43B5-8CF1-69581B9FA2A1} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4}
Duplicati/Agent/Program.cs
+20 -15
View File
@@ -30,9 +30,9 @@ using Duplicati.Library.Interface;
using Duplicati.Library.Logging;
using Duplicati.Library.Main;
using Duplicati.Library.RemoteControl;
using Duplicati.Library.RestAPI;
using Duplicati.Library.Utility;
using Duplicati.Server;
using Duplicati.WebserverCore.Abstractions;
using Duplicati.WebserverCore.Services;
using Microsoft.Extensions.DependencyInjection;
@@ -225,6 +225,9 @@ public static class Program
if (string.IsNullOrWhiteSpace(agentConfig.SettingsEncryptionKey))
agentConfig = agentConfig with { SettingsEncryptionKey = Environment.GetEnvironmentVariable(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME) };
// Prepare the application settings for the server
var applicationSettings = new ApplicationSettings();
// Apply the secret provider, if present
if (!string.IsNullOrWhiteSpace(agentConfig.SecretProvider))
{
@@ -245,7 +248,7 @@ public static class Program
options["secret-provider-cache"] = agentConfig.SecretProviderCache.ToString();
options["secret-provider-pattern"] = agentConfig.SecretProviderPattern;
FIXMEGlobal.SecretProvider = await SecretProviderHelper.ApplySecretProviderAsync([], [], options, TempFolder.SystemTempPath, null, CancellationToken.None);
applicationSettings.SecretProvider = await SecretProviderHelper.ApplySecretProviderAsync([], [], options, TempFolder.SystemTempPath, null, CancellationToken.None);
// Apply the secret provider to the agent configuration
foreach (var prop in agentProps)
@@ -299,7 +302,7 @@ public static class Program
}
var t = await Task.WhenAny(
Task.Run(() => StartLocalServer(agentConfig, settings, cts.Token)),
Task.Run(() => StartLocalServer(agentConfig, applicationSettings, settings, cts.Token)),
KeepRemoteConnection.Start(
settings.ServerUrl,
settings.JWT,
@@ -307,7 +310,7 @@ public static class Program
settings.ServerCertificates,
cts.Token,
OnConnect,
m => ReKey(m, agentConfig),
m => ReKey(applicationSettings, m, agentConfig),
OnControl,
OnMessage
)
@@ -320,16 +323,16 @@ public static class Program
}
private static Task<Dictionary<string, string?>> OnConnect(Dictionary<string, string?> metadata)
=> FIXMEGlobal.Provider.GetRequiredService<IRemoteControllerHandler>().OnConnect(metadata);
=> Server.Program.DuplicatiWebserver.Provider.GetRequiredService<IRemoteControllerHandler>().OnConnect(metadata);
private static Task OnControl(KeepRemoteConnection.ControlMessage message)
=> FIXMEGlobal.Provider.GetRequiredService<IRemoteControllerHandler>().OnControl(message);
=> Server.Program.DuplicatiWebserver.Provider.GetRequiredService<IRemoteControllerHandler>().OnControl(message);
private static Task OnMessage(KeepRemoteConnection.CommandMessage message)
=> FIXMEGlobal.Provider.GetRequiredService<IRemoteControllerHandler>().OnMessage(message);
=> Server.Program.DuplicatiWebserver.Provider.GetRequiredService<IRemoteControllerHandler>().OnMessage(message);
private static Task ReKey(ClaimedClientData keydata, CommandLineArguments agentConfig)
private static Task ReKey(IApplicationSettings applicationSettings, ClaimedClientData keydata, CommandLineArguments agentConfig)
{
// ReKey is handled here because we store the config outside of the database
Log.WriteMessage(LogMessageType.Verbose, LogTag, "ReKey", "Rekeying the settings");
@@ -340,7 +343,7 @@ public static class Program
settings = settings with { ServerCertificates = keydata.ServerCertificates };
// Only allow re-keying if the settings encryption key is not set
if (!FIXMEGlobal.SettingsEncryptionKeyProvidedExternally)
if (!applicationSettings.SettingsEncryptionKeyProvidedExternally)
{
if (!string.IsNullOrWhiteSpace(keydata.LocalEncryptionKey) && settings.SettingsEncryptionKey != keydata.LocalEncryptionKey)
{
@@ -358,14 +361,15 @@ public static class Program
/// <summary>
/// Runs the server, restarting on crashes
/// </summary>
/// <param name="applicationSettings">The application settings for the server</param>
/// <param name="args">The commandline arguments passed to the server</param>
/// <param name="cancellationToken">The cancellation token to use for the process</param>
/// <returns>An awaitable task</returns>
private static async Task RunServer(string[] args, CancellationToken cancellationToken)
private static async Task RunServer(IApplicationSettings applicationSettings, string[] args, CancellationToken cancellationToken)
{
cancellationToken.Register(() =>
{
Server.Program.ApplicationExitEvent.Set();
applicationSettings.ApplicationExitEvent.Set();
});
var lastRestart = DateTime.Now;
@@ -374,7 +378,7 @@ public static class Program
try
{
lastRestart = DateTime.Now;
Server.Program.Main(args);
Server.Program.Main(applicationSettings, args);
}
catch (Exception ex)
{
@@ -445,10 +449,11 @@ public static class Program
/// Starts the local webserver in locked down mode, restarting on crashes
/// </summary>
/// <param name="agentConfig">The agent configuration</param>
/// <param name="applicationSettings">The application settings for the server</param>
/// <param name="settings">The settings for the agent</param>
/// <param name="cancellationToken">The cancellation token that stops the server</param>
/// <returns>An awaitable task</returns>
private static async Task StartLocalServer(CommandLineArguments agentConfig, Settings settings, CancellationToken cancellationToken)
private static async Task StartLocalServer(CommandLineArguments agentConfig, IApplicationSettings applicationSettings, Settings settings, CancellationToken cancellationToken)
{
// TODO: Look into pipes for Kestrel to prevent network access
@@ -487,10 +492,10 @@ public static class Program
.ToArray();
// Set the global origin
FIXMEGlobal.Origin = "Agent";
applicationSettings.Origin = "Agent";
// Start the server
await RunServer(args, cancellationToken);
await RunServer(applicationSettings, args, cancellationToken);
}
}
Duplicati/CommandLine/ServerUtil/Connection.cs
+2 -2
View File
@@ -24,6 +24,7 @@ using System.Net.Security;
using System.Text.Json;
using Duplicati.Library.AutoUpdater;
using Duplicati.WebserverCore.Middlewares;
using Duplicati.WebserverCore.Services;
namespace Duplicati.CommandLine.ServerUtil;
@@ -141,7 +142,6 @@ public class Connection
/// <returns>The connection</returns>
public static async Task<Connection> Connect(Settings settings, bool obtainRefreshToken = false, OutputInterceptor? console = null)
{
if (console != null)
console.AppendConsoleMessage($"Connecting to {settings.HostUrl}...");
else
@@ -201,7 +201,7 @@ public class Connection
if (File.Exists(Path.Combine(DataFolderManager.GetDataFolder(DataFolderManager.AccessMode.ProbeOnly), DataFolderManager.SERVER_DATABASE_FILENAME)))
{
string? cfg = null;
using (var connection = Server.Program.GetDatabaseConnection(opts, true))
using (var connection = Server.Program.GetDatabaseConnection(new ApplicationSettings(), opts, true))
{
cfg = connection.ApplicationSettings.JWTConfig;
if (settings.HostUrl.Scheme == "https" && connection.ApplicationSettings.ServerSSLCertificate != null && trustedCertificateHashes.Count == 0)
Duplicati/GUI/Duplicati.GUI.TrayIcon/HostedInstanceKeeper.cs
+7 -4
View File
@@ -21,6 +21,7 @@
using System;
using System.Linq;
using Duplicati.WebserverCore.Abstractions;
namespace Duplicati.GUI.TrayIcon
{
@@ -32,9 +33,11 @@ namespace Duplicati.GUI.TrayIcon
private readonly System.Threading.Thread m_runner;
private Exception m_runnerException = null;
public Action InstanceShutdown;
private readonly IApplicationSettings applicationSettings;
public HostedInstanceKeeper(string[] args)
public HostedInstanceKeeper(IApplicationSettings applicationSettings, string[] args)
{
this.applicationSettings = applicationSettings;
m_runner = new System.Threading.Thread(_ =>
{
try
@@ -44,13 +47,13 @@ namespace Duplicati.GUI.TrayIcon
if (args == null || !args.Any(x => x.Trim().StartsWith("--" + Server.WebServerLoader.OPTION_PORT + "=", StringComparison.OrdinalIgnoreCase)))
args = (args ?? new string[0]).Union(new string[] { "--" + Server.WebServerLoader.OPTION_PORT + "=8200,8300,8400,8500,8600,8700,8800,8900,8989" }).ToArray();
Server.Program.Main(args);
Server.Program.Main(applicationSettings, args);
}
catch (Exception ex)
{
m_runnerException = ex;
Server.Program.ServerStartedEvent?.Set();
Server.Program.ApplicationExitEvent?.Set();
applicationSettings.ApplicationExitEvent?.Set();
}
finally
{
@@ -85,7 +88,7 @@ namespace Duplicati.GUI.TrayIcon
{
try
{
Server.Program.ApplicationExitEvent.Set();
applicationSettings.ApplicationExitEvent.Set();
if (!m_runner.Join(TimeSpan.FromSeconds(10)))
{
m_runner.Interrupt();
Duplicati/GUI/Duplicati.GUI.TrayIcon/HttpServerConnection.cs
+7 -5
View File
@@ -334,9 +334,10 @@ namespace Duplicati.GUI.TrayIcon
private async Task ObtainAccessTokenAsync()
{
// If we host the server, issue the access token from the service
if (FIXMEGlobal.IsServerStarted && m_passwordSource == Program.PasswordSource.HostedServer)
var sp = Server.Program.DuplicatiWebserver?.Provider;
if (sp != null && m_passwordSource == Program.PasswordSource.HostedServer)
{
var provider = FIXMEGlobal.Provider.GetRequiredService<IJWTTokenProvider>();
var provider = sp.GetRequiredService<IJWTTokenProvider>();
m_accesstoken = provider.CreateAccessToken("trayicon", provider.TemporaryFamilyId);
return;
}
@@ -412,12 +413,13 @@ namespace Duplicati.GUI.TrayIcon
string signinjwt = null;
// If we host the server, issue the token from the service
if (FIXMEGlobal.IsServerStarted && m_passwordSource == Program.PasswordSource.HostedServer)
var sp = Server.Program.DuplicatiWebserver?.Provider;
if (sp != null && m_passwordSource == Program.PasswordSource.HostedServer)
{
if (FIXMEGlobal.DataConnection.ApplicationSettings.DisableSigninTokens)
if (sp.GetRequiredService<Server.Database.Connection>().ApplicationSettings.DisableSigninTokens)
return null;
signinjwt = FIXMEGlobal.Provider.GetRequiredService<IJWTTokenProvider>().CreateSigninToken("trayicon");
signinjwt = sp.GetRequiredService<IJWTTokenProvider>().CreateSigninToken("trayicon");
}
// If we have database access, grab the issuer key from the db and issue a token
Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs
+16 -11
View File
@@ -30,9 +30,11 @@ using CoCoL;
using Duplicati.CommandLine;
using Duplicati.Library.AutoUpdater;
using Duplicati.Library.Interface;
using Duplicati.Library.RestAPI;
using Duplicati.Library.Utility;
using Duplicati.Server;
using Duplicati.Server.Database;
using Duplicati.WebserverCore.Services;
using Microsoft.Extensions.DependencyInjection;
using Uri = System.Uri;
namespace Duplicati.GUI.TrayIcon
@@ -144,12 +146,13 @@ namespace Duplicati.GUI.TrayIcon
try
{
// Tell the hosted server it was started by the TrayIcon
FIXMEGlobal.Origin = "Tray icon";
var applicationSettings = new ApplicationSettings();
applicationSettings.Origin = "Tray icon";
passwordSource = PasswordSource.HostedServer;
// Ignore TrayIcon specific settings
foreach (var c in BasicSupportedCommands.Select(x => x.Name))
Server.Program.ValidationIgnoredOptions.Add(c);
hosted = new HostedInstanceKeeper(_args);
hosted = new HostedInstanceKeeper(applicationSettings, _args);
}
catch (Exception ex)
{
@@ -164,17 +167,18 @@ namespace Duplicati.GUI.TrayIcon
// We have a hosted server, if this is the first run,
// we should open the main page
openui = Server.Program.IsFirstRun || Server.Program.ServerPortChanged;
var connection = Server.Program.DuplicatiWebserver.Provider.GetRequiredService<Connection>();
openui = connection.ApplicationSettings.IsFirstRun || connection.ApplicationSettings.ServerPortChanged;
var scheme = Server.Program.DataConnection.ApplicationSettings.UseHTTPS ? "https" : "http";
var scheme = connection.ApplicationSettings.UseHTTPS ? "https" : "http";
serverURL = new UriBuilder(serverURL)
{
Port = Server.Program.ServerPort,
Port = Server.Program.DuplicatiWebserver.Port,
Scheme = scheme
}.Uri;
if (Server.Program.DataConnection.ApplicationSettings.UseHTTPS && string.IsNullOrWhiteSpace(acceptedHostCertificate))
acceptedHostCertificate = Server.Program.DataConnection.ApplicationSettings.ServerSSLCertificate?.FirstOrDefault(x => x.HasPrivateKey)?.GetCertHashString();
if (connection.ApplicationSettings.UseHTTPS && string.IsNullOrWhiteSpace(acceptedHostCertificate))
acceptedHostCertificate = connection.ApplicationSettings.ServerSSLCertificate?.FirstOrDefault(x => x.HasPrivateKey)?.GetCertHashString();
}
else if (Utility.ParseBoolOption(options, READCONFIGFROMDB_OPTION))
@@ -182,7 +186,7 @@ namespace Duplicati.GUI.TrayIcon
if (File.Exists(Path.Combine(DataFolderManager.GetDataFolder(DataFolderManager.AccessMode.ReadWritePermissionSet), DataFolderManager.SERVER_DATABASE_FILENAME)))
{
passwordSource = PasswordSource.Database;
databaseConnection = Server.Program.GetDatabaseConnection(options, true);
databaseConnection = Server.Program.GetDatabaseConnection(new ApplicationSettings(), options, true);
if (databaseConnection != null)
{
@@ -267,8 +271,9 @@ No password provided, unable to connect to server, exiting");
}
tk.ShowUrlInWindow(t.Result);
Server.Program.IsFirstRun = false;
Server.Program.ServerPortChanged = false;
var connection = Server.Program.DuplicatiWebserver.Provider.GetRequiredService<Connection>();
connection.ApplicationSettings.IsFirstRun = false;
connection.ApplicationSettings.ServerPortChanged = false;
});
}
Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs
+6 -1
View File
@@ -198,9 +198,14 @@ namespace Duplicati.Library.Backend.AliyunOSS
var client = GetClient();
try
{
var metadata = new ObjectMetadata
{
ContentLength = stream.Length
};
using var timeoutStream = stream.ObserveReadTimeout(_timeouts.ReadWriteTimeout, false);
var objectResult = await Task.Factory.FromAsync(
(cb, state) => client.BeginPutObject(bucketName, objectName, timeoutStream, cb, state),
(cb, state) => client.BeginPutObject(bucketName, objectName, timeoutStream, metadata, cb, state),
client.EndPutObject,
null).ConfigureAwait(false);
Duplicati/Library/Backend/Backblaze/B2.cs
+2 -41
View File
@@ -310,43 +310,8 @@ public class B2 : IStreamingBackend
public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
TempFile? tmp = null;
var measure = stream;
while (measure is OverrideableStream os &&
os.GetType().GetField("m_basestream", BindingFlags.NonPublic | BindingFlags.Instance)?.GetValue(os) is Stream baseStream)
{
measure = baseStream;
}
if (measure == null)
throw new Exception($"Unable to unwrap stream from: {stream.GetType()}");
string sha1;
if (measure.CanSeek)
{
var p = measure.Position;
// Compute the hash
using (var hashalg = HashFactory.CreateHasher("SHA1"))
sha1 = Utility.Utility.ByteArrayAsHexString(hashalg.ComputeHash(measure));
measure.Position = p;
}
else
{
// No seeking possible, use a temp file
tmp = new TempFile();
await using (var sr = File.OpenWrite(tmp))
using (var hasher = HashFactory.CreateHasher("SHA1"))
await using (var hc = new HashCalculatingStream(measure, hasher))
{
await Utility.Utility.CopyStreamAsync(hc, sr, cancelToken).ConfigureAwait(false);
sha1 = hc.GetFinalHashString();
}
stream = File.OpenRead(tmp);
}
(stream, var sha1, var tmp) = await Utility.Utility.CalculateThrottledStreamHash(stream, "SHA1", cancelToken).ConfigureAwait(false);
using var _ = tmp;
if (_filecache == null)
await RebuildFileCache(cancelToken).ConfigureAwait(false);
@@ -400,10 +365,6 @@ public class B2 : IStreamingBackend
throw;
}
finally
{
tmp?.Dispose();
}
}
/// <summary>
Duplicati/Library/Backend/CloudFiles/CloudFiles.cs
-450
View File
@@ -1,450 +0,0 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using Duplicati.Library.Utility.Options;
using System.Net;
using System.Runtime.CompilerServices;
using System.Security.Cryptography;
namespace Duplicati.Library.Backend
{
// ReSharper disable once UnusedMember.Global
// This class is instantiated dynamically in the BackendLoader.
public class CloudFiles : IBackend, IStreamingBackend
{
/// <summary>
/// The log tag for this class
/// </summary>
private static readonly string LOGTAG = Logging.Log.LogTagFromType<CloudFiles>();
public const string AUTH_URL_US = "https://identity.api.rackspacecloud.com/auth";
public const string AUTH_URL_UK = "https://lon.auth.api.rackspacecloud.com/v1.0";
private const string DUMMY_HOSTNAME = "api.mosso.com";
private const string AUTH_USERNAME_OPTION = "cloudfiles-username";
private const string AUTH_PASSWORD_OPTION = "cloudfiles-accesskey";
private const int ITEM_LIST_LIMIT = 1000;
private readonly string m_username;
private readonly string m_password;
private readonly string m_path;
private string? m_storageUrl = null;
private string? m_authToken = null;
private readonly string m_authUrl;
private readonly TimeoutOptionsHelper.Timeouts m_timeouts;
// ReSharper disable once UnusedMember.Global
// This constructor is needed by the BackendLoader.
public CloudFiles()
{
m_username = null!;
m_password = null!;
m_path = null!;
m_authUrl = null!;
m_timeouts = null!;
}
// ReSharper disable once UnusedMember.Global
// This constructor is needed by the BackendLoader.
public CloudFiles(string url, Dictionary<string, string?> options)
{
var uri = new Utility.Uri(url);
var auth = AuthOptionsHelper.ParseWithAlias(options, uri, AUTH_USERNAME_OPTION, AUTH_PASSWORD_OPTION);
if (!auth.HasUsername)
throw new UserInformationException(Strings.CloudFiles.NoUserIDError, "CloudFilesNoUserID");
if (auth.HasPassword)
throw new UserInformationException(Strings.CloudFiles.NoAPIKeyError, "CloudFilesNoApiKey");
(m_username, m_password) = auth.GetCredentials();
//Fallback to the previous format
if (url.Contains(DUMMY_HOSTNAME))
{
Logging.Log.WriteWarningMessage(LOGTAG, "CloudFilesDeprecatedFormat", null, Strings.CloudFiles.DeprecatedFormatWarning(DUMMY_HOSTNAME));
var u = new System.Uri(url);
if (!string.IsNullOrEmpty(u.UserInfo))
{
if (u.UserInfo.IndexOf(":", StringComparison.Ordinal) >= 0)
{
m_username = u.UserInfo.Substring(0, u.UserInfo.IndexOf(":", StringComparison.Ordinal));
m_password = u.UserInfo.Substring(u.UserInfo.IndexOf(":", StringComparison.Ordinal) + 1);
}
else
{
m_username = u.UserInfo;
}
}
//We use the api.mosso.com hostname.
//This allows the use of containers that have names that are not valid hostnames,
// such as container names with spaces in them
if (u.Host.Equals(DUMMY_HOSTNAME))
m_path = Library.Utility.Uri.UrlDecode(u.PathAndQuery);
else
m_path = u.Host + Library.Utility.Uri.UrlDecode(u.PathAndQuery);
}
else
{
m_path = uri.HostAndPath;
}
if (m_path.EndsWith("/", StringComparison.Ordinal))
m_path = m_path.Substring(0, m_path.Length - 1);
if (!m_path.StartsWith("/", StringComparison.Ordinal))
m_path = "/" + m_path;
var authUrl = options.GetValueOrDefault("cloudfiles-authentication-url");
if (string.IsNullOrEmpty(authUrl))
authUrl = Utility.Utility.ParseBoolOption(options, "cloudfiles-uk-account") ? AUTH_URL_UK : AUTH_URL_US;
m_authUrl = authUrl;
m_timeouts = TimeoutOptionsHelper.Parse(options);
}
#region IBackend Members
public string DisplayName
{
get { return Strings.CloudFiles.DisplayName; }
}
public string ProtocolKey
{
get { return "cloudfiles"; }
}
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
var extraUrl = "?format=xml&limit=" + ITEM_LIST_LIMIT.ToString();
var markerUrl = "";
bool repeat;
do
{
var doc = new System.Xml.XmlDocument();
var req = await CreateRequest("", extraUrl + markerUrl, cancelToken).ConfigureAwait(false);
try
{
await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ =>
{
var areq = new Utility.AsyncHttpRequest(req);
using (var resp = (HttpWebResponse)areq.GetResponse())
using (var s = areq.GetResponseStream())
doc.Load(s);
}).ConfigureAwait(false);
}
catch (WebException wex)
{
if (markerUrl == "") //Only check on first iteration
if (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound)
throw new FolderMissingException(wex);
//Other error, just re-throw
throw;
}
var lst = doc.SelectNodes("container/object");
//Perhaps the folder does not exist?
//The response should be 404 from the server, but it is not :(
if (lst == null || lst.Count == 0 && markerUrl == "") //Only on first iteration
{
try { await CreateFolderAsync(cancelToken).ConfigureAwait(false); }
catch { } //Ignore
}
if (lst == null)
yield break;
var lastItemName = "";
foreach (System.Xml.XmlNode n in lst)
{
var name = n["name"]?.InnerText;
if (!long.TryParse(n["bytes"]?.InnerText, out var size))
size = -1;
if (!DateTime.TryParse(n["last_modified"]?.InnerText, out var mod))
mod = new DateTime();
lastItemName = name;
yield return new FileEntry(name, size, mod, mod);
}
repeat = lst.Count == ITEM_LIST_LIMIT;
if (repeat)
markerUrl = "&marker=" + Library.Utility.Uri.UrlEncode(lastItemName);
} while (repeat);
}
public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken)
{
using (var fs = File.OpenRead(filename))
await PutAsync(remotename, fs, cancelToken);
}
public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken)
{
using (var fs = File.Create(filename))
await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false);
}
public async Task DeleteAsync(string remotename, CancellationToken cancelToken)
{
try
{
var req = await CreateRequest("/" + remotename, "", cancelToken).ConfigureAwait(false);
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ =>
{
req.Method = "DELETE";
var areq = new AsyncHttpRequest(req);
using (var resp = (HttpWebResponse)areq.GetResponse())
{
if (resp.StatusCode == HttpStatusCode.NotFound)
throw new FileMissingException();
if ((int)resp.StatusCode >= 300)
throw new WebException(Strings.CloudFiles.FileDeleteError, null, WebExceptionStatus.ProtocolError, resp);
else
using (areq.GetResponseStream())
{ }
}
}).ConfigureAwait(false);
}
catch (WebException wex)
{
if (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound)
throw new FileMissingException(wex);
else
throw;
}
}
public IList<ICommandLineArgument> SupportedCommands =>
[
new CommandLineArgument(AUTH_USERNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.CloudFiles.DescriptionUsernameShort, Strings.CloudFiles.DescriptionUsernameLong, null, [AuthOptionsHelper.AuthUsernameOption] ),
new CommandLineArgument(AUTH_PASSWORD_OPTION, CommandLineArgument.ArgumentType.Password, Strings.CloudFiles.DescriptionPasswordShort, Strings.CloudFiles.DescriptionPasswordLong, null, [AuthOptionsHelper.AuthPasswordOption]),
new CommandLineArgument("cloudfiles-uk-account", CommandLineArgument.ArgumentType.Boolean, Strings.CloudFiles.DescriptionUKAccountShort, Strings.CloudFiles.DescriptionUKAccountLong("cloudfiles-authentication-url", AUTH_URL_UK)),
new CommandLineArgument("cloudfiles-authentication-url", CommandLineArgument.ArgumentType.String, Strings.CloudFiles.DescriptionAuthenticationURLShort, Strings.CloudFiles.DescriptionAuthenticationURLLong_v2("cloudfiles-uk-account"), AUTH_URL_US),
.. TimeoutOptionsHelper.GetOptions(),
];
public string Description
{
get { return Strings.CloudFiles.Description_v2; }
}
#endregion
#region IBackend_v2 Members
public Task TestAsync(CancellationToken cancelToken) =>
//The "Folder not found" is not detectable :(
this.TestListAsync(cancelToken);
public async Task CreateFolderAsync(CancellationToken cancelToken)
{
var createReq = await CreateRequest("", "", cancelToken).ConfigureAwait(false);
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ =>
{
createReq.Method = "PUT";
var areq = new AsyncHttpRequest(createReq);
using (var resp = (HttpWebResponse)areq.GetResponse())
{ }
}).ConfigureAwait(false);
}
#endregion
#region IDisposable Members
public void Dispose()
{
}
#endregion
#region IStreamingBackend Members
public Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(
new string?[] {
new System.Uri(m_authUrl).Host,
string.IsNullOrWhiteSpace(m_storageUrl) ? null : new System.Uri(m_storageUrl).Host
}
.WhereNotNullOrWhiteSpace()
.ToArray()
);
public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
{
var req = await CreateRequest("/" + remotename, "", cancelToken).ConfigureAwait(false);
req.Method = "GET";
var areq = new AsyncHttpRequest(req);
using (var resp = areq.GetResponse())
using (var s = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponseStream()).ConfigureAwait(false))
using (var timeoutStream = s.ObserveReadTimeout(m_timeouts.ReadWriteTimeout))
using (var hasher = MD5.Create())
using (var mds = new HashCalculatingStream(timeoutStream, hasher))
{
var md5Hash = resp.Headers["ETag"];
await Utility.Utility.CopyStreamAsync(mds, stream, true, cancelToken).ConfigureAwait(false);
if (!string.Equals(mds.GetFinalHashString(), md5Hash, StringComparison.OrdinalIgnoreCase))
throw new Exception(Strings.CloudFiles.ETagVerificationError);
}
}
public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
{
var req = await CreateRequest("/" + remotename, "", cancelToken).ConfigureAwait(false);
req.Method = "PUT";
req.ContentType = "application/octet-stream";
try { req.ContentLength = stream.Length; }
catch { }
// TODO: When reviewing this, lets build a common method to unwrap the stream passed from
// BackendManager.PutOperation, so we can use the same logic to compute hashes in all backends
//If we can pre-calculate the MD5 hash before transmission, do so
/*if (stream.CanSeek)
{
System.Security.Cryptography.MD5 md5 = System.Security.Cryptography.MD5.Create();
req.Headers["ETag"] = Core.Utility.ByteArrayAsHexString(md5.ComputeHash(stream)).ToLower(System.Globalization.CultureInfo.InvariantCulture);
stream.Seek(0, System.IO.SeekOrigin.Begin);
using (System.IO.Stream s = req.GetRequestStream())
Core.Utility.CopyStream(stream, s);
//Reset the timeout to the default value of 100 seconds to
// avoid blocking the GetResponse() call
req.Timeout = 100000;
//The server handles the eTag verification for us, and gives an error if the hash was a mismatch
using (HttpWebResponse resp = (HttpWebResponse)req.GetResponse())
if ((int)resp.StatusCode >= 300)
throw new WebException(Strings.CloudFiles.FileUploadError, null, WebExceptionStatus.ProtocolError, resp);
}
else //Otherwise use a client-side calculation
*/
//TODO: We cannot use the local MD5 calculation, because that could involve a throttled read,
// and may invoke various events
{
string? fileHash = null;
long streamLen = -1;
try { streamLen = stream.Length; }
catch { }
var areq = new AsyncHttpRequest(req);
using (var s = areq.GetRequestStream(streamLen))
using (var timeoutStream = s.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout))
using (var hasher = MD5.Create())
using (var mds = new HashCalculatingStream(timeoutStream, hasher))
{
await Utility.Utility.CopyStreamAsync(stream, mds, tryRewindSource: true, cancelToken: cancelToken);
fileHash = mds.GetFinalHashString();
}
string? md5Hash = null;
//We need to verify the eTag locally
try
{
using (HttpWebResponse resp = (HttpWebResponse)areq.GetResponse())
if ((int)resp.StatusCode >= 300)
throw new WebException(Strings.CloudFiles.FileUploadError, null, WebExceptionStatus.ProtocolError, resp);
else
md5Hash = resp.Headers["ETag"];
}
catch (WebException wex)
{
//Catch 404 and turn it into a FolderNotFound error
if (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound)
throw new FolderMissingException(wex);
//Other error, just re-throw
throw;
}
if (md5Hash == null || !string.Equals(md5Hash, fileHash, StringComparison.OrdinalIgnoreCase))
{
//Remove the broken file
try { await DeleteAsync(remotename, cancelToken); }
catch { }
throw new Exception(Strings.CloudFiles.ETagVerificationError);
}
}
}
#endregion
private async Task<HttpWebRequest> CreateRequest(string remotename, string query, CancellationToken cancelToken)
{
//If this is the first call, get an authentication token
if (string.IsNullOrEmpty(m_authToken) || string.IsNullOrEmpty(m_storageUrl))
{
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ =>
{
var authReq = (HttpWebRequest)HttpWebRequest.Create(m_authUrl);
authReq.Headers.Add("X-Auth-User", m_username);
authReq.Headers.Add("X-Auth-Key", m_password);
authReq.Method = "GET";
var areq = new AsyncHttpRequest(authReq);
using (var resp = areq.GetResponse())
{
m_storageUrl = resp.Headers["X-Storage-Url"];
m_authToken = resp.Headers["X-Auth-Token"];
}
if (string.IsNullOrEmpty(m_authToken) || string.IsNullOrEmpty(m_storageUrl))
throw new Exception(Strings.CloudFiles.UnexpectedResponseError);
}).ConfigureAwait(false);
}
var req = (HttpWebRequest)HttpWebRequest.Create(m_storageUrl + UrlEncode(m_path + remotename) + query);
req.Headers.Add("X-Auth-Token", UrlEncode(m_authToken!));
req.UserAgent = "Duplicati CloudFiles Backend v" + System.Reflection.Assembly.GetExecutingAssembly().GetName().Version;
req.KeepAlive = false;
req.PreAuthenticate = true;
req.AllowWriteStreamBuffering = false;
return req;
}
private static string UrlEncode(string value)
{
return Utility.Uri.UrlEncode(value).Replace("+", "%20").Replace("%2f", "/");
}
}
}
Duplicati/Library/Backend/CloudFiles/Duplicati.Library.Backend.CloudFiles.csproj
-25
View File
@@ -1,25 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<OutputType>Library</OutputType>
<RootNamespace>Duplicati.Library.Backend</RootNamespace>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\Interface\Duplicati.Library.Interface.csproj" />
<ProjectReference Include="..\..\Localization\Duplicati.Library.Localization.csproj" />
<ProjectReference Include="..\..\Utility\Duplicati.Library.Utility.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.DotNet.Analyzers.Compatibility" Version="0.2.12-alpha">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup>
</Project>
Duplicati/Library/Backend/CloudFiles/Strings.cs
-44
View File
@@ -1,44 +0,0 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.Strings
{
internal static class CloudFiles
{
public static string Description_v2 { get { return LC.L(@"This backend can read and write data to CloudFiles. Allowed format is ""cloudfiles://container/folder""."); } }
public static string DisplayName { get { return LC.L(@"Rackspace CloudFiles"); } }
public static string DescriptionAuthenticationURLLong_v2(string optionname) { return LC.L(@"CloudFiles use different servers for authentication based on where the account resides. Use this option to set an alternate authentication URL. This option overrides --{0}.", optionname); }
public static string DescriptionAuthenticationURLShort { get { return LC.L(@"Provide another authentication URL"); } }
public static string DescriptionPasswordLong { get { return LC.L(@"The API Access Key used to authenticate with CloudFiles."); } }
public static string DescriptionPasswordShort { get { return LC.L(@"Supply the access key used to connect to the server"); } }
public static string DescriptionUKAccountLong(string optionname, string optionvalue) { return LC.L(@"Duplicati will assume that the credentials given are for a US account. Use this option if the account is a UK based account. Note that this is equivalent to setting --{0}={1}.", optionname, optionvalue); }
public static string DescriptionUKAccountShort { get { return LC.L(@"Use a UK account"); } }
public static string DescriptionUsernameLong { get { return LC.L(@"The username used to authenticate with CloudFiles."); } }
public static string DescriptionUsernameShort { get { return LC.L(@"Supply the username used to authenticate with CloudFiles"); } }
public static string ETagVerificationError { get { return LC.L(@"MD5 Hash (ETag) verification failed"); } }
public static string FileDeleteError { get { return LC.L(@"Failed to delete file"); } }
public static string FileUploadError { get { return LC.L(@"Failed to upload file"); } }
public static string NoAPIKeyError { get { return LC.L(@"No CloudFiles API Access Key given"); } }
public static string NoUserIDError { get { return LC.L(@"No CloudFiles userID given"); } }
public static string UnexpectedResponseError { get { return LC.L(@"Unexpected CloudFiles response. Perhaps the API has changed?"); } }
public static string DeprecatedFormatWarning(string hostname) { return LC.L(@$"The URL is using the marker hostname {hostname} which is no longer supported. Please use the new format ""cloudfiles://container/folder"" instead."); }
}
}
Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs
+1 -1
View File
@@ -216,7 +216,7 @@ namespace Duplicati.Library.Backend
var (accessKeyId, accessKeySecret) = _auth.GetCredentials();
var host = await GetRegionEndpointAsync("https://api.idrivee2.com/api/service/get_region_end_point/" + accessKeyId, cancellationToken).ConfigureAwait(false);
_s3Client = new S3AwsClient(accessKeyId, accessKeySecret, null, host, null, true, false, _timeouts, _options);
_s3Client = new S3AwsClient(accessKeyId, accessKeySecret, null, host, null, true, false, false, _timeouts, _options);
return _s3Client;
}
Duplicati/Library/Backend/Jottacloud/Jottacloud.cs
+59 -96
View File
@@ -489,108 +489,71 @@ namespace Duplicati.Library.Backend
// we specify the file size in a custom header. And if the stream is not seek-able we are not able
// to use stream.Length, so we are back at square one.
// TODO: This unwrapping code is shared with CloudFiles
TempFile? tmpFile = null;
Stream? baseStream = stream;
while (baseStream is OverrideableStream)
baseStream = typeof(OverrideableStream).GetField("m_basestream", System.Reflection.BindingFlags.DeclaredOnly | System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic)?.GetValue(baseStream) as System.IO.Stream;
if (baseStream == null)
throw new Exception(string.Format("Unable to unwrap stream from: {0}", stream.GetType()));
string md5Hash;
if (baseStream.CanSeek)
{
var originalPosition = baseStream.Position;
using (var md5 = MD5.Create())
md5Hash = Utility.Utility.ByteArrayAsHexString(md5.ComputeHash(baseStream));
baseStream.Position = originalPosition;
}
else
{
// No seeking possible, use a temp file
tmpFile = new TempFile();
using (var os = File.OpenWrite(tmpFile))
using (var hasher = MD5.Create())
using (var md5 = new HashCalculatingStream(baseStream, hasher))
{
await Utility.Utility.CopyStreamAsync(md5, os, true, cancelToken).ConfigureAwait(false);
md5Hash = md5.GetFinalHashString();
}
stream = File.OpenRead(tmpFile);
}
try
{
// Create request, with query parameter, and a few custom headers.
// NB: If we wanted to we could send the same POST request as below but without the file contents
// and with "cphash=[md5Hash]" as the only query parameter. Then we will get an HTTP 200 (OK) response
// if an identical file already exists, and we can skip uploading the new file. We will get
// HTTP 404 (Not Found) if file does not exists or it exists with a different hash, in which
// case we must send a new request to upload the new content.
var fileSize = stream.Length;
var req = CreateRequest(WebRequestMethods.Http.Post, remotename, "umode=nomultipart", true);
req.Headers.Add("JMd5", md5Hash); // Not required, but it will make the server verify the content and mark the file as corrupt if there is a mismatch.
req.Headers.Add("JSize", fileSize.ToString()); // Required, and used to mark file as incomplete if we upload something be the total size of the original file!
// File time stamp headers: Since we are working with a stream here we do not know the local file's timestamps,
// and then we can just omit the JCreated and JModified and let the server automatically set the current time.
//req.Headers.Add("JCreated", timeCreated);
//req.Headers.Add("JModified", timeModified);
req.ContentType = "application/octet-stream";
req.ContentLength = fileSize;
(stream, var md5Hash, var tmp) = await Utility.Utility.CalculateThrottledStreamHash(stream, "MD5", cancelToken).ConfigureAwait(false);
using var _ = tmp;
// Write post data request
var areq = new AsyncHttpRequest(req);
using (var rs = areq.GetRequestStream())
using (var ts = rs.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout))
await Utility.Utility.CopyStreamAsync(stream, rs, true, cancelToken).ConfigureAwait(false);
// Send request, and check response
using (var resp = (HttpWebResponse)areq.GetResponse())
{
if (resp.StatusCode != HttpStatusCode.Created)
throw new WebException(Strings.Jottacloud.FileUploadError, null, WebExceptionStatus.ProtocolError, resp);
// Create request, with query parameter, and a few custom headers.
// NB: If we wanted to we could send the same POST request as below but without the file contents
// and with "cphash=[md5Hash]" as the only query parameter. Then we will get an HTTP 200 (OK) response
// if an identical file already exists, and we can skip uploading the new file. We will get
// HTTP 404 (Not Found) if file does not exists or it exists with a different hash, in which
// case we must send a new request to upload the new content.
var fileSize = stream.Length;
var req = CreateRequest(WebRequestMethods.Http.Post, remotename, "umode=nomultipart", true);
req.Headers.Add("JMd5", md5Hash); // Not required, but it will make the server verify the content and mark the file as corrupt if there is a mismatch.
req.Headers.Add("JSize", fileSize.ToString()); // Required, and used to mark file as incomplete if we upload something be the total size of the original file!
// File time stamp headers: Since we are working with a stream here we do not know the local file's timestamps,
// and then we can just omit the JCreated and JModified and let the server automatically set the current time.
//req.Headers.Add("JCreated", timeCreated);
//req.Headers.Add("JModified", timeModified);
req.ContentType = "application/octet-stream";
req.ContentLength = fileSize;
// Request seems to be successful, but we must verify the response XML content to be sure that the file
// was correctly uploaded: The server will verify the JSize header and mark the file as incomplete if
// there was mismatch, and it will verify the JMd5 header and mark the file as corrupt if there was a hash
// mismatch. The returned XML contains a file element, and if upload was error free it contains a single
// child element "currentRevision", which has a "state" child element with the string "COMPLETED".
// If there was a problem we should have a "latestRevision" child element, and this will have state with
// value "INCOMPLETE" or "CORRUPT". If the file was new or had no previous complete versions the latestRevision
// will be the only child, but if not there may also be a "currentRevision" representing the previous
// complete version - and then we need to detect the case where our upload failed but there was an existing
// complete version!
using (var rs = areq.GetResponseStream())
// Write post data request
var areq = new AsyncHttpRequest(req);
using (var rs = areq.GetRequestStream())
using (var ts = rs.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout))
await Utility.Utility.CopyStreamAsync(stream, rs, true, cancelToken).ConfigureAwait(false);
// Send request, and check response
using (var resp = (HttpWebResponse)areq.GetResponse())
{
if (resp.StatusCode != HttpStatusCode.Created)
throw new WebException(Strings.Jottacloud.FileUploadError, null, WebExceptionStatus.ProtocolError, resp);
// Request seems to be successful, but we must verify the response XML content to be sure that the file
// was correctly uploaded: The server will verify the JSize header and mark the file as incomplete if
// there was mismatch, and it will verify the JMd5 header and mark the file as corrupt if there was a hash
// mismatch. The returned XML contains a file element, and if upload was error free it contains a single
// child element "currentRevision", which has a "state" child element with the string "COMPLETED".
// If there was a problem we should have a "latestRevision" child element, and this will have state with
// value "INCOMPLETE" or "CORRUPT". If the file was new or had no previous complete versions the latestRevision
// will be the only child, but if not there may also be a "currentRevision" representing the previous
// complete version - and then we need to detect the case where our upload failed but there was an existing
// complete version!
using (var rs = areq.GetResponseStream())
{
var doc = new System.Xml.XmlDocument();
try { doc.Load(rs); }
catch (System.Xml.XmlException)
{
var doc = new System.Xml.XmlDocument();
try { doc.Load(rs); }
catch (System.Xml.XmlException)
{
throw new WebException(Strings.Jottacloud.FileUploadError, WebExceptionStatus.ProtocolError);
}
bool uploadCompletedSuccessfully = false;
var xFile = doc["file"];
if (xFile != null)
{
var xRevState = xFile.SelectSingleNode("latestRevision");
if (xRevState == null)
{
xRevState = xFile.SelectSingleNode("currentRevision/state");
if (xRevState != null)
uploadCompletedSuccessfully = xRevState.InnerText == "COMPLETED"; // Success: There is no "latestRevision", only a "currentRevision" (and it specifies the file is complete, but I think it always will).
}
}
if (!uploadCompletedSuccessfully) // Report error (and we just let the incomplete/corrupt file revision stay on the server..)
throw new WebException(Strings.Jottacloud.FileUploadError, WebExceptionStatus.ProtocolError);
throw new WebException(Strings.Jottacloud.FileUploadError, WebExceptionStatus.ProtocolError);
}
bool uploadCompletedSuccessfully = false;
var xFile = doc["file"];
if (xFile != null)
{
var xRevState = xFile.SelectSingleNode("latestRevision");
if (xRevState == null)
{
xRevState = xFile.SelectSingleNode("currentRevision/state");
if (xRevState != null)
uploadCompletedSuccessfully = xRevState.InnerText == "COMPLETED"; // Success: There is no "latestRevision", only a "currentRevision" (and it specifies the file is complete, but I think it always will).
}
}
if (!uploadCompletedSuccessfully) // Report error (and we just let the incomplete/corrupt file revision stay on the server..)
throw new WebException(Strings.Jottacloud.FileUploadError, WebExceptionStatus.ProtocolError);
}
}
finally
{
try
{
if (tmpFile != null)
tmpFile.Dispose();
}
catch { }
}
}
}
}
Duplicati/Library/Backend/Rclone/Rclone.cs
+2 -2
View File
@@ -158,9 +158,9 @@ namespace Duplicati.Library.Backend
process.BeginOutputReadLine();
process.BeginErrorReadLine();
using var timer = timeout == Timeout.InfiniteTimeSpan
var timer = timeout == Timeout.InfiniteTimeSpan
? new TaskCompletionSource<bool>().Task
: Task.Delay(timeout);
: Task.Delay(timeout, cancelToken);
var killed = false;
while (!process.HasExited)
Duplicati/Library/Backend/S3/S3AwsClient.cs
+14 -2
View File
@@ -61,6 +61,10 @@ namespace Duplicati.Library.Backend
/// The option to specify if chunk encoding should be used
/// </summary>
private readonly bool m_useChunkEncoding;
/// <summary>
/// The option to specify if payload signing should be disabled
/// </summary>
private readonly bool m_disablePayloadSigning;
/// <summary>
/// The DNS host of the S3 server
@@ -93,7 +97,7 @@ namespace Duplicati.Library.Backend
]);
public S3AwsClient(string awsID, string awsKey, string? locationConstraint, string servername,
string? storageClass, bool useSSL, bool disableChunkEncoding, TimeoutOptionsHelper.Timeouts timeouts, Dictionary<string, string?> options)
string? storageClass, bool useSSL, bool disableChunkEncoding, bool disablePayloadSigning, TimeoutOptionsHelper.Timeouts timeouts, Dictionary<string, string?> options)
{
var cfg = GetDefaultAmazonS3Config();
cfg.UseHttp = !useSSL;
@@ -109,6 +113,7 @@ namespace Duplicati.Library.Backend
m_storageClass = storageClass;
m_dnsHost = string.IsNullOrWhiteSpace(cfg.ServiceURL) ? null : new System.Uri(cfg.ServiceURL).Host;
m_useChunkEncoding = !disableChunkEncoding;
m_disablePayloadSigning = disablePayloadSigning;
m_archiveClasses = ParseStorageClasses(options.GetValueOrDefault(S3_ARCHIVE_CLASSES_OPTION));
}
@@ -217,13 +222,20 @@ namespace Duplicati.Library.Backend
public virtual async Task AddFileStreamAsync(string bucketName, string keyName, Stream source,
CancellationToken cancelToken)
{
(source, var md5, var tmp) = await Utility.Utility.CalculateThrottledStreamHash(source, "MD5", cancelToken).ConfigureAwait(false);
using var _ = tmp;
md5 = Convert.ToBase64String(Utility.Utility.HexStringAsByteArray(md5));
using var ts = source.ObserveReadTimeout(m_timeouts.ReadWriteTimeout, false);
var objectAddRequest = new PutObjectRequest
{
BucketName = bucketName,
Key = keyName,
InputStream = ts,
UseChunkEncoding = m_useChunkEncoding
UseChunkEncoding = m_useChunkEncoding,
MD5Digest = md5,
DisablePayloadSigning = m_disablePayloadSigning
};
if (!string.IsNullOrWhiteSpace(m_storageClass))
objectAddRequest.StorageClass = new S3StorageClass(m_storageClass);
Duplicati/Library/Backend/S3/S3Backend.cs
+7 -4
View File
@@ -41,6 +41,7 @@ namespace Duplicati.Library.Backend
private const string SSL_OPTION = "use-ssl";
private const string S3_CLIENT_OPTION = "s3-client";
private const string S3_DISABLE_CHUNK_ENCODING_OPTION = "s3-disable-chunk-encoding";
private const string S3_DISABLE_PAYLOAD_SIGNING_OPTION = "s3-disable-payload-signing";
private const string S3_LIST_API_VERSION_OPTION = "s3-list-api-version";
private const string S3_RECURSIVE_LIST = "s3-recursive-list";
@@ -242,14 +243,15 @@ namespace Duplicati.Library.Backend
if (!options.ContainsKey("s3-ext-forcepathstyle") && !hostname.EndsWith(".amazonaws.com", StringComparison.OrdinalIgnoreCase))
options["s3-ext-forcepathstyle"] = "true";
var disableChunkEncoding = Utility.Utility.ParseBoolOption(options, S3_DISABLE_CHUNK_ENCODING_OPTION);
var s3ClientOptionValue = options.GetValueOrDefault(S3_CLIENT_OPTION);
(var awsID, var awsKey) = auth.GetCredentials();
if (string.IsNullOrWhiteSpace(s3ClientOptionValue) || string.Equals(s3ClientOptionValue, "aws", StringComparison.OrdinalIgnoreCase))
{
m_s3Client = new S3AwsClient(awsID, awsKey, locationConstraint, hostname, storageClass, useSSL, disableChunkEncoding, timeout, options);
var disableChunkEncoding = Utility.Utility.ParseBoolOption(options, S3_DISABLE_CHUNK_ENCODING_OPTION);
var disablePayloadSigning = Utility.Utility.ParseBoolOption(options, S3_DISABLE_PAYLOAD_SIGNING_OPTION);
m_s3Client = new S3AwsClient(awsID, awsKey, locationConstraint, hostname, storageClass, useSSL, disableChunkEncoding, disablePayloadSigning, timeout, options);
}
else if (string.Equals(s3ClientOptionValue, "minio", StringComparison.OrdinalIgnoreCase))
{
@@ -328,14 +330,15 @@ namespace Duplicati.Library.Backend
var exts = S3AwsClient.GetAwsExtendedOptions();
return [
new CommandLineArgument("aws-access-key-id", CommandLineArgument.ArgumentType.String, Strings.S3Backend.AMZUserIDDescriptionShort, Strings.S3Backend.AMZUserIDDescriptionLong, null, [AuthOptionsHelper.AuthUsernameOption], null),
new CommandLineArgument("aws-secret-access-key", CommandLineArgument.ArgumentType.Password, Strings.S3Backend.AMZKeyDescriptionShort, Strings.S3Backend.AMZKeyDescriptionLong,null, [AuthOptionsHelper.AuthPasswordOption], null ),
new CommandLineArgument(AUTH_USERNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.S3Backend.AMZUserIDDescriptionShort, Strings.S3Backend.AMZUserIDDescriptionLong, null, [AuthOptionsHelper.AuthUsernameOption], null),
new CommandLineArgument(AUTH_PASSWORD_OPTION, CommandLineArgument.ArgumentType.Password, Strings.S3Backend.AMZKeyDescriptionShort, Strings.S3Backend.AMZKeyDescriptionLong,null, [AuthOptionsHelper.AuthPasswordOption], null ),
new CommandLineArgument(STORAGECLASS_OPTION, CommandLineArgument.ArgumentType.String, Strings.S3Backend.S3StorageclassDescriptionShort, Strings.S3Backend.S3StorageclassDescriptionLong, "", null, KNOWN_S3_STORAGE_CLASSES.Select(x => x.Value).WhereNotNullOrWhiteSpace().ToArray()),
new CommandLineArgument(SERVER_NAME, CommandLineArgument.ArgumentType.String, Strings.S3Backend.S3ServerNameDescriptionShort, Strings.S3Backend.S3ServerNameDescriptionLong(hostnames.ToString()), DEFAULT_S3_HOST),
new CommandLineArgument(LOCATION_OPTION, CommandLineArgument.ArgumentType.String, Strings.S3Backend.S3LocationDescriptionShort, Strings.S3Backend.S3LocationDescriptionLong(locations.ToString())),
new CommandLineArgument(SSL_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionUseSSLShort, Strings.S3Backend.DescriptionUseSSLLong),
new CommandLineArgument(S3_CLIENT_OPTION, CommandLineArgument.ArgumentType.Enumeration, Strings.S3Backend.S3ClientDescriptionShort, Strings.S3Backend.S3ClientDescriptionLong, "aws", null, new string[] { "aws", "minio" }),
new CommandLineArgument(S3_DISABLE_CHUNK_ENCODING_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionDisableChunkEncodingShort, Strings.S3Backend.DescriptionDisableChunkEncodingLong, "false"),
new CommandLineArgument(S3_DISABLE_PAYLOAD_SIGNING_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionDisablePayloadSigningShort, Strings.S3Backend.DescriptionDisablePayloadSigningLong, "false"),
new CommandLineArgument(S3AwsClient.S3_ARCHIVE_CLASSES_OPTION, CommandLineArgument.ArgumentType.Flags, Strings.S3Backend.S3ArchiveClassesDescriptionShort, Strings.S3Backend.S3ArchiveClassesDescriptionLong, string.Join(",", S3AwsClient.DEFAULT_ARCHIVE_CLASSES.Select(x => x.Value)), null, KNOWN_S3_STORAGE_CLASSES.Select(x => x.Value).WhereNotNullOrWhiteSpace().ToArray()),
new CommandLineArgument(S3_LIST_API_VERSION_OPTION, CommandLineArgument.ArgumentType.Enumeration, Strings.S3Backend.DescriptionListApiVersionShort, Strings.S3Backend.DescriptionListApiVersionLong, "v1", null, ["v1", "v2"]),
new CommandLineArgument(S3_RECURSIVE_LIST, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionRecursiveListShort, Strings.S3Backend.DescriptionRecursiveListLong, "false"),
Duplicati/Library/Backend/S3/Strings.cs
+2
View File
@@ -47,6 +47,8 @@ namespace Duplicati.Library.Backend.Strings
public static string DescriptionUseSSLShort { get { return LC.L(@"Instruct Duplicati to use an SSL (https) connection"); } }
public static string DescriptionDisableChunkEncodingLong { get { return LC.L(@"This disables chunk encoding for the aws client, which is not supported by all S3 providers."); } }
public static string DescriptionDisableChunkEncodingShort { get { return LC.L(@"Disable chunk encoding (aws client only)"); } }
public static string DescriptionDisablePayloadSigningLong { get { return LC.L(@"This disables payload signing for the aws client, which is not supported by all S3 providers."); } }
public static string DescriptionDisablePayloadSigningShort { get { return LC.L(@"Disable payload signing (aws client only)"); } }
public static string S3StorageclassDescriptionLong { get { return LC.L(@"Use this option to specify a storage class. If this option is not used, the server will choose a default storage class."); } }
public static string S3StorageclassDescriptionShort { get { return LC.L(@"Specify storage class"); } }
public static string S3ArchiveClassesDescriptionShort { get { return LC.L(@"Specify archive storage class"); } }
Duplicati/Library/Backends/BackendModules.cs
-1
View File
@@ -42,7 +42,6 @@ public static class BackendModules
new Backend.AzureBlob.AzureBlobBackend(),
new Backend.Backblaze.B2(),
new Backend.Box.BoxBackend(),
new Backend.CloudFiles(),
new Backend.Dropbox(),
new Backend.FTP(),
new Backend.AlternateFTPBackend(),
Duplicati/Library/Logging/ILogDestination.cs
+15 -19
View File
@@ -18,22 +18,18 @@
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.Text;
namespace Duplicati.Library.Logging
{
/// <summary>
/// Interface for a log destination
/// </summary>
public interface ILogDestination
{
/// <summary>
/// The function called when a message is logged
/// </summary>
/// <param name="entry">The message to log</param>
void WriteMessage(LogEntry entry);
}
}
namespace Duplicati.Library.Logging
{
/// <summary>
/// Interface for a log destination
/// </summary>
public interface ILogDestination
{
/// <summary>
/// The function called when a message is logged
/// </summary>
/// <param name="entry">The message to log</param>
void WriteMessage(LogEntry entry);
}
}
Duplicati/Library/Logging/LogEntry.cs
+19 -19
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
Duplicati/Library/Main/Operation/DeleteHandler.cs
+43 -37
View File
@@ -89,50 +89,56 @@ namespace Duplicati.Library.Main.Operation
versionsToDelete = versionsToDelete.OrderBy(x => x.Version).Skip(1).ToList();
}
if (versionsToDelete.Count > 0)
Logging.Log.WriteInformationMessage(LOGTAG, "DeleteRemoteFileset", "Deleting {0} remote fileset(s) ...", versionsToDelete.Count);
var lst = db.DropFilesetsFromTable(versionsToDelete.Select(x => x.Time).ToArray(), rtr.Transaction).ToArray();
foreach (var f in lst)
db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, rtr.Transaction);
if (!m_options.Dryrun)
rtr.Commit("CommitBeforeDelete");
foreach (var f in lst)
if (versionsToDelete.Count == 0)
{
if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
{
await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
return;
}
if (!m_options.Dryrun)
await backendManager.DeleteAsync(f.Key, f.Value, false, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
else
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFileset", "Would delete remote fileset: {0}", f.Key);
}
await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
var count = lst.Length;
if (!m_options.Dryrun)
{
if (count == 0)
Logging.Log.WriteInformationMessage(LOGTAG, "DeleteResults", "No remote filesets were deleted");
else
Logging.Log.WriteInformationMessage(LOGTAG, "DeleteResults", "Deleted {0} remote fileset(s)", count);
Logging.Log.WriteInformationMessage(LOGTAG, "NoFilesetsToDelete", "No remote filesets should be deleted");
}
else
{
Logging.Log.WriteInformationMessage(LOGTAG, "DeleteRemoteFileset", "Deleting {0} remote fileset(s) ...", versionsToDelete.Count);
if (count == 0)
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteResults", "No remote filesets would be deleted");
var lst = db.DropFilesetsFromTable(versionsToDelete.Select(x => x.Time).ToArray(), rtr.Transaction).ToArray();
foreach (var f in lst)
db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, rtr.Transaction);
if (!m_options.Dryrun)
rtr.Commit("CommitBeforeDelete");
foreach (var f in lst)
{
if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
{
await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
return;
}
if (!m_options.Dryrun)
await backendManager.DeleteAsync(f.Key, f.Value, false, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
else
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFileset", "Would delete remote fileset: {0}", f.Key);
}
await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
var count = lst.Length;
if (!m_options.Dryrun)
{
if (count == 0)
Logging.Log.WriteInformationMessage(LOGTAG, "DeleteResults", "No remote filesets were deleted");
else
Logging.Log.WriteInformationMessage(LOGTAG, "DeleteResults", "Deleted {0} remote fileset(s)", count);
}
else
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteResults", "{0} remote fileset(s) would be deleted", count);
{
if (count > 0 && m_options.Dryrun)
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteHelp", "Remove --dry-run to actually delete files");
if (count == 0)
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteResults", "No remote filesets would be deleted");
else
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteResults", "{0} remote fileset(s) would be deleted", count);
if (count > 0 && m_options.Dryrun)
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteHelp", "Remove --dry-run to actually delete files");
}
}
if (!m_options.NoAutoCompact && (forceCompact || versionsToDelete.Count > 0))
Duplicati/Library/RestAPI/Abstractions/IApplicationSettings.cs
+64
View File
@@ -0,0 +1,64 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Threading;
using Duplicati.Library.Interface;
namespace Duplicati.WebserverCore.Abstractions;
/// <summary>
/// Interface for application-wide settings for the server.
/// </summary>
public interface IApplicationSettings
{
/// <summary>
/// Action to start or stop the usage reporter
/// </summary>
Action? StartOrStopUsageReporter { get; set; }
/// <summary>
/// Gets the folder where Duplicati data is stored
/// </summary>
string DataFolder { get; }
/// <summary>
/// Used to check the origin of the web server (e.g. Tray icon or a stand alone Server)
/// </summary>
string Origin { get; set; }
/// <summary>
/// The application exit event
/// </summary>
ManualResetEvent ApplicationExitEvent { get; }
/// <summary>
/// The shared secret provider from the server invocation
/// </summary>
ISecretProvider? SecretProvider { get; set; }
/// <summary>
/// Flag to indicate if the settings encryption key was provided externally
/// </summary>
bool SettingsEncryptionKeyProvidedExternally { get; set; }
}
Duplicati/Library/RestAPI/Abstractions/ILogWriteHandler.cs
+143
View File
@@ -0,0 +1,143 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using Duplicati.Library.Interface;
using Duplicati.Library.Logging;
namespace Duplicati.WebserverCore.Abstractions;
/// <summary>
/// Handles logging from the server,
/// and provides an entry point for the runner
/// to redirect log output to a file
/// </summary>
public interface ILogWriteHandler : ILogDestination, IDisposable
{
/// <summary>
/// Represents a single log event
/// </summary>
public struct LiveLogEntry
{
/// <summary>
/// The context key used for conveying the backup ID
/// </summary>
public const string LOG_EXTRA_BACKUPID = "BackupID";
/// <summary>
/// The context key used for conveying the task ID
/// </summary>
public const string LOG_EXTRA_TASKID = "TaskID";
/// <summary>
/// A unique ID that sequentially increments
/// </summary>
private static long _id;
/// <summary>
/// The time the message was logged
/// </summary>
public readonly DateTime When;
/// <summary>
/// The ID assigned to the message
/// </summary>
public readonly long ID;
/// <summary>
/// The logged message
/// </summary>
public readonly string Message;
/// <summary>
/// The log tag
/// </summary>
public readonly string Tag;
/// <summary>
/// The message ID
/// </summary>
public readonly string MessageID;
/// <summary>
/// The message ID
/// </summary>
public readonly string? ExceptionID;
/// <summary>
/// The message type
/// </summary>
public readonly LogMessageType Type;
/// <summary>
/// Exception data attached to the message
/// </summary>
public readonly Exception? Exception;
/// <summary>
/// The backup ID, if any
/// </summary>
public readonly string? BackupID;
/// <summary>
/// The task ID, if any
/// </summary>
public readonly string TaskID;
/// <summary>
/// Initializes a new instance of the <see cref="LiveLogEntry"/> struct.
/// </summary>
/// <param name="entry">The log entry to store</param>
public LiveLogEntry(LogEntry entry)
{
this.ID = System.Threading.Interlocked.Increment(ref _id);
this.When = entry.When;
this.Message = entry.FormattedMessage;
this.Type = entry.Level;
this.Exception = entry.Exception;
this.Tag = entry.FilterTag;
this.MessageID = entry.Id;
this.BackupID = entry[LOG_EXTRA_BACKUPID];
this.TaskID = entry[LOG_EXTRA_TASKID];
if (entry.Exception == null)
this.ExceptionID = null;
else if (entry.Exception is UserInformationException exception)
this.ExceptionID = exception.HelpID;
else
this.ExceptionID = entry.Exception.GetType().FullName;
}
}
void RenewTimeout(LogMessageType type);
void SetServerFile(string path, LogMessageType level);
void AppendLogDestination(ILogDestination destination, LogMessageType level);
LiveLogEntry[] AfterTime(DateTime offset, LogMessageType level);
LiveLogEntry[] AfterID(long id, LogMessageType level, int pagesize);
}
Duplicati/Library/RestAPI/Abstractions/IWorkerThreadsManager.cs → Duplicati/Library/RestAPI/Abstractions/IProgressStateProviderService.cs
+12 -12
View File
@@ -18,20 +18,20 @@
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using Duplicati.Library.Utility;
using Duplicati.Server;
namespace Duplicati.Library.RestAPI.Abstractions;
namespace Duplicati.WebserverCore.Abstractions;
public interface IWorkerThreadsManager
/// <summary>
/// Produces system information.
/// </summary>
public interface IProgressStateProviderService
{
void Spawn(Action<Runner.IRunnerData> item);
Tuple<long, string>? CurrentTask { get; }
WorkerThread<Runner.IRunnerData>? WorkerThread { get; }
void UpdateThrottleSpeeds(string? uploadSpeed, string? downloadSpeed);
long AddTask(Runner.IRunnerData data, bool skipQueue = false);
}
/// <summary>
/// Handler for progress state generation.
/// </summary>
Func<Server.Serialization.Interface.IProgressEventData>? GenerateProgressState { get; set; }
}
Duplicati/Library/RestAPI/Abstractions/IQueueRunnerService.cs
+107
View File
@@ -0,0 +1,107 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Collections.Generic;
using Duplicati.Library.Interface;
using Duplicati.Server.Serialization.Interface;
namespace Duplicati.WebserverCore.Abstractions;
/// <summary>
/// A cached task result
/// </summary>
/// <param name="TaskID">The task ID</param>
/// <param name="BackupId">The backup ID</param>
/// <param name="TaskStarted">The time the task started</param>
/// <param name="TaskFinished">The time the task finished</param>
/// <param name="Exception">The exception that was thrown</param>
public sealed record CachedTaskResult(long TaskID, string? BackupId, DateTime? TaskStarted, DateTime? TaskFinished, Exception? Exception);
/// <summary>
/// Class to encapsulate a thread that runs a list of queued operations
/// </summary>
/// <typeparam name="Tx">The type to operate on</typeparam>
public interface IQueueRunnerService
{
/// <summary>
/// Returns a copy of the current tasks in the queue
/// </summary>
/// <returns>A list of queued tasks</returns>
List<IQueuedTask> GetCurrentTasks();
/// <summary>
/// Gets a flag indicating if the queue is currently executing a task
/// </summary>
/// <returns>True if the queue is executing a task, false otherwise</returns>
bool GetIsActive();
/// <summary>
/// Returns the currently executing task in the queue
/// </summary>
/// <returns>The currently executing task, or null if no task is executing</returns>
IQueuedTask? GetCurrentTask();
/// <summary>
/// Gets the cached task results for a given task ID
/// </summary>
/// <param name="taskID">The task ID</param>
/// <returns>The cached task result</returns>
CachedTaskResult? GetCachedTaskResults(long taskID);
/// <summary>
/// Adds a task to the queue
/// </summary>
/// <param name="task">The task to add</param>
long AddTask(IQueuedTask task);
/// <summary>
/// Adds a task to the queue, optionally skipping the queue
/// </summary>
/// <param name="task">The task to add</param>
/// <param name="skipQueue">Whether to skip the queue</param>
long AddTask(IQueuedTask task, bool skipQueue);
/// <summary>
/// Removes a task from the queue
/// </summary>
/// <param name="wait">Whether to wait for the task to finish</param>
void Terminate(bool wait);
/// <summary>
/// Resumes processing items in the queue
/// </summary>
void Resume();
/// <summary>
/// Pauses processing items in the queue
/// </summary>
void Pause();
/// <summary>
/// Gets the IDs of the tasks in the worker queue
/// </summary>
/// <returns>A list of tuples containing the task ID and backup ID</returns>
IList<Tuple<long, string?>> GetQueueWithIds();
/// <summary>
/// Runs a task immediately, bypassing the queue.
/// Note that the task will run concurrently with the queue tasks and may cause database lock issues.
/// </summary>
/// <param name="task">The task to run</param>
IBasicResults? RunImmediately(IQueuedTask task);
}
Duplicati/Library/RestAPI/Abstractions/IScheduler.cs → Duplicati/Library/RestAPI/Abstractions/ISchedulerService.cs
+1 -24
View File
@@ -20,25 +20,12 @@
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using Duplicati.Library.Utility;
using Duplicati.Server;
using Duplicati.Server.Serialization.Interface;
namespace Duplicati.WebserverCore.Abstractions;
public interface IScheduler
public interface ISchedulerService
{
/// <summary>
/// Initializes scheduler
/// </summary>
/// <param name="worker">The worker thread</param>
void Init(WorkerThread<Runner.IRunnerData> worker);
/// <summary>
/// Gets the current ids in the scheduler queue
/// </summary>
IList<Tuple<long, string>> GetSchedulerQueueIds();
/// <summary>
/// Gets the current proposed schedule
/// </summary>
@@ -50,21 +37,11 @@ public interface IScheduler
/// <param name="wait">True if the call should block until the thread has exited, false otherwise</param>
void Terminate(bool wait);
/// <summary>
/// Subscribes to the event that is triggered when the schedule changes
/// </summary>
void SubScribeToNewSchedule(Action handler);
/// <summary>
/// A snapshot copy of the current schedule list
/// </summary>
List<KeyValuePair<DateTime, ISchedule>> Schedule { get; }
/// <summary>
/// A snapshot copy of the current worker queue, that is items that are scheduled, but waiting for execution
/// </summary>
List<Runner.IRunnerData> WorkerQueue { get; }
/// <summary>
/// Forces the scheduler to re-evaluate the order.
/// Call this method if something changes
Duplicati/Library/RestAPI/Database/Connection.cs
+63 -29
View File
@@ -32,6 +32,7 @@ using Duplicati.Library.AutoUpdater;
using System.Data;
using Duplicati.Library.Main.Database;
using System.Globalization;
using Microsoft.Extensions.DependencyInjection;
#nullable enable
@@ -39,6 +40,11 @@ namespace Duplicati.Server.Database
{
public class Connection : IDisposable
{
/// <summary>
/// The placeholder for passwords in the UI
/// </summary>
public const string PASSWORD_PLACEHOLDER = "**********";
private readonly IDbConnection m_connection;
private readonly IDbCommand m_errorcmd;
public readonly object m_lock = new object();
@@ -47,6 +53,11 @@ namespace Duplicati.Server.Database
private readonly Dictionary<string, Backup> m_temporaryBackups = new Dictionary<string, Backup>();
private readonly bool m_encryptSensitiveFields;
private readonly EncryptedFieldHelper.KeyInstance? m_key;
private IServiceProvider? m_serviceProvider;
private INotificationUpdateService? m_notificationUpdateService;
private EventPollNotify? m_eventPollNotifyer;
private readonly string m_dataFolder;
private static readonly HashSet<string> _encryptedFields =
BackendLoader.Backends.SelectMany(x => x.SupportedCommands ?? [])
.Concat(EncryptionLoader.Modules.SelectMany(x => x.SupportedCommands ?? []))
@@ -66,14 +77,31 @@ namespace Duplicati.Server.Database
])
.ToHashSet(StringComparer.OrdinalIgnoreCase);
public Connection(IDbConnection connection, bool disableFieldEncryption, EncryptedFieldHelper.KeyInstance? key)
public Connection(IDbConnection connection, bool disableFieldEncryption, EncryptedFieldHelper.KeyInstance? key, string dataFolder, Action startOrStopUsageReporter)
{
m_dataFolder = dataFolder;
m_encryptSensitiveFields = !disableFieldEncryption;
m_key = key;
m_connection = connection;
m_errorcmd = m_connection.CreateCommand(@"INSERT INTO ""ErrorLog"" (""BackupID"", ""Message"", ""Exception"", ""Timestamp"") VALUES (@BackupId,@Message,@Exception,@Timestamp)");
this.ApplicationSettings = new ServerSettings(this);
this.ApplicationSettings = new ServerSettings(this, startOrStopUsageReporter);
}
/// <summary>
/// The service provider is used to resolve dependencies
/// </summary>
internal IServiceProvider? ServiceProvider => m_serviceProvider;
/// <summary>
/// Set the service provider to be used for resolving dependencies
/// </summary>
/// <param name="sp">The service provider</param>
public void SetServiceProvider(IServiceProvider sp)
{
m_serviceProvider = sp;
m_notificationUpdateService = sp?.GetRequiredService<INotificationUpdateService>();
m_eventPollNotifyer = sp?.GetRequiredService<EventPollNotify>();
}
public bool IsEncryptingFields => m_encryptSensitiveFields;
@@ -125,7 +153,7 @@ namespace Duplicati.Server.Database
this.ApplicationSettings.PreloadSettingsHash = settingsHash;
}
public void LogError(string backupid, string message, Exception ex)
public void LogError(string? backupid, string message, Exception ex)
{
lock (m_lock)
{
@@ -310,7 +338,7 @@ namespace Duplicati.Server.Database
cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Option"" (""BackupID"", ""Filter"", ""Name"", ""Value"") VALUES (@BackupId, @Filter, @Name, @Value)"),
(cmd, f) =>
{
if (FIXMEGlobal.PASSWORD_PLACEHOLDER.Equals(f.Value))
if (PASSWORD_PLACEHOLDER.Equals(f.Value))
throw new Exception("Attempted to save a property with the placeholder password");
cmd.SetParameterValue("@BackupId", id)
@@ -608,8 +636,8 @@ namespace Duplicati.Server.Database
}
}
FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId();
FIXMEGlobal.StatusEventNotifyer.SignalNewEvent();
m_notificationUpdateService?.IncrementLastDataUpdateId();
m_eventPollNotifyer?.SignalNewEvent();
}
private void AddOrUpdateBackup(IBackup item, bool updateSchedule, ISchedule? schedule)
@@ -619,7 +647,7 @@ namespace Duplicati.Server.Database
bool update = item.ID != null;
if (!update && item.DBPath == null)
{
var folder = FIXMEGlobal.DataFolder;
var folder = m_dataFolder;
if (!System.IO.Directory.Exists(folder))
System.IO.Directory.CreateDirectory(folder);
@@ -652,7 +680,7 @@ namespace Duplicati.Server.Database
},
(cmd, n) =>
{
if (n.TargetURL.IndexOf(FIXMEGlobal.PASSWORD_PLACEHOLDER, StringComparison.Ordinal) >= 0)
if (n.TargetURL.IndexOf(PASSWORD_PLACEHOLDER, StringComparison.Ordinal) >= 0)
throw new Exception("Attempted to save a backup with the password placeholder");
if (update && long.Parse(n.ID) <= 0)
throw new Exception("Invalid update, cannot update application settings through update method");
@@ -722,8 +750,8 @@ namespace Duplicati.Server.Database
}
tr.Commit();
FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId();
FIXMEGlobal.StatusEventNotifyer.SignalNewEvent();
m_notificationUpdateService?.IncrementLastDataUpdateId();
m_eventPollNotifyer?.SignalNewEvent();
}
}
}
@@ -735,8 +763,8 @@ namespace Duplicati.Server.Database
{
AddOrUpdateSchedule(item, tr);
tr.Commit();
FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId();
FIXMEGlobal.StatusEventNotifyer.SignalNewEvent();
m_notificationUpdateService?.IncrementLastDataUpdateId();
m_eventPollNotifyer?.SignalNewEvent();
}
}
@@ -800,8 +828,8 @@ namespace Duplicati.Server.Database
}
}
FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId();
FIXMEGlobal.StatusEventNotifyer.SignalNewEvent();
m_notificationUpdateService?.IncrementLastDataUpdateId();
m_eventPollNotifyer?.SignalNewEvent();
}
public void DeleteBackup(IBackup backup)
@@ -820,8 +848,8 @@ namespace Duplicati.Server.Database
lock (m_lock)
DeleteFromDb("Schedule", ID);
FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId();
FIXMEGlobal.StatusEventNotifyer.SignalNewEvent();
m_notificationUpdateService?.IncrementLastDataUpdateId();
m_eventPollNotifyer?.SignalNewEvent();
}
public void DeleteSchedule(ISchedule schedule)
@@ -906,21 +934,27 @@ namespace Duplicati.Server.Database
return false;
DeleteFromDb(typeof(Notification).Name, id);
FIXMEGlobal.DataConnection.ApplicationSettings.UnackedError = notifications.Any(x => x.ID != id && x.Type == Duplicati.Server.Serialization.NotificationType.Error);
FIXMEGlobal.DataConnection.ApplicationSettings.UnackedWarning = notifications.Any(x => x.ID != id && x.Type == Duplicati.Server.Serialization.NotificationType.Warning);
this.ApplicationSettings.UnackedError = notifications.Any(x => x.ID != id && x.Type == Duplicati.Server.Serialization.NotificationType.Error);
this.ApplicationSettings.UnackedWarning = notifications.Any(x => x.ID != id && x.Type == Duplicati.Server.Serialization.NotificationType.Warning);
}
// Guard against dismissing notifications before the provider is initialized
if (FIXMEGlobal.Provider != null)
{
FIXMEGlobal.NotificationUpdateService.IncrementLastNotificationUpdateId();
FIXMEGlobal.StatusEventNotifyer.SignalNewEvent();
}
m_notificationUpdateService?.IncrementLastNotificationUpdateId();
m_eventPollNotifyer?.SignalNewEvent();
return true;
}
public void RegisterNotification(Serialization.NotificationType type, string title, string message, Exception ex, string backupid, string action, string logid, string messageid, string logtag, Func<INotification, INotification[], INotification> conflicthandler)
public void RegisterNotification(
Serialization.NotificationType type,
string title,
string message,
Exception? ex,
string? backupid,
string action,
string? logid,
string? messageid,
string? logtag,
Func<INotification, INotification[], INotification> conflicthandler)
{
lock (m_lock)
{
@@ -949,13 +983,13 @@ namespace Duplicati.Server.Database
OverwriteAndUpdateDb(null, null, [notification], false);
if (type == Serialization.NotificationType.Error)
FIXMEGlobal.DataConnection.ApplicationSettings.UnackedError = true;
ApplicationSettings.UnackedError = true;
else if (type == Serialization.NotificationType.Warning)
FIXMEGlobal.DataConnection.ApplicationSettings.UnackedWarning = true;
ApplicationSettings.UnackedWarning = true;
}
FIXMEGlobal.NotificationUpdateService.IncrementLastNotificationUpdateId();
FIXMEGlobal.StatusEventNotifyer.SignalNewEvent();
m_notificationUpdateService?.IncrementLastNotificationUpdateId();
m_eventPollNotifyer?.SignalNewEvent();
}
//Workaround to clean up the database after invalid settings update
Duplicati/Library/RestAPI/Database/ServerSettings.cs
+19 -15
View File
@@ -1,4 +1,4 @@
// Copyright (C) 2025, The Duplicati Team
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
@@ -29,6 +29,8 @@ using System.Text;
using System.Security.Cryptography.X509Certificates;
using Duplicati.Library.Utility;
using Duplicati.Library.AutoUpdater;
using Microsoft.Extensions.DependencyInjection;
using Duplicati.WebserverCore.Abstractions;
#nullable enable
@@ -78,10 +80,12 @@ namespace Duplicati.Server.Database
private readonly Dictionary<string, string?> settings;
private readonly Connection databaseConnection;
private UpdateInfo? m_latestUpdate;
private UpdateInfo? latestUpdate;
private readonly Action? startOrStopUsageReporter;
internal ServerSettings(Connection con)
internal ServerSettings(Connection con, Action startOrStopUsageReporter)
{
this.startOrStopUsageReporter = startOrStopUsageReporter;
settings = new Dictionary<string, string?>();
databaseConnection = con;
ReloadSettings();
@@ -113,7 +117,7 @@ namespace Duplicati.Server.Database
lock (databaseConnection.m_lock)
{
m_latestUpdate = null;
latestUpdate = null;
if (clearExisting)
settings.Clear();
@@ -142,17 +146,17 @@ namespace Duplicati.Server.Database
Value = n.Value
}, Database.Connection.SERVER_SETTINGS_ID);
if (FIXMEGlobal.IsServerStarted)
var provider = databaseConnection.ServiceProvider;
if (provider != null)
{
FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId();
FIXMEGlobal.StatusEventNotifyer.SignalNewEvent();
provider?.GetRequiredService<INotificationUpdateService>()?.IncrementLastDataUpdateId();
provider?.GetRequiredService<EventPollNotify>()?.SignalNewEvent();
// If throttle options were changed, update now
FIXMEGlobal.WorkerThreadsManager.UpdateThrottleSpeeds(UploadSpeedLimit, DownloadSpeedLimit);
provider?.GetRequiredService<IQueueRunnerService>()?.GetCurrentTask()?.UpdateThrottleSpeeds(UploadSpeedLimit, DownloadSpeedLimit);
}
// In case the usage reporter is enabled or disabled, refresh now
if (FIXMEGlobal.StartOrStopUsageReporter != null)
FIXMEGlobal.StartOrStopUsageReporter();
startOrStopUsageReporter?.Invoke();
}
public string? StartupDelayDuration
@@ -602,7 +606,7 @@ namespace Duplicati.Server.Database
lock (databaseConnection.m_lock)
settings[CONST.UPDATE_CHECK_INTERVAL] = value;
SaveSettings();
FIXMEGlobal.UpdatePoller.Reschedule();
databaseConnection?.ServiceProvider?.GetRequiredService<UpdatePollThread>()?.Reschedule();
}
}
@@ -631,11 +635,11 @@ namespace Duplicati.Server.Database
try
{
if (m_latestUpdate != null)
return m_latestUpdate;
if (latestUpdate != null)
return latestUpdate;
using (var tr = new System.IO.StringReader(updateNew))
return m_latestUpdate = Server.Serialization.Serializer.Deserialize<Library.AutoUpdater.UpdateInfo>(tr);
return latestUpdate = Server.Serialization.Serializer.Deserialize<Library.AutoUpdater.UpdateInfo>(tr);
}
catch
{
@@ -655,7 +659,7 @@ namespace Duplicati.Server.Database
result = sb.ToString();
}
m_latestUpdate = value;
latestUpdate = value;
lock (databaseConnection.m_lock)
settings[CONST.UPDATE_CHECK_NEW_VERSION] = result;
Duplicati/Library/RestAPI/FIXMEGlobal.cs
-135
View File
@@ -1,135 +0,0 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Server;
using System;
using System.Collections.Generic;
using Duplicati.Library.RestAPI.Abstractions;
using Duplicati.Library.Utility;
using Duplicati.WebserverCore.Abstractions;
using Microsoft.Extensions.DependencyInjection;
using Duplicati.Library.Interface;
namespace Duplicati.Library.RestAPI
{
/**
* In the absense of dependancy injection, there is a significant amount of variables exposed through Program as globals.
* This causes a problem decoupling classes and leads to circular dependancies.
*/
public static class FIXMEGlobal
{
/// <summary>
/// The placeholder for passwords in the UI
/// </summary>
public const string PASSWORD_PLACEHOLDER = "**********";
public static IServiceProvider Provider { get; set; }
/// <summary>
/// This is the only access to the database
/// </summary>
public static Server.Database.Connection DataConnection;
/// <summary>
/// A delegate method for creating a copy of the current progress state
/// </summary>
public static Func<Duplicati.Server.Serialization.Interface.IProgressEventData> GenerateProgressState;
/// <summary>
/// The status event signaler, used to control long polling of status updates
/// </summary>
public static EventPollNotify StatusEventNotifyer => Provider.GetRequiredService<EventPollNotify>();
/// <summary>
/// For keeping and incrementing last last events Ids of db save and last notification
/// </summary>
public static INotificationUpdateService NotificationUpdateService => Provider.GetRequiredService<INotificationUpdateService>();
/// <summary>
/// Checks if the server has started and is listening for events
/// </summary>
public static bool IsServerStarted => Provider != null;
/// <summary>
/// This is the working thread
/// </summary>
public static WorkerThread<Runner.IRunnerData> WorkThread =>
Provider.GetRequiredService<IWorkerThreadsManager>().WorkerThread;
public static IWorkerThreadsManager WorkerThreadsManager =>
Provider.GetRequiredService<IWorkerThreadsManager>();
public static Action StartOrStopUsageReporter;
/// <summary>
/// Gets the folder where Duplicati data is stored
/// </summary>
public static string DataFolder;
/// <summary>
/// This is the scheduling thread
/// </summary>
public static IScheduler Scheduler => Provider.GetRequiredService<IScheduler>();
/// <summary>
/// The log redirect handler
/// </summary>
public static readonly LogWriteHandler LogHandler = new LogWriteHandler();
/// <summary>
/// The update poll thread.
/// </summary>
public static UpdatePollThread UpdatePoller => Provider.GetRequiredService<UpdatePollThread>();
/// <summary>
/// Used to check the origin of the web server (e.g. Tray icon or a stand alone Server)
/// </summary>
public static string Origin = "Server";
/// <summary>
/// The application exit event
/// </summary>
public static System.Threading.ManualResetEvent ApplicationExitEvent;
/// <summary>
/// List of completed task results
/// </summary>
public static readonly List<KeyValuePair<long, Exception>> TaskResultCache = new List<KeyValuePair<long, Exception>>();
/// <summary>
/// This is the lock to be used before manipulating the shared resources
/// </summary>
public static readonly object MainLock = new object();
/// <summary>
/// The shared secret provider from the server invocation
/// </summary>
public static ISecretProvider SecretProvider { get; set; }
/// <summary>
/// Flag to indicate if the settings encryption key was provided externally
/// </summary>
public static bool SettingsEncryptionKeyProvidedExternally { get; set; }
}
}
Duplicati/Library/RestAPI/LogWriteHandler.cs
+32 -127
View File
@@ -1,30 +1,30 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Linq;
using Duplicati.Library.Logging;
using System.Collections.Generic;
using Duplicati.Library.Interface;
using Duplicati.Library.Main;
using Duplicati.WebserverCore.Abstractions;
using Duplicati.Library.Logging;
namespace Duplicati.Server
{
@@ -33,7 +33,7 @@ namespace Duplicati.Server
/// and provides an entry point for the runner
/// to redirect log output to a file
/// </summary>
public class LogWriteHandler : ILogDestination, IDisposable
public class LogWriteHandler : ILogWriteHandler
{
/// <summary>
/// The number of messages to keep when inactive
@@ -44,101 +44,6 @@ namespace Duplicati.Server
/// </summary>
private const int ACTIVE_SIZE = 5000;
/// <summary>
/// The context key used for conveying the backup ID
/// </summary>
public const string LOG_EXTRA_BACKUPID = "BackupID";
/// <summary>
/// The context key used for conveying the task ID
/// </summary>
public const string LOG_EXTRA_TASKID = "TaskID";
/// <summary>
/// Represents a single log event
/// </summary>
public struct LogEntry
{
/// <summary>
/// A unique ID that sequentially increments
/// </summary>
private static long _id;
/// <summary>
/// The time the message was logged
/// </summary>
public readonly DateTime When;
/// <summary>
/// The ID assigned to the message
/// </summary>
public readonly long ID;
/// <summary>
/// The logged message
/// </summary>
public readonly string Message;
/// <summary>
/// The log tag
/// </summary>
public readonly string Tag;
/// <summary>
/// The message ID
/// </summary>
public readonly string MessageID;
/// <summary>
/// The message ID
/// </summary>
public readonly string ExceptionID;
/// <summary>
/// The message type
/// </summary>
public readonly LogMessageType Type;
/// <summary>
/// Exception data attached to the message
/// </summary>
public readonly Exception Exception;
/// <summary>
/// The backup ID, if any
/// </summary>
public readonly string BackupID;
/// <summary>
/// The task ID, if any
/// </summary>
public readonly string TaskID;
/// <summary>
/// Initializes a new instance of the <see cref="Duplicati.Server.LogWriteHandler+LogEntry"/> struct.
/// </summary>
/// <param name="entry">The log entry to store</param>
public LogEntry(Duplicati.Library.Logging.LogEntry entry)
{
this.ID = System.Threading.Interlocked.Increment(ref _id);
this.When = entry.When;
this.Message = entry.FormattedMessage;
this.Type = entry.Level;
this.Exception = entry.Exception;
this.Tag = entry.FilterTag;
this.MessageID = entry.Id;
this.BackupID = entry[LOG_EXTRA_BACKUPID];
this.TaskID = entry[LOG_EXTRA_TASKID];
if (entry.Exception == null)
this.ExceptionID = null;
else if (entry.Exception is UserInformationException exception)
this.ExceptionID = exception.HelpID;
else
this.ExceptionID = entry.Exception.GetType().FullName;
}
}
/// <summary>
/// Basic implementation of a ring-buffer
/// </summary>
@@ -208,7 +113,7 @@ namespace Duplicati.Server
private readonly DateTime[] m_timeouts;
private readonly object m_lock = new object();
private volatile bool m_anytimeouts = false;
private RingBuffer<LogEntry> m_buffer;
private RingBuffer<ILogWriteHandler.LiveLogEntry> m_buffer;
private readonly ControllerMultiLogTarget m_target = new ControllerMultiLogTarget(null, LogMessageType.Warning, null);
@@ -218,7 +123,7 @@ namespace Duplicati.Server
{
var fields = Enum.GetValues(typeof(LogMessageType));
m_timeouts = new DateTime[fields.Length];
m_buffer = new RingBuffer<LogEntry>(INACTIVE_SIZE);
m_buffer = new RingBuffer<ILogWriteHandler.LiveLogEntry>(INACTIVE_SIZE);
}
public void RenewTimeout(LogMessageType type)
@@ -228,7 +133,7 @@ namespace Duplicati.Server
m_timeouts[(int)type] = DateTime.Now.AddSeconds(30);
m_anytimeouts = true;
if (m_buffer == null || m_buffer.Size == INACTIVE_SIZE)
m_buffer = new RingBuffer<LogEntry>(ACTIVE_SIZE, m_buffer);
m_buffer = new RingBuffer<ILogWriteHandler.LiveLogEntry>(ACTIVE_SIZE, m_buffer);
}
}
@@ -248,7 +153,7 @@ namespace Duplicati.Server
UpdateLogLevel();
}
public LogEntry[] AfterTime(DateTime offset, LogMessageType level)
public ILogWriteHandler.LiveLogEntry[] AfterTime(DateTime offset, LogMessageType level)
{
RenewTimeout(level);
UpdateLogLevel();
@@ -257,13 +162,13 @@ namespace Duplicati.Server
lock (m_lock)
{
if (m_buffer == null)
return new LogEntry[0];
return new ILogWriteHandler.LiveLogEntry[0];
return m_buffer.FlatArray((x) => x.When > offset && x.Type >= level);
}
}
public LogEntry[] AfterID(long id, LogMessageType level, int pagesize)
public ILogWriteHandler.LiveLogEntry[] AfterID(long id, LogMessageType level, int pagesize)
{
RenewTimeout(level);
UpdateLogLevel();
@@ -271,7 +176,7 @@ namespace Duplicati.Server
lock (m_lock)
{
if (m_buffer == null)
return new LogEntry[0];
return [];
var buffer = m_buffer.FlatArray((x) => x.ID > id && x.Type >= level);
// Return the <page_size> newest entries
@@ -305,7 +210,7 @@ namespace Duplicati.Server
#region ILog implementation
public void WriteMessage(Duplicati.Library.Logging.LogEntry entry)
public void WriteMessage(LogEntry entry)
{
if (entry.Level < m_logLevel)
return;
@@ -329,13 +234,13 @@ namespace Duplicati.Server
UpdateLogLevel();
m_anytimeouts = false;
if (m_buffer == null || m_buffer.Size != INACTIVE_SIZE)
m_buffer = new RingBuffer<LogEntry>(INACTIVE_SIZE, m_buffer);
m_buffer = new RingBuffer<ILogWriteHandler.LiveLogEntry>(INACTIVE_SIZE, m_buffer);
}
}
if (m_buffer != null)
m_buffer.Enqueue(new LogEntry(entry));
m_buffer.Enqueue(new ILogWriteHandler.LiveLogEntry(entry));
}
}
Duplicati/Library/RestAPI/Runner.cs
+220 -175
View File
@@ -19,6 +19,8 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Linq;
using System.Collections.Generic;
@@ -26,46 +28,57 @@ using Duplicati.Library.Interface;
using Duplicati.Server.Serialization;
using Duplicati.Library.RestAPI;
using Duplicati.Library.Utility;
using System.Threading.Tasks;
using Duplicati.Server.Database;
using Duplicati.Server.Serialization.Interface;
using Duplicati.WebserverCore.Abstractions;
namespace Duplicati.Server
{
public static class Runner
{
public interface IRunnerData : Duplicati.Server.Serialization.Interface.IQueuedTask
public interface IRunnerData : Serialization.Interface.IQueuedTask
{
Duplicati.Server.Serialization.Interface.IBackup Backup { get; }
IDictionary<string, string> ExtraOptions { get; }
string[] FilterStrings { get; }
string[] ExtraArguments { get; }
Serialization.Interface.IBackup? Backup { get; }
IDictionary<string, string?>? ExtraOptions { get; }
string[]? FilterStrings { get; }
string[]? ExtraArguments { get; }
int PageSize { get; }
int PageOffset { get; }
void Stop();
void Abort();
void Pause(bool alsoTransfers);
void Resume();
void UpdateThrottleSpeed(string uploadSpeed, string downloadSpeed);
void SetController(Duplicati.Library.Main.Controller controller);
void SetController(Library.Main.Controller? controller);
}
private class RunnerData : IRunnerData
{
private static long RunnerTaskID = 1;
public Duplicati.Server.Serialization.DuplicatiOperation Operation { get; internal set; }
public Duplicati.Server.Serialization.Interface.IBackup Backup { get; internal set; }
public IDictionary<string, string> ExtraOptions { get; internal set; }
public string[] FilterStrings { get; internal set; }
public Func<Task>? OnStarting { get; set; }
public Func<Exception?, Task>? OnFinished { get; set; }
public string BackupID { get { return Backup.ID; } }
/// <summary>
/// Callback to be executed when the task is finished.
/// Only supported for the delete operation for now.
/// </summary>
internal Action<IRunnerData>? AfterTaskFinished { get; set; }
public DuplicatiOperation Operation { get; internal set; }
public Serialization.Interface.IBackup? Backup { get; internal set; }
public IDictionary<string, string?>? ExtraOptions { get; internal set; }
public string[]? FilterStrings { get; internal set; }
public string? BackupID { get { return Backup?.ID; } }
public long TaskID { get { return m_taskID; } }
public string[] ExtraArguments { get; internal set; }
public string[]? ExtraArguments { get; internal set; }
public int PageSize { get; internal set; } = 0;
public int PageOffset { get; internal set; } = 0;
internal Duplicati.Library.Main.Controller Controller { get; set; }
public DateTime? TaskStarted { get; set; }
public DateTime? TaskFinished { get; set; }
public void SetController(Duplicati.Library.Main.Controller controller)
internal Library.Main.Controller? Controller { get; set; }
public void SetController(Library.Main.Controller? controller)
{
Controller = controller;
}
@@ -93,7 +106,7 @@ namespace Duplicati.Server
public long OriginalUploadSpeed { get; set; }
public long OriginalDownloadSpeed { get; set; }
public void UpdateThrottleSpeed(string uploadSpeed, string downloadSpeed)
public void UpdateThrottleSpeeds(string? uploadSpeed, string? downloadSpeed)
{
var controller = this.Controller;
if (controller == null)
@@ -108,14 +121,14 @@ namespace Duplicati.Server
try
{
if (!string.IsNullOrWhiteSpace(uploadSpeed))
server_upload_throttle = Duplicati.Library.Utility.Sizeparser.ParseSize(uploadSpeed, "kb");
server_upload_throttle = Sizeparser.ParseSize(uploadSpeed, "kb");
}
catch { }
try
{
if (!string.IsNullOrWhiteSpace(downloadSpeed))
server_download_throttle = Duplicati.Library.Utility.Sizeparser.ParseSize(downloadSpeed, "kb");
server_download_throttle = Sizeparser.ParseSize(downloadSpeed, "kb");
}
catch { }
@@ -159,13 +172,13 @@ namespace Duplicati.Server
return new CustomRunnerTask(runner);
}
public static IRunnerData CreateTask(Duplicati.Server.Serialization.DuplicatiOperation operation, Duplicati.Server.Serialization.Interface.IBackup backup, IDictionary<string, string> extraOptions = null, string[] filterStrings = null, string[] extraArguments = null, int pageSize = 0, int pageOffset = 0)
public static IRunnerData CreateTask(DuplicatiOperation operation, IBackup backup, IDictionary<string, string?>? extraOptions = null, string[]? filterStrings = null, string[]? extraArguments = null, int pageSize = 0, int pageOffset = 0)
{
return new RunnerData()
{
Operation = operation,
Backup = backup,
ExtraOptions = extraOptions,
ExtraOptions = extraOptions ?? new Dictionary<string, string?>(),
FilterStrings = filterStrings,
ExtraArguments = extraArguments,
PageSize = pageSize,
@@ -173,15 +186,26 @@ namespace Duplicati.Server
};
}
public static IRunnerData CreateListTask(Duplicati.Server.Serialization.Interface.IBackup backup, string[] filters, bool onlyPrefix, bool allVersions, bool folderContents, DateTime time)
public static IRunnerData CreateDeleteTask(IBackup backup, IDictionary<string, string?> extraOptions, Action<IRunnerData>? afterTaskFinished = null)
{
var dict = new Dictionary<string, string>();
return new RunnerData()
{
Operation = DuplicatiOperation.Delete,
Backup = backup,
ExtraOptions = extraOptions,
AfterTaskFinished = afterTaskFinished
};
}
public static IRunnerData CreateListTask(IBackup backup, string[]? filters, bool onlyPrefix, bool allVersions, bool folderContents, DateTime time)
{
var dict = new Dictionary<string, string?>();
if (onlyPrefix)
dict["list-prefix-only"] = "true";
if (allVersions)
dict["all-versions"] = "true";
if (time.Ticks > 0)
dict["time"] = Duplicati.Library.Utility.Utility.SerializeDateTime(time.ToUniversalTime());
dict["time"] = Utility.SerializeDateTime(time.ToUniversalTime());
if (folderContents)
dict["list-folder-contents"] = "true";
@@ -192,19 +216,19 @@ namespace Duplicati.Server
filters);
}
public static IRunnerData CreateListFilesetsTask(Duplicati.Server.Serialization.Interface.IBackup backup, Dictionary<string, string> extraOptions = null)
public static IRunnerData CreateListFilesetsTask(IBackup backup, Dictionary<string, string?>? extraOptions = null)
{
return CreateTask(
DuplicatiOperation.ListFilesets,
backup,
extraOptions ?? new Dictionary<string, string>());
extraOptions ?? new Dictionary<string, string?>());
}
public static IRunnerData CreateListFolderContents(Duplicati.Server.Serialization.Interface.IBackup backup, string[] folders, DateTime time, int pageSize, int pageOffset)
public static IRunnerData CreateListFolderContents(IBackup backup, string[]? folders, DateTime time, int pageSize, int pageOffset)
{
var dict = new Dictionary<string, string>();
var dict = new Dictionary<string, string?>();
if (time.Ticks > 0)
dict["time"] = Duplicati.Library.Utility.Utility.SerializeDateTime(time.ToUniversalTime());
dict["time"] = Utility.SerializeDateTime(time.ToUniversalTime());
return CreateTask(
DuplicatiOperation.ListFolderContents,
@@ -215,9 +239,9 @@ namespace Duplicati.Server
pageOffset: pageOffset);
}
public static IRunnerData ListFileVersionsTask(Duplicati.Server.Serialization.Interface.IBackup backup, string[] filepaths, int pageSize, int pageOffset)
public static IRunnerData ListFileVersionsTask(IBackup backup, string[]? filepaths, int pageSize, int pageOffset)
{
var dict = new Dictionary<string, string>();
var dict = new Dictionary<string, string?>();
return CreateTask(
DuplicatiOperation.ListFileVersions,
backup,
@@ -227,11 +251,11 @@ namespace Duplicati.Server
pageOffset: pageOffset);
}
public static IRunnerData CreateSearchEntriesTask(Duplicati.Server.Serialization.Interface.IBackup backup, string[] filters, string[] folders, DateTime time, int pageSize, int pageOffset)
public static IRunnerData CreateSearchEntriesTask(IBackup backup, string[]? filters, string[]? folders, DateTime time, int pageSize, int pageOffset)
{
var dict = new Dictionary<string, string>();
var dict = new Dictionary<string, string?>();
if (time.Ticks > 0)
dict["time"] = Duplicati.Library.Utility.Utility.SerializeDateTime(time.ToUniversalTime());
dict["time"] = Utility.SerializeDateTime(time.ToUniversalTime());
return CreateTask(
DuplicatiOperation.SearchEntries,
@@ -244,17 +268,17 @@ namespace Duplicati.Server
}
public static IRunnerData CreateRestoreTask(Duplicati.Server.Serialization.Interface.IBackup backup, string[] filters,
DateTime time, string restoreTarget, bool overwrite, bool restore_permissions,
bool skip_metadata, string passphrase)
public static IRunnerData CreateRestoreTask(IBackup backup, string[]? filters,
DateTime time, string? restoreTarget, bool overwrite, bool restore_permissions,
bool skip_metadata, string? passphrase)
{
var dict = new Dictionary<string, string>
var dict = new Dictionary<string, string?>
{
["time"] = Library.Utility.Utility.SerializeDateTime(time.ToUniversalTime()),
["overwrite"] = overwrite ? Boolean.TrueString : Boolean.FalseString,
["restore-permissions"] = restore_permissions ? Boolean.TrueString : Boolean.FalseString,
["skip-metadata"] = skip_metadata ? Boolean.TrueString : Boolean.FalseString,
["allow-passphrase-change"] = Boolean.TrueString
["time"] = Utility.SerializeDateTime(time.ToUniversalTime()),
["overwrite"] = overwrite ? bool.TrueString : bool.FalseString,
["restore-permissions"] = restore_permissions ? bool.TrueString : bool.FalseString,
["skip-metadata"] = skip_metadata ? bool.TrueString : bool.FalseString,
["allow-passphrase-change"] = bool.TrueString
};
if (!string.IsNullOrWhiteSpace(restoreTarget))
dict["restore-path"] = SpecialFolders.ExpandEnvironmentVariables(restoreTarget);
@@ -267,26 +291,26 @@ namespace Duplicati.Server
dict,
filters);
}
private class MessageSink : Duplicati.Library.Main.IMessageSink
private class MessageSink : Library.Main.IMessageSink
{
private class ProgressState : Server.Serialization.Interface.IProgressEventData
private class ProgressState : IProgressEventData
{
private readonly string m_backupID;
private readonly string? m_backupID;
private readonly long m_taskID;
internal Duplicati.Library.Main.BackendActionType m_backendAction;
internal string m_backendPath;
internal Library.Main.BackendActionType m_backendAction;
internal string? m_backendPath;
internal long m_backendFileSize;
internal long m_backendFileProgress;
internal long m_backendSpeed;
internal bool m_backendIsBlocking;
internal string m_currentFilename;
internal string? m_currentFilename;
internal long m_currentFilesize;
internal long m_currentFileoffset;
internal bool m_currentFilecomplete;
internal Duplicati.Library.Main.OperationPhase m_phase;
internal Library.Main.OperationPhase m_phase;
internal float m_overallProgress;
internal long m_processedFileCount;
internal long m_processedFileSize;
@@ -294,7 +318,7 @@ namespace Duplicati.Server
internal long m_totalFileSize;
internal bool m_stillCounting;
public ProgressState(long taskId, string backupId)
public ProgressState(long taskId, string? backupId)
{
m_backupID = backupId;
m_taskID = taskId;
@@ -306,15 +330,15 @@ namespace Duplicati.Server
}
#region IProgressEventData implementation
public string BackupID { get { return m_backupID; } }
public string? BackupID { get { return m_backupID; } }
public long TaskID { get { return m_taskID; } }
public string BackendAction { get { return m_backendAction.ToString(); } }
public string BackendPath { get { return m_backendPath; } }
public string? BackendPath { get { return m_backendPath; } }
public long BackendFileSize { get { return m_backendFileSize; } }
public long BackendFileProgress { get { return m_backendFileProgress; } }
public long BackendSpeed { get { return m_backendSpeed; } }
public bool BackendIsBlocking { get { return m_backendIsBlocking; } }
public string CurrentFilename { get { return m_currentFilename; } }
public string? CurrentFilename { get { return m_currentFilename; } }
public long CurrentFilesize { get { return m_currentFilesize; } }
public long CurrentFileoffset { get { return m_currentFileoffset; } }
public bool CurrentFilecomplete { get { return m_currentFilecomplete; } }
@@ -329,16 +353,16 @@ namespace Duplicati.Server
}
private readonly ProgressState m_state;
private Duplicati.Library.Main.IBackendProgress m_backendProgress;
private Duplicati.Library.Main.IOperationProgress m_operationProgress;
private Library.Main.IBackendProgress? m_backendProgress;
private Library.Main.IOperationProgress? m_operationProgress;
private readonly object m_lock = new object();
public MessageSink(long taskId, string backupId)
public MessageSink(long taskId, string? backupId)
{
m_state = new ProgressState(taskId, backupId);
}
public Server.Serialization.Interface.IProgressEventData Copy()
public Serialization.Interface.IProgressEventData Copy()
{
lock (m_lock)
{
@@ -387,16 +411,18 @@ namespace Duplicati.Server
#endregion
}
public static string GetCommandLine(IRunnerData data)
public static string GetCommandLine(Connection databaseConnection, IRunnerData data)
{
var backup = data.Backup;
if (backup == null)
throw new ArgumentNullException(nameof(backup));
var options = ApplyOptions(backup, GetCommonOptions());
var options = ApplyOptions(databaseConnection, backup, GetCommonOptions(databaseConnection));
if (data.ExtraOptions != null)
foreach (var k in data.ExtraOptions)
options[k.Key] = k.Value;
var cf = FIXMEGlobal.DataConnection.Filters;
var cf = databaseConnection.Filters;
var bf = backup.Filters;
var sources =
@@ -411,36 +437,38 @@ namespace Duplicati.Server
);
var cmd = new System.Text.StringBuilder();
cmd.Append(Library.Utility.Utility.WrapAsCommandLine(new string[] { exe, "backup", backup.TargetURL }, false));
cmd.Append(Utility.WrapAsCommandLine([exe, "backup", backup.TargetURL], false));
cmd.Append(" ");
cmd.Append(Library.Utility.Utility.WrapAsCommandLine(sources, true));
cmd.Append(Utility.WrapAsCommandLine(sources, true));
// TODO: We should check each option to see if it is a path, and allow expansion on that
foreach (var opt in options)
cmd.AppendFormat(" --{0}={1}", opt.Key, Library.Utility.Utility.WrapCommandLineElement(opt.Value, false));
cmd.AppendFormat(" --{0}={1}", opt.Key, Utility.WrapCommandLineElement(opt.Value, false));
if (cf != null)
foreach (var f in cf)
cmd.AppendFormat(" --{0}={1}", f.Include ? "include" : "exclude", Library.Utility.Utility.WrapCommandLineElement(f.Expression, true));
cmd.AppendFormat(" --{0}={1}", f.Include ? "include" : "exclude", Utility.WrapCommandLineElement(f.Expression, true));
if (bf != null)
foreach (var f in bf)
cmd.AppendFormat(" --{0}={1}", f.Include ? "include" : "exclude", Library.Utility.Utility.WrapCommandLineElement(f.Expression, true));
cmd.AppendFormat(" --{0}={1}", f.Include ? "include" : "exclude", Utility.WrapCommandLineElement(f.Expression, true));
return cmd.ToString();
}
public static string[] GetCommandLineParts(IRunnerData data)
public static string[] GetCommandLineParts(Connection databaseConnection, IRunnerData data)
{
var backup = data.Backup;
if (backup == null)
throw new ArgumentNullException(nameof(backup));
var options = ApplyOptions(backup, GetCommonOptions());
var options = ApplyOptions(databaseConnection, backup, GetCommonOptions(databaseConnection));
if (data.ExtraOptions != null)
foreach (var k in data.ExtraOptions)
options[k.Key] = k.Value;
var cf = FIXMEGlobal.DataConnection.Filters;
var cf = databaseConnection.Filters;
var bf = backup.Filters;
var sources =
@@ -469,44 +497,56 @@ namespace Duplicati.Server
return parts.ToArray();
}
public static Duplicati.Library.Interface.IBasicResults Run(IRunnerData data, bool fromQueue)
public static IBasicResults? Run(Connection databaseConnection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IProgressStateProviderService progressStateProviderService, IApplicationSettings applicationSettings, IQueuedTask data, bool fromQueue)
{
if (data is IRunnerData runnerData)
return RunInternal(databaseConnection, eventPollNotify, notificationUpdateService, progressStateProviderService, applicationSettings, runnerData, fromQueue);
throw new ArgumentException("Invalid task type", nameof(data));
}
private static IBasicResults? RunInternal(Connection databaseConnection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IProgressStateProviderService progressStateProviderService, IApplicationSettings applicationSettings, IRunnerData data, bool fromQueue)
{
data.TaskStarted = DateTime.Now;
if (data is CustomRunnerTask task)
{
try
{
var sink = new MessageSink(task.TaskID, null);
FIXMEGlobal.GenerateProgressState = sink.Copy;
FIXMEGlobal.StatusEventNotifyer.SignalNewEvent();
progressStateProviderService.GenerateProgressState = sink.Copy;
eventPollNotify.SignalNewEvent();
task.Run(sink);
}
catch (Exception ex)
{
FIXMEGlobal.DataConnection.LogError(string.Empty, "Failed while executing custom task", ex);
databaseConnection.LogError(string.Empty, "Failed while executing custom task", ex);
}
finally
{
data.TaskFinished = DateTime.Now;
}
return null;
}
var backup = data.Backup;
if (backup.Metadata == null)
{
backup.Metadata = new Dictionary<string, string>();
}
if (backup == null)
throw new ArgumentNullException(nameof(backup));
Duplicati.Library.Utility.TempFolder tempfolder = null;
backup.Metadata ??= new Dictionary<string, string>();
TempFolder? tempfolder = null;
try
{
var sink = new MessageSink(data.TaskID, backup.ID);
if (fromQueue)
{
FIXMEGlobal.GenerateProgressState = () => sink.Copy();
FIXMEGlobal.StatusEventNotifyer.SignalNewEvent();
progressStateProviderService.GenerateProgressState = () => sink.Copy();
eventPollNotify.SignalNewEvent();
}
var options = ApplyOptions(backup, GetCommonOptions());
var options = ApplyOptions(databaseConnection, backup, GetCommonOptions(databaseConnection));
if (data.ExtraOptions != null)
foreach (var k in data.ExtraOptions)
options[k.Key] = k.Value;
@@ -514,14 +554,14 @@ namespace Duplicati.Server
// Pack in the system or task config for easy restore
if (data.Operation == DuplicatiOperation.Backup && options.ContainsKey("store-task-config"))
{
tempfolder = StoreTaskConfigAndGetTempFolder(data, options);
tempfolder = StoreTaskConfigAndGetTempFolder(databaseConnection, data, options);
}
// Attach a log scope that tags all messages to relay the TaskID and BackupID
using (Library.Logging.Log.StartScope(log =>
{
log[LogWriteHandler.LOG_EXTRA_TASKID] = data.TaskID.ToString();
log[LogWriteHandler.LOG_EXTRA_BACKUPID] = data.BackupID;
log[ILogWriteHandler.LiveLogEntry.LOG_EXTRA_TASKID] = data.TaskID.ToString();
log[ILogWriteHandler.LiveLogEntry.LOG_EXTRA_BACKUPID] = data.BackupID;
}))
using (tempfolder)
@@ -542,22 +582,23 @@ namespace Duplicati.Server
catch { }
((RunnerData)data).Controller = controller;
data.UpdateThrottleSpeed(FIXMEGlobal.DataConnection.ApplicationSettings.UploadSpeedLimit, FIXMEGlobal.DataConnection.ApplicationSettings.DownloadSpeedLimit);
var appSettings = databaseConnection.ApplicationSettings;
data.UpdateThrottleSpeeds(appSettings.UploadSpeedLimit, appSettings.DownloadSpeedLimit);
// Pass on the provider, will be replaced if configured in the backup
controller.SetSecretProvider(FIXMEGlobal.SecretProvider);
controller.SetSecretProvider(applicationSettings.SecretProvider);
if (backup.Metadata.ContainsKey("LastCompactFinished"))
controller.LastCompact = Library.Utility.Utility.DeserializeDateTime(backup.Metadata["LastCompactFinished"]);
controller.LastCompact = Utility.DeserializeDateTime(backup.Metadata["LastCompactFinished"]);
if (backup.Metadata.ContainsKey("LastVacuumFinished"))
controller.LastVacuum = Library.Utility.Utility.DeserializeDateTime(backup.Metadata["LastVacuumFinished"]);
controller.LastVacuum = Utility.DeserializeDateTime(backup.Metadata["LastVacuumFinished"]);
switch (data.Operation)
{
case DuplicatiOperation.Backup:
{
var filter = ApplyFilter(backup, GetCommonFilter());
var filter = ApplyFilter(backup, GetCommonFilter(databaseConnection));
var sources =
(from n in backup.Sources
let p = SpecialFolders.ExpandEnvironmentVariables(n)
@@ -565,62 +606,62 @@ namespace Duplicati.Server
select p).ToArray();
var r = controller.Backup(sources, filter);
UpdateMetadata(backup, r);
UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r);
return r;
}
case DuplicatiOperation.List:
{
var r = controller.List(data.FilterStrings, null);
UpdateMetadata(backup, r);
UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r);
return r;
}
case DuplicatiOperation.Repair:
{
var r = controller.Repair(data.FilterStrings == null ? null : new Library.Utility.FilterExpression(data.FilterStrings));
UpdateMetadata(backup, r);
UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r);
return r;
}
case DuplicatiOperation.RepairUpdate:
{
var r = controller.UpdateDatabaseWithVersions();
UpdateMetadata(backup, r);
UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r);
return r;
}
case DuplicatiOperation.Remove:
{
var r = controller.Delete();
UpdateMetadata(backup, r);
UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r);
return r;
}
case DuplicatiOperation.Restore:
{
var r = controller.Restore(data.FilterStrings);
UpdateMetadata(backup, r);
UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r);
return r;
}
case DuplicatiOperation.Verify:
{
var r = controller.Test();
UpdateMetadata(backup, r);
UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r);
return r;
}
case DuplicatiOperation.Compact:
{
var r = controller.Compact();
UpdateMetadata(backup, r);
UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r);
return r;
}
case DuplicatiOperation.CreateReport:
{
using (var tf = new Duplicati.Library.Utility.TempFile())
using (var tf = new Library.Utility.TempFile())
{
var r = controller.CreateLogDatabase(tf);
var tempid = FIXMEGlobal.DataConnection.RegisterTempFile("create-bug-report", r.TargetPath, DateTime.Now.AddDays(3));
var tempid = databaseConnection.RegisterTempFile("create-bug-report", r.TargetPath, DateTime.Now.AddDays(3));
if (string.Equals(tf, r.TargetPath, Library.Utility.Utility.ClientFilenameStringComparison))
if (string.Equals(tf, r.TargetPath, Utility.ClientFilenameStringComparison))
tf.Protected = true;
FIXMEGlobal.DataConnection.RegisterNotification(
databaseConnection.RegisterNotification(
NotificationType.Information,
"Bugreport ready",
"Bugreport is ready for download",
@@ -640,38 +681,40 @@ namespace Duplicati.Server
case DuplicatiOperation.ListRemote:
{
var r = controller.ListRemote();
UpdateMetadata(backup, r);
UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r);
return r;
}
case DuplicatiOperation.Delete:
{
if (Library.Utility.Utility.ParseBoolOption(data.ExtraOptions.AsReadOnly(), "delete-remote-files"))
controller.DeleteAllRemoteFiles();
if (Library.Utility.Utility.ParseBoolOption(data.ExtraOptions.AsReadOnly(), "delete-local-db"))
if (data.ExtraOptions != null)
{
string dbpath;
options.TryGetValue("dbpath", out dbpath);
if (Utility.ParseBoolOption(data.ExtraOptions.AsReadOnly(), "delete-remote-files"))
controller.DeleteAllRemoteFiles();
if (!string.IsNullOrWhiteSpace(dbpath) && System.IO.File.Exists(dbpath))
System.IO.File.Delete(dbpath);
if (Utility.ParseBoolOption(data.ExtraOptions.AsReadOnly(), "delete-local-db"))
{
options.TryGetValue("dbpath", out var dbpath);
if (!string.IsNullOrWhiteSpace(dbpath) && System.IO.File.Exists(dbpath))
System.IO.File.Delete(dbpath);
}
}
FIXMEGlobal.DataConnection.DeleteBackup(backup);
FIXMEGlobal.Scheduler.Reschedule();
databaseConnection.DeleteBackup(backup);
(data as RunnerData)?.AfterTaskFinished?.Invoke(data);
return null;
}
case DuplicatiOperation.Vacuum:
{
var r = controller.Vacuum();
UpdateMetadata(backup, r);
UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r);
return r;
}
case DuplicatiOperation.ListFilesets:
{
var r = controller.ListFilesets();
UpdateMetadata(backup, r);
UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r);
return r;
}
case DuplicatiOperation.ListFolderContents:
@@ -696,8 +739,9 @@ namespace Duplicati.Server
}
catch (Exception ex)
{
FIXMEGlobal.DataConnection.LogError(data.Backup.ID, string.Format("Failed while executing {0} \"{1}\" (id: {2})", data.Operation, data.Backup.Name, data.Backup.ID), ex);
UpdateMetadataError(data.Backup, ex);
databaseConnection.LogError(data.Backup?.ID, string.Format("Failed while executing {0} \"{1}\" (id: {2})", data.Operation, data.Backup?.Name, data.Backup?.ID), ex);
if (data.Backup != null)
UpdateMetadataError(databaseConnection, notificationUpdateService, data.Backup, ex);
Library.UsageReporter.Reporter.Report(ex);
if (!fromQueue)
@@ -707,29 +751,33 @@ namespace Duplicati.Server
}
finally
{
((RunnerData)data).Controller = null;
data.SetController(null);
data.TaskFinished = DateTime.Now;
}
}
private static Duplicati.Library.Utility.TempFolder StoreTaskConfigAndGetTempFolder(IRunnerData data, Dictionary<string, string> options)
private static TempFolder? StoreTaskConfigAndGetTempFolder(Connection databaseConnection, IRunnerData data, Dictionary<string, string?> options)
{
if (data.Backup == null)
throw new ArgumentNullException(nameof(data.Backup));
var all_tasks = string.Equals(options["store-task-config"], "all", StringComparison.OrdinalIgnoreCase) || string.Equals(options["store-task-config"], "*", StringComparison.OrdinalIgnoreCase);
var this_task = Duplicati.Library.Utility.Utility.ParseBool(options["store-task-config"], false);
var this_task = Utility.ParseBool(options["store-task-config"], false);
options.Remove("store-task-config");
Duplicati.Library.Utility.TempFolder tempfolder = null;
TempFolder? tempfolder = null;
if (all_tasks || this_task)
{
tempfolder = new Duplicati.Library.Utility.TempFolder();
tempfolder = new TempFolder();
var temppath = System.IO.Path.Combine(tempfolder, "task-setup.json");
using (var tempfile = Duplicati.Library.Utility.TempFile.WrapExistingFile(temppath))
using (var tempfile = Library.Utility.TempFile.WrapExistingFile(temppath))
{
object taskdata = null;
object? taskdata = null;
if (all_tasks)
taskdata = FIXMEGlobal.DataConnection.Backups.Where(x => !x.IsTemporary).Select(x => FIXMEGlobal.DataConnection.PrepareBackupForExport(FIXMEGlobal.DataConnection.GetBackup(x.ID)));
taskdata = databaseConnection.Backups.Where(x => !x.IsTemporary).Select(x => databaseConnection.PrepareBackupForExport(databaseConnection.GetBackup(x.ID)!));
else
taskdata = new[] { FIXMEGlobal.DataConnection.PrepareBackupForExport(data.Backup) };
taskdata = new[] { databaseConnection.PrepareBackupForExport(data.Backup) };
using (var fs = System.IO.File.OpenWrite(tempfile))
using (var sw = new System.IO.StreamWriter(fs, System.Text.Encoding.UTF8))
@@ -737,7 +785,7 @@ namespace Duplicati.Server
tempfile.Protected = true;
options.TryGetValue("control-files", out string controlfiles);
options.TryGetValue("control-files", out var controlfiles);
if (string.IsNullOrWhiteSpace(controlfiles))
controlfiles = tempfile;
@@ -750,20 +798,20 @@ namespace Duplicati.Server
return tempfolder;
}
private static void UpdateMetadataError(Duplicati.Server.Serialization.Interface.IBackup backup, Exception ex)
private static void UpdateMetadataError(Connection databaseConnection, INotificationUpdateService notificationUpdateService, IBackup backup, Exception ex)
{
backup.Metadata["LastErrorDate"] = Library.Utility.Utility.SerializeDateTime(DateTime.UtcNow);
backup.Metadata["LastErrorDate"] = Utility.SerializeDateTime(DateTime.UtcNow);
backup.Metadata["LastErrorMessage"] = ex.Message;
if (!backup.IsTemporary)
FIXMEGlobal.DataConnection.SetMetadata(backup.Metadata, long.Parse(backup.ID), null);
databaseConnection.SetMetadata(backup.Metadata, long.Parse(backup.ID), null);
string messageid = null;
string? messageid = null;
if (ex is UserInformationException exception)
messageid = exception.HelpID;
FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId();
FIXMEGlobal.DataConnection.RegisterNotification(
notificationUpdateService.IncrementLastDataUpdateId();
databaseConnection.RegisterNotification(
NotificationType.Error,
backup.IsTemporary ?
"Error" : string.Format("Error while running {0}", backup.Name),
@@ -781,31 +829,31 @@ namespace Duplicati.Server
);
}
private static void UpdateMetadataLastCompact(Duplicati.Server.Serialization.Interface.IBackup backup, Duplicati.Library.Interface.ICompactResults r)
private static void UpdateMetadataLastCompact(IBackup backup, ICompactResults r)
{
if (r != null)
{
backup.Metadata["LastCompactDuration"] = r.Duration.ToString();
backup.Metadata["LastCompactStarted"] = Library.Utility.Utility.SerializeDateTime(r.BeginTime.ToUniversalTime());
backup.Metadata["LastCompactFinished"] = Library.Utility.Utility.SerializeDateTime(r.EndTime.ToUniversalTime());
backup.Metadata["LastCompactStarted"] = Utility.SerializeDateTime(r.BeginTime.ToUniversalTime());
backup.Metadata["LastCompactFinished"] = Utility.SerializeDateTime(r.EndTime.ToUniversalTime());
}
}
private static void UpdateMetadataLastVacuum(Duplicati.Server.Serialization.Interface.IBackup backup, Duplicati.Library.Interface.IVacuumResults r)
private static void UpdateMetadataLastVacuum(IBackup backup, IVacuumResults r)
{
if (r != null)
{
backup.Metadata["LastVacuumDuration"] = r.Duration.ToString();
backup.Metadata["LastVacuumStarted"] = Library.Utility.Utility.SerializeDateTime(r.BeginTime.ToUniversalTime());
backup.Metadata["LastVacuumFinished"] = Library.Utility.Utility.SerializeDateTime(r.EndTime.ToUniversalTime());
backup.Metadata["LastVacuumStarted"] = Utility.SerializeDateTime(r.BeginTime.ToUniversalTime());
backup.Metadata["LastVacuumFinished"] = Utility.SerializeDateTime(r.EndTime.ToUniversalTime());
}
}
private static void UpdateMetadata(Duplicati.Server.Serialization.Interface.IBackup backup, Duplicati.Library.Interface.IParsedBackendStatistics r)
private static void UpdateMetadataStatistics(IBackup backup, IParsedBackendStatistics r)
{
if (r != null)
{
backup.Metadata["LastBackupDate"] = Library.Utility.Utility.SerializeDateTime(r.LastBackupDate.ToUniversalTime());
backup.Metadata["LastBackupDate"] = Utility.SerializeDateTime(r.LastBackupDate.ToUniversalTime());
backup.Metadata["BackupListCount"] = r.BackupListCount.ToString();
backup.Metadata["TotalQuotaSpace"] = r.TotalQuotaSpace.ToString();
backup.Metadata["FreeQuotaSpace"] = r.FreeQuotaSpace.ToString();
@@ -814,28 +862,28 @@ namespace Duplicati.Server
backup.Metadata["TargetFilesSize"] = r.KnownFileSize.ToString();
backup.Metadata["TargetFilesCount"] = r.KnownFileCount.ToString();
backup.Metadata["TargetFilesetsCount"] = r.KnownFilesets.ToString();
backup.Metadata["TargetSizeString"] = Duplicati.Library.Utility.Utility.FormatSizeString(r.KnownFileSize);
backup.Metadata["TargetSizeString"] = Utility.FormatSizeString(r.KnownFileSize);
}
}
private static void UpdateMetadata(Duplicati.Server.Serialization.Interface.IBackup backup, Duplicati.Library.Interface.IBasicResults result)
private static void UpdateMetadataBase(Connection databaseConnection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IBackup backup, IBasicResults result)
{
if (result is IRestoreResults r1)
{
backup.Metadata["LastRestoreDuration"] = r1.Duration.ToString();
backup.Metadata["LastRestoreStarted"] = Library.Utility.Utility.SerializeDateTime(result.BeginTime.ToUniversalTime());
backup.Metadata["LastRestoreFinished"] = Library.Utility.Utility.SerializeDateTime(result.EndTime.ToUniversalTime());
backup.Metadata["LastRestoreStarted"] = Utility.SerializeDateTime(result.BeginTime.ToUniversalTime());
backup.Metadata["LastRestoreFinished"] = Utility.SerializeDateTime(result.EndTime.ToUniversalTime());
}
if (result is IParsedBackendStatistics r2 && !result.Interrupted)
{
UpdateMetadata(backup, r2);
UpdateMetadataStatistics(backup, r2);
}
if (result is IBackendStatsticsReporter r3 && !result.Interrupted)
{
if (r3.BackendStatistics is IParsedBackendStatistics statistics)
UpdateMetadata(backup, statistics);
UpdateMetadataStatistics(backup, statistics);
}
if (result is ICompactResults r4 && !result.Interrupted)
@@ -858,8 +906,8 @@ namespace Duplicati.Server
backup.Metadata["SourceFilesSize"] = r.SizeOfExaminedFiles.ToString();
backup.Metadata["SourceFilesCount"] = r.ExaminedFiles.ToString();
backup.Metadata["SourceSizeString"] = Duplicati.Library.Utility.Utility.FormatSizeString(r.SizeOfExaminedFiles);
backup.Metadata["LastBackupStarted"] = Library.Utility.Utility.SerializeDateTime(r.BeginTime.ToUniversalTime());
backup.Metadata["LastBackupFinished"] = Library.Utility.Utility.SerializeDateTime(r.EndTime.ToUniversalTime());
backup.Metadata["LastBackupStarted"] = Utility.SerializeDateTime(r.BeginTime.ToUniversalTime());
backup.Metadata["LastBackupFinished"] = Utility.SerializeDateTime(r.EndTime.ToUniversalTime());
backup.Metadata["LastBackupDuration"] = r.Duration.ToString();
if (r.CompactResults != null)
@@ -889,7 +937,7 @@ namespace Duplicati.Server
titleType = "Warning";
}
FIXMEGlobal.DataConnection.RegisterNotification(
databaseConnection.RegisterNotification(
r.FilesWithError == 0 && !r.Errors.Any() ? NotificationType.Warning : NotificationType.Error,
backup.IsTemporary ? "Warning" : $"{titleType} while running {backup.Name}",
message,
@@ -935,7 +983,7 @@ namespace Duplicati.Server
else if (result.ParsedResult == ParsedResultType.Error && result.Errors.Count() == 1)
message = $"Error: {result.Errors.Single()}";
FIXMEGlobal.DataConnection.RegisterNotification(
databaseConnection.RegisterNotification(
type,
title,
message,
@@ -950,10 +998,10 @@ namespace Duplicati.Server
}
if (!backup.IsTemporary)
FIXMEGlobal.DataConnection.SetMetadata(backup.Metadata, long.Parse(backup.ID), null);
databaseConnection.SetMetadata(backup.Metadata, long.Parse(backup.ID), null);
FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId();
FIXMEGlobal.StatusEventNotifyer.SignalNewEvent();
notificationUpdateService.IncrementLastDataUpdateId();
eventPollNotify.SignalNewEvent();
}
private static bool TestIfOptionApplies()
@@ -962,23 +1010,20 @@ namespace Duplicati.Server
return true;
}
private static void DisableModule(string module, Dictionary<string, string> options)
private static void DisableModule(string module, Dictionary<string, string?> options)
{
string disabledModules;
string enabledModules;
if (options.TryGetValue("enable-module", out enabledModules))
if (options.TryGetValue("enable-module", out var enabledModules))
{
var emods = (enabledModules ?? "").Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
options["enable-module"] = string.Join(",", emods.Where(x => module.Equals(x, StringComparison.OrdinalIgnoreCase)));
}
options.TryGetValue("disable-module", out disabledModules);
options.TryGetValue("disable-module", out var disabledModules);
var mods = (disabledModules ?? "").Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
options["disable-module"] = string.Join(",", mods.Union(new string[] { module }).Distinct(StringComparer.OrdinalIgnoreCase));
}
internal static Dictionary<string, string> ApplyOptions(Duplicati.Server.Serialization.Interface.IBackup backup, Dictionary<string, string> options)
internal static Dictionary<string, string?> ApplyOptions(Connection databaseConnection, Serialization.Interface.IBackup backup, Dictionary<string, string?> options)
{
options["backup-name"] = backup.Name;
options["dbpath"] = backup.DBPath;
@@ -998,7 +1043,7 @@ namespace Duplicati.Server
DisableModule("console-password-input", options);
// Patch in additional report urls
var additionalReportUrl = FIXMEGlobal.DataConnection.ApplicationSettings.AdditionalReportUrl;
var additionalReportUrl = databaseConnection.ApplicationSettings.AdditionalReportUrl;
if (!string.IsNullOrWhiteSpace(additionalReportUrl))
{
options["send-http-json-urls"] = string.Join(";",
@@ -1012,7 +1057,7 @@ namespace Duplicati.Server
return options;
}
private static Library.Utility.IFilter ApplyFilter(Serialization.Interface.IBackup backup, Library.Utility.IFilter filter)
private static Library.Utility.IFilter? ApplyFilter(IBackup backup, Library.Utility.IFilter? filter)
{
var f2 = backup.Filters;
if (f2 != null && f2.Length > 0)
@@ -1024,26 +1069,26 @@ namespace Duplicati.Server
? SpecialFolders.ExpandEnvironmentVariablesRegexp(n.Expression)
: SpecialFolders.ExpandEnvironmentVariables(n.Expression)
orderby n.Order
select (Library.Utility.IFilter)(new Library.Utility.FilterExpression(exp, n.Include)))
.Aggregate((a, b) => Library.Utility.FilterExpression.Combine(a, b));
select (Library.Utility.IFilter)new FilterExpression(exp, n.Include))
.Aggregate((a, b) => FilterExpression.Combine(a, b));
filter = Library.Utility.FilterExpression.Combine(filter, nf);
filter = FilterExpression.Combine(filter, nf);
}
return filter;
}
public static Dictionary<string, string> GetCommonOptions()
public static Dictionary<string, string?> GetCommonOptions(Connection databaseConnection)
{
return
(from n in FIXMEGlobal.DataConnection.Settings
(from n in databaseConnection.Settings
where TestIfOptionApplies()
select n).ToDictionary(k => k.Name.StartsWith("--", StringComparison.Ordinal) ? k.Name.Substring(2) : k.Name, k => k.Value);
select n).ToDictionary(k => k.Name.StartsWith("--", StringComparison.Ordinal) ? k.Name.Substring(2) : k.Name, k => (string?)k.Value);
}
private static Duplicati.Library.Utility.IFilter GetCommonFilter()
private static Library.Utility.IFilter? GetCommonFilter(Connection databaseConnection)
{
var filters = FIXMEGlobal.DataConnection.Filters;
var filters = databaseConnection.Filters;
if (filters == null || filters.Length == 0)
return null;
@@ -1051,8 +1096,8 @@ namespace Duplicati.Server
(from n in filters
orderby n.Order
let exp = Environment.ExpandEnvironmentVariables(n.Expression)
select (Duplicati.Library.Utility.IFilter)(new Duplicati.Library.Utility.FilterExpression(exp, n.Include)))
.Aggregate((a, b) => Duplicati.Library.Utility.FilterExpression.Combine(a, b));
select (Library.Utility.IFilter)new FilterExpression(exp, n.Include))
.Aggregate((a, b) => FilterExpression.Combine(a, b));
}
}
}
Duplicati/Library/RestAPI/Scheduler.cs
+58 -78
View File
@@ -19,6 +19,8 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using Duplicati.Server.Serialization.Interface;
using System;
@@ -27,7 +29,9 @@ using System.Text;
using System.Linq;
using System.Threading;
using Duplicati.Library.Utility;
using Duplicati.Library.RestAPI;
using System.Threading.Tasks;
using Duplicati.WebserverCore.Abstractions;
using Duplicati.Server.Database;
// TODO: Rewrite this class.
// It should just signal what new backups to run, and not mix with the worker thread.
@@ -39,7 +43,7 @@ namespace Duplicati.Server
/// </summary>
public class Scheduler
{
private static readonly string LOGTAG = Duplicati.Library.Logging.Log.LogTagFromType<Scheduler>();
private static readonly string LOGTAG = Library.Logging.Log.LogTagFromType<Scheduler>();
/// <summary>
/// The thread that runs the scheduler
@@ -51,11 +55,6 @@ namespace Duplicati.Server
/// </summary>
private volatile bool m_terminate;
/// <summary>
/// The worker thread that is invoked to do work
/// </summary>
private WorkerThread<Runner.IRunnerData> m_worker;
/// <summary>
/// The wait event
/// </summary>
@@ -67,9 +66,13 @@ namespace Duplicati.Server
private readonly object m_lock = new object();
/// <summary>
/// An event that is raised when the schedule changes
/// The queue runner service
/// </summary>
public event EventHandler NewSchedule;
private readonly IQueueRunnerService m_queueRunnerService;
/// <summary>
/// The data connection
/// </summary>
private readonly Connection m_dataConnection;
/// <summary>
/// The currently scheduled items
@@ -79,45 +82,31 @@ namespace Duplicati.Server
/// <summary>
/// List of update tasks, used to set the timestamp on the schedule once completed
/// </summary>
private Dictionary<Server.Runner.IRunnerData, Tuple<ISchedule, DateTime, DateTime>> m_updateTasks;
private Dictionary<Runner.IRunnerData, Tuple<ISchedule, DateTime, DateTime>> m_updateTasks;
/// <summary>
/// Constructs a new scheduler
/// </summary>
public Scheduler()
/// <param name="connection">The data connection</param>
/// <param name="queueRunnerService">The queue runner service</param>
public Scheduler(Connection connection, IQueueRunnerService queueRunnerService)
{
}
/// <summary>
/// Initializes scheduler
/// </summary>
/// <param name="worker">The worker thread</param>
public void Init(WorkerThread<Runner.IRunnerData> worker)
{
m_worker = worker;
m_dataConnection = connection;
m_queueRunnerService = queueRunnerService;
m_thread = new Thread(new ThreadStart(Runner));
m_worker.CompletedWork += OnCompleted;
m_worker.StartingWork += OnStartingWork;
m_schedule = new KeyValuePair<DateTime, ISchedule>[0];
m_schedule = [];
m_terminate = false;
m_event = new AutoResetEvent(false);
m_updateTasks = new Dictionary<Server.Runner.IRunnerData, Tuple<ISchedule, DateTime, DateTime>>();
m_updateTasks = new Dictionary<Runner.IRunnerData, Tuple<ISchedule, DateTime, DateTime>>();
m_thread.IsBackground = true;
m_thread.Name = "TaskScheduler";
m_thread.Start();
}
public IList<Tuple<long, string>> GetSchedulerQueueIds()
{
return (from n in WorkerQueue
where n.Backup != null
select new Tuple<long, string>(n.TaskID, n.Backup.ID)).ToList();
}
public IList<Tuple<string, DateTime>> GetProposedSchedule()
{
return (
from n in FIXMEGlobal.Scheduler.Schedule
from n in this.Schedule
let backupid = (from t in n.Value.Tags
where t != null && t.StartsWith("ID=", StringComparison.Ordinal)
select t.Substring("ID=".Length)).FirstOrDefault()
@@ -147,14 +136,6 @@ namespace Duplicati.Server
}
}
/// <summary>
/// A snapshot copy of the current worker queue, that is items that are scheduled, but waiting for execution
/// </summary>
public List<Runner.IRunnerData> WorkerQueue
{
get { return m_worker?.CurrentTasks?.Where(t => t != null)?.ToList() ?? []; }
}
/// <summary>
/// Terminates the thread. Any items still in queue will be removed
/// </summary>
@@ -233,9 +214,9 @@ namespace Duplicati.Server
return res;
}
private void OnCompleted(WorkerThread<Runner.IRunnerData> worker, Runner.IRunnerData task)
private Task OnCompleted(Runner.IRunnerData task)
{
Tuple<ISchedule, DateTime, DateTime> t = null;
Tuple<ISchedule, DateTime, DateTime>? t = null;
lock (m_lock)
{
if (task != null && m_updateTasks.TryGetValue(task, out t))
@@ -246,26 +227,28 @@ namespace Duplicati.Server
{
t.Item1.Time = t.Item2;
t.Item1.LastRun = t.Item3;
FIXMEGlobal.DataConnection.AddOrUpdateSchedule(t.Item1);
m_dataConnection.AddOrUpdateSchedule(t.Item1);
}
return Task.CompletedTask;
}
private void OnStartingWork(WorkerThread<Runner.IRunnerData> worker, Runner.IRunnerData task)
private Task OnStartingWork(Runner.IRunnerData task)
{
if (task is null)
{
return;
}
return Task.CompletedTask;
lock (m_lock)
{
if (m_updateTasks.TryGetValue(task, out Tuple<ISchedule, DateTime, DateTime> scheduleInfo))
if (m_updateTasks.TryGetValue(task, out var scheduleInfo))
{
// Item2 is the scheduled start time (Time in the Schedule table).
// Item3 is the actual start time (LastRun in the Schedule table).
m_updateTasks[task] = Tuple.Create(scheduleInfo.Item1, scheduleInfo.Item2, DateTime.UtcNow);
}
}
return Task.CompletedTask;
}
/// <summary>
@@ -280,8 +263,8 @@ namespace Duplicati.Server
// to avoid frequent db lookups
//Determine schedule list
var timeZoneInfo = FIXMEGlobal.DataConnection.ApplicationSettings.Timezone;
var lst = FIXMEGlobal.DataConnection.Schedules;
var timeZoneInfo = m_dataConnection.ApplicationSettings.Timezone;
var lst = m_dataConnection.Schedules;
foreach (var sc in lst)
{
if (!string.IsNullOrEmpty(sc.Repeat))
@@ -315,7 +298,7 @@ namespace Duplicati.Server
}
catch (Exception ex)
{
FIXMEGlobal.DataConnection.LogError(sc.ID.ToString(), "Scheduler failed to find next date",
m_dataConnection.LogError(sc.ID.ToString(), "Scheduler failed to find next date",
ex);
}
@@ -324,37 +307,37 @@ namespace Duplicati.Server
{
var jobsToRun = new List<Server.Runner.IRunnerData>();
//TODO: Cache this to avoid frequent lookups
foreach (var id in FIXMEGlobal.DataConnection.GetBackupIDsForTags(sc.Tags).Distinct()
foreach (var id in m_dataConnection.GetBackupIDsForTags(sc.Tags).Distinct()
.Select(x => x.ToString()))
{
//See if it is already queued
var tmplst = from n in m_worker.CurrentTasks
where n.Operation == Duplicati.Server.Serialization.DuplicatiOperation.Backup
select n.Backup;
var tastTemp = m_worker.CurrentTask;
var tmplst = from n in m_queueRunnerService.GetCurrentTasks()
where n.Operation == Serialization.DuplicatiOperation.Backup
select n.BackupID;
var tastTemp = m_queueRunnerService.GetCurrentTask();
if (tastTemp != null && tastTemp.Operation ==
Duplicati.Server.Serialization.DuplicatiOperation.Backup)
tmplst = tmplst.Union(new[] { tastTemp.Backup });
Serialization.DuplicatiOperation.Backup)
tmplst = tmplst.Union(new[] { tastTemp.BackupID });
//If it is not already in queue, put it there
if (!tmplst.Any(x => x.ID == id))
if (!tmplst.Any(x => x == id))
{
var entry = FIXMEGlobal.DataConnection.GetBackup(id);
var entry = m_dataConnection.GetBackup(id);
if (entry != null)
{
Dictionary<string, string> options = Duplicati.Server.Runner.GetCommonOptions();
Duplicati.Server.Runner.ApplyOptions(entry, options);
if ((new Duplicati.Library.Main.Options(options)).DisableOnBattery &&
(Duplicati.Library.Utility.Power.PowerSupply.GetSource() ==
Duplicati.Library.Utility.Power.PowerSupply.Source.Battery))
var options = Server.Runner.GetCommonOptions(m_dataConnection);
Server.Runner.ApplyOptions(m_dataConnection, entry, options);
if (new Library.Main.Options(options).DisableOnBattery &&
(Library.Utility.Power.PowerSupply.GetSource() ==
Library.Utility.Power.PowerSupply.Source.Battery))
{
Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG,
Library.Logging.Log.WriteInformationMessage(LOGTAG,
"BackupDisabledOnBattery",
"Scheduled backup disabled while on battery power.");
}
else
{
Dictionary<string, string> taskOptions = null;
Dictionary<string, string?>? taskOptions = null;
try
{
var nextRun = GetNextValidTime(start,
@@ -362,15 +345,17 @@ namespace Duplicati.Server
Math.Max(DateTime.UtcNow.AddSeconds(1).Ticks, start.AddSeconds(1).Ticks),
DateTimeKind.Utc), sc.Repeat, sc.AllowedDays, timeZoneInfo);
taskOptions = new Dictionary<string, string>()
taskOptions = new Dictionary<string, string?>()
{ { "next-scheduled-run", Utility.SerializeDateTime(nextRun.ToUniversalTime()) } };
}
catch
{
}
jobsToRun.Add(Server.Runner.CreateTask(
Serialization.DuplicatiOperation.Backup, entry, taskOptions));
var job = Server.Runner.CreateTask(Serialization.DuplicatiOperation.Backup, entry, taskOptions);
job.OnStarting = () => OnStartingWork(job);
job.OnFinished = (_) => OnCompleted(job);
jobsToRun.Add(job);
}
}
}
@@ -386,12 +371,12 @@ namespace Duplicati.Server
}
catch (Exception ex)
{
FIXMEGlobal.DataConnection.LogError(sc.ID.ToString(),
m_dataConnection.LogError(sc.ID.ToString(),
"Scheduler failed to find next date", ex);
continue;
}
Server.Runner.IRunnerData lastJob = jobsToRun.LastOrDefault();
var lastJob = jobsToRun.LastOrDefault();
if (lastJob != null)
{
lock (m_lock)
@@ -403,7 +388,7 @@ namespace Duplicati.Server
}
foreach (var job in jobsToRun)
m_worker.AddTask(job);
m_queueRunnerService.AddTask(job);
if (start < DateTime.UtcNow)
{
@@ -428,11 +413,6 @@ namespace Duplicati.Server
foreach (var c in (from n in scheduled where !existing.ContainsKey(n.Key) select n.Key).ToArray())
scheduled.Remove(c);
//Raise event if needed
// TODO: This triggers a new data event and a reconnect with long-poll
if (NewSchedule != null)
NewSchedule(this, null);
int waittime = 0;
//Figure out a sensible amount of time to sleep the thread
Duplicati/Library/RestAPI/Serializable/ServerSettings.cs
+9 -26
View File
@@ -18,9 +18,8 @@
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Linq;
using Duplicati.Library.RestAPI;
using Duplicati.Server.Serialization.Interface;
namespace Duplicati.Server.Serializable
@@ -38,7 +37,7 @@ namespace Duplicati.Server.Serializable
/// <summary>
/// Constructor for backend interface
/// </summary>
public DynamicModule(Duplicati.Library.Interface.IBackend backend)
public DynamicModule(Library.Interface.IBackend backend)
{
this.Key = backend.ProtocolKey;
this.Description = backend.Description;
@@ -49,7 +48,7 @@ namespace Duplicati.Server.Serializable
/// <summary>
/// Constructor for compression module interface
/// </summary>
public DynamicModule(Duplicati.Library.Interface.ICompression module)
public DynamicModule(Library.Interface.ICompression module)
{
this.Key = module.FilenameExtension;
this.Description = module.Description;
@@ -60,7 +59,7 @@ namespace Duplicati.Server.Serializable
/// <summary>
/// Constructor for encryption module interface
/// </summary>
public DynamicModule(Duplicati.Library.Interface.IEncryption module)
public DynamicModule(Library.Interface.IEncryption module)
{
this.Key = module.FilenameExtension;
this.Description = module.Description;
@@ -71,7 +70,7 @@ namespace Duplicati.Server.Serializable
/// <summary>
/// Constructor for generic module interface
/// </summary>
public DynamicModule(Duplicati.Library.Interface.IGenericModule module)
public DynamicModule(Library.Interface.IGenericModule module)
{
this.Key = module.Key;
this.Description = module.Description;
@@ -82,7 +81,7 @@ namespace Duplicati.Server.Serializable
/// <summary>
/// Constructor for webmodule interface
/// </summary>
public DynamicModule(Duplicati.Library.Interface.IWebModule module)
public DynamicModule(Library.Interface.IWebModule module)
{
this.Key = module.Key;
this.Description = module.Description;
@@ -93,7 +92,7 @@ namespace Duplicati.Server.Serializable
/// <summary>
/// Constructor for sercretprovider interface
/// </summary>
public DynamicModule(Duplicati.Library.Interface.ISecretProvider module)
public DynamicModule(Library.Interface.ISecretProvider module)
{
this.Key = module.Key;
this.Description = module.Description;
@@ -116,13 +115,13 @@ namespace Duplicati.Server.Serializable
/// <summary>
/// The options supported by the module
/// </summary>
public Duplicati.Library.Interface.ICommandLineArgument[] Options { get; private set; }
public Library.Interface.ICommandLineArgument[] Options { get; private set; }
}
/// <summary>
/// Gets all supported options
/// </summary>
public static Duplicati.Library.Interface.ICommandLineArgument[] Options
public static Library.Interface.ICommandLineArgument[] Options
{
get
{
@@ -242,21 +241,5 @@ namespace Duplicati.Server.Serializable
.ToArray();
}
}
/// <summary>
/// The filters that are applied to all backups
/// </summary>
public static IFilter[] Filters
{
get { return FIXMEGlobal.DataConnection.Filters; }
}
/// <summary>
/// The settings applied to all backups by default
/// </summary>
public static ISetting[] Settings
{
get { return FIXMEGlobal.DataConnection.Settings; }
}
}
}
Duplicati/Library/Utility/Duplicati.Library.Utility.csproj
+1 -1
View File
@@ -8,7 +8,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Duplicati.StreamUtil" Version="1.0.3" />
<PackageReference Include="Duplicati.StreamUtil" Version="1.0.5" />
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
<PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
<PackageReference Include="System.Management" Version="8.0.0" />
Duplicati/Library/Utility/Options/AuthIdOptionsHelper.cs
+1 -1
View File
@@ -44,7 +44,7 @@ public static class AuthIdOptionsHelper
/// <returns>The authentication options</returns>
public static CommandLineArgument[] GetOptions(string tokenurl, string? prefix = null) =>
[
new CommandLineArgument($"{prefix}{AuthIdOption}", CommandLineArgument.ArgumentType.String, Strings.AuthIdSettingsHelper.AuthidShort, Strings.AuthIdSettingsHelper.AuthidLong(tokenurl))
new CommandLineArgument($"{prefix}{AuthIdOption}", CommandLineArgument.ArgumentType.Password, Strings.AuthIdSettingsHelper.AuthidShort, Strings.AuthIdSettingsHelper.AuthidLong(tokenurl))
];
/// <summary>
Duplicati/Library/Utility/OverrideableStream.cs
+5
View File
@@ -37,6 +37,11 @@ namespace Duplicati.Library.Utility
/// </summary>
protected Stream m_basestream;
/// <summary>
/// The base stream that is wrapped
/// </summary>
public Stream BaseStream => m_basestream;
/// <summary>
/// Creates a new <see cref="OverrideableStream"/> instance
/// </summary>
Duplicati/Library/Utility/Utility.cs
+88 -1
View File
@@ -23,6 +23,7 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.IO;
using System.Linq;
@@ -1397,7 +1398,8 @@ namespace Duplicati.Library.Utility
/// <returns>The wrapped commandline element.</returns>
/// <param name="arg">The argument to wrap.</param>
/// <param name="allowEnvExpansion">A flag indicating if environment variables are allowed to be expanded</param>
public static string WrapCommandLineElement(string arg, bool allowEnvExpansion)
[return: NotNullIfNotNull("arg")]
public static string? WrapCommandLineElement(string? arg, bool allowEnvExpansion)
{
if (string.IsNullOrWhiteSpace(arg))
return arg;
@@ -1847,5 +1849,90 @@ namespace Duplicati.Library.Utility
: Timeout.Infinite
};
}
/// <summary>
/// The types of streams that are considered basic (i.e. not wrapped)
/// </summary>
private static readonly IReadOnlySet<Type> _basicStreamTypes = new HashSet<Type>
{
typeof(FileStream),
typeof(MemoryStream),
typeof(NetworkStream),
typeof(BufferedStream),
typeof(System.IO.Compression.DeflateStream),
typeof(System.IO.Compression.GZipStream),
typeof(System.IO.Compression.ZLibStream)
};
/// <summary>
/// Unwraps a stream from layers of wrapping streams
/// </summary>
/// <param name="stream">The stream to unwrap</param>
/// <returns>The unwrapped stream</returns>
public static Stream UnwrapThrottledStream(this Stream stream)
{
if (stream == null)
throw new ArgumentNullException(nameof(stream));
var previousStream = stream;
do
{
previousStream = stream;
while (stream is WrappingStream wrappingStream)
stream = wrappingStream.BaseStream;
while (stream is OverrideableStream overrideableStream)
stream = overrideableStream.BaseStream;
} while (stream != previousStream);
#if DEBUG
if (!_basicStreamTypes.Contains(stream.GetType()))
throw new InvalidOperationException($"The unwrapped stream is not a basic stream, but a {stream.GetType()}");
#endif
return stream;
}
/// <summary>
/// Calculates the hash of a throttled stream and returns the stream to read
/// </summary>
/// <param name="stream">The source stream
/// <param name="hashalgorithm">The hash algorithm to use</param>
/// <param name="cancelToken">The cancellation token to observe</param>
/// <returns>A tuple with the stream, the hash and a temporary file if used</returns>
public static async Task<(Stream content, string hash, TempFile? tmpfile)> CalculateThrottledStreamHash(Stream stream, string hashalgorithm, CancellationToken cancelToken)
{
TempFile? tmp = null;
string contentHash;
var measure = stream.UnwrapThrottledStream();
if (measure.CanSeek)
{
var p = measure.Position;
// Compute the hash
using (var hashalg = HashFactory.CreateHasher(hashalgorithm))
contentHash = ByteArrayAsHexString(hashalg.ComputeHash(measure));
measure.Position = p;
}
else
{
// No seeking possible, use a temp file
tmp = new TempFile();
await using (var sr = File.OpenWrite(tmp))
using (var hasher = HashFactory.CreateHasher(hashalgorithm))
await using (var hc = new HashCalculatingStream(measure, hasher))
{
await CopyStreamAsync(hc, sr, cancelToken).ConfigureAwait(false);
contentHash = hc.GetFinalHashString();
}
stream = File.OpenRead(tmp);
}
return (stream, contentHash, tmp);
}
}
}
Duplicati/Library/Utility/WorkerThread.cs
-401
View File
@@ -1,401 +0,0 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading;
// TODO: Delete this class.
// It is essentially a queue that is processed by a worker thread, and can be implemented using a BlockingCollection or similar.
namespace Duplicati.Library.Utility
{
/// <summary>
/// Class to encapsulate a thread that runs a list of queued operations
/// </summary>
/// <typeparam name="Tx">The type to operate on</typeparam>
public class WorkerThread<Tx> where Tx : class
{
/// <summary>
/// Locking object for shared data
/// </summary>
private readonly object m_lock = new object();
/// <summary>
/// The wait event
/// </summary>
private readonly AutoResetEvent m_event;
/// <summary>
/// The internal list of tasks to perform
/// </summary>
private Queue<Tx> m_tasks;
/// <summary>
/// A flag used to terminate the thread
/// </summary>
private volatile bool m_terminate;
/// <summary>
/// The coordinating thread
/// </summary>
private Thread m_thread;
/// <summary>
/// A value indicating if the coordinating thread is running
/// </summary>
private volatile bool m_active;
/// <summary>
/// The current task being processed
/// </summary>
private Tx m_currentTask;
/// <summary>
/// A callback that performs the actual work on the item
/// </summary>
private readonly Action<Tx> m_delegate;
/// <summary>
/// An event that is raised when the runner state changes
/// </summary>
public event Action<WorkerThread<Tx>, RunState> WorkerStateChanged;
/// <summary>
/// Event that occurs when a new operation is being processed
/// </summary>
public event Action<WorkerThread<Tx>, Tx> StartingWork;
/// <summary>
/// Event that occurs when an operation has completed
/// </summary>
public event Action<WorkerThread<Tx>, Tx> CompletedWork;
/// <summary>
/// Event that occurs when an error is detected
/// </summary>
public event Action<WorkerThread<Tx>, Tx, Exception> OnError;
/// <summary>
/// An event that occurs when a new task is added to the queue or an existing one is removed
/// </summary>
public event Action<WorkerThread<Tx>> WorkQueueChanged;
/// <summary>
/// The internal state
/// </summary>
private volatile RunState m_state;
/// <summary>
/// The states the scheduler can take
/// </summary>
public enum RunState
{
/// <summary>
/// The program is running as normal
/// </summary>
Run,
/// <summary>
/// The program is suspended by the user
/// </summary>
Paused
}
/// <summary>
/// Constructs a new WorkerThread
/// </summary>
/// <param name="item">The callback that performs the work</param>
public WorkerThread(Action<Tx> item, bool paused)
{
m_delegate = item;
m_event = new AutoResetEvent(paused);
m_terminate = false;
m_tasks = new Queue<Tx>();
m_state = paused ? WorkerThread<Tx>.RunState.Paused : WorkerThread<Tx>.RunState.Run;
m_thread = new Thread(new ThreadStart(Runner));
m_thread.IsBackground = true;
m_thread.Name = "WorkerThread<" + typeof(Tx).Name + ">";
m_thread.Start();
}
/// <summary>
/// Gets a copy of the current queue
/// </summary>
public List<Tx> CurrentTasks
{
get
{
lock (m_lock)
return new List<Tx>(m_tasks);
}
}
/// <summary>
/// Gets a value indicating if the worker is running
/// </summary>
public bool Active
{
get { return m_active; }
}
/// <summary>
/// Adds a task to the queue
/// </summary>
/// <param name="task">The task to add</param>
public void AddTask(Tx task)
{
lock (m_lock)
{
m_tasks.Enqueue(task);
m_event.Set();
}
if (WorkQueueChanged != null)
WorkQueueChanged(this);
}
/// <summary>
/// An overloaded AddTask method that allows a task to skip to the front of a queue
/// It does this by creating a new queue, adding the new task first, and then adding
/// all the old tasks to the new queue. It's cleaner to use a linked list,
/// but the performance difference is negligible on such a small queue.
/// </summary>
/// <param name="task">Task.</param>
/// <param name="skipQueue">If set to <c>true</c> skip queue.</param>
public void AddTask(Tx task, bool skipQueue)
{
if (!skipQueue)
{
// Fall back to default AddTask method
AddTask(task);
return;
}
lock (m_lock)
{
Queue<Tx> newQueue = new Queue<Tx>();
newQueue.Enqueue(task);
while (m_tasks.Count > 0)
{
Tx n = m_tasks.Dequeue();
newQueue.Enqueue(n);
}
m_tasks = newQueue;
m_event.Set();
}
if (WorkQueueChanged != null)
WorkQueueChanged(this);
}
/// <summary>
/// Removes a task from the queue, does not remove the task if it is currently running
/// </summary>
/// <param name="task">The task to remove</param>
public void RemoveTask(Tx task)
{
lock (m_lock)
{
Queue<Tx> tmp = new Queue<Tx>();
while (m_tasks.Count > 0)
{
Tx n = m_tasks.Dequeue();
if (n != task)
tmp.Enqueue(n);
}
m_tasks = tmp;
}
if (WorkQueueChanged != null)
WorkQueueChanged(this);
}
/// <summary>
/// This will clear the pending queue
/// <param name="abortThread">True if the current running thread should be aborted</param>
/// </summary>
public void ClearQueue(bool abortThread)
{
lock (m_lock)
m_tasks.Clear();
if (abortThread)
{
try
{
m_thread.Interrupt();
m_thread.Join(500);
}
catch
{
}
m_thread = new Thread(new ThreadStart(Runner));
m_thread.Start();
}
}
/// <summary>
/// Gets a reference to the currently executing task.
/// BEWARE: This is not protected by a mutex, DO NOT MODIFY IT!!!!
/// </summary>
public Tx CurrentTask
{
get
{
return m_currentTask;
}
}
/// <summary>
/// Terminates the thread. Any items still in queue will be removed
/// </summary>
/// <param name="wait">True if the call should block until the thread has exited, false otherwise</param>
public void Terminate(bool wait)
{
m_terminate = true;
m_event.Set();
if (wait)
m_thread.Join();
}
/// <summary>
/// This is the thread entry point
/// </summary>
private void Runner()
{
while (!m_terminate)
{
m_currentTask = null;
lock (m_lock)
if (m_state == WorkerThread<Tx>.RunState.Run && m_tasks.Count > 0)
m_currentTask = m_tasks.Dequeue();
if (m_currentTask == null && !m_terminate)
{
if (m_state == WorkerThread<Tx>.RunState.Run)
m_event.WaitOne(); //Sleep until signaled
else
{
if (WorkerStateChanged != null)
WorkerStateChanged(this, m_state);
//Sleep for brief periods, until signaled
while (!m_terminate && m_state != WorkerThread<Tx>.RunState.Run)
m_event.WaitOne(1000 * 60 * 5, false);
//If we were not terminated, we are now ready to run
if (!m_terminate)
{
m_state = WorkerThread<Tx>.RunState.Run;
if (WorkerStateChanged != null)
WorkerStateChanged(this, m_state);
}
}
}
if (m_terminate)
return;
if (m_currentTask == null && m_state == WorkerThread<Tx>.RunState.Run)
lock (m_lock)
if (m_tasks.Count > 0)
m_currentTask = m_tasks.Dequeue();
if (m_currentTask == null)
continue;
if (StartingWork != null)
StartingWork(this, m_currentTask);
try
{
m_active = true;
m_delegate(m_currentTask);
}
catch (Exception ex)
{
//TODO: Here where Thread.ResetAbort() was called we shall integrate the CancelationToken pattern.
if (OnError != null)
try { OnError(this, m_currentTask, ex); }
catch
{
// ignored
}
}
finally
{
//TODO: Here where Thread.ResetAbort() was called we shall integrate the CancelationToken pattern.
m_active = false;
}
var task = m_currentTask;
m_currentTask = null;
if (CompletedWork != null)
try { CompletedWork(this, task); }
catch (Exception ex)
{
try { OnError(this, task, ex); }
catch
{
// ignored
}
}
}
}
/// <summary>
/// Gets the current run state
/// </summary>
public RunState State { get { return m_state; } }
/// <summary>
/// Instructs Duplicati to run scheduled backups
/// </summary>
public void Resume()
{
m_state = RunState.Run;
m_event.Set();
}
/// <summary>
/// Instructs Duplicati to pause scheduled backups
/// </summary>
public void Pause()
{
m_state = RunState.Paused;
m_event.Set();
}
/// <summary>
/// Waits the specified number of milliseconds for the thread to terminate
/// </summary>
/// <param name="millisecondTimeout">The number of milliseconds to wait</param>
/// <returns>True if the thread is terminated, false if a timeout occured</returns>
public bool Join(int millisecondTimeout)
{
if (m_thread != null)
return m_thread.Join(millisecondTimeout);
return true;
}
}
}
Duplicati/Server/Duplicati.Server.Serialization/Interface/IQueuedTask.cs
+68 -13
View File
@@ -17,16 +17,71 @@
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
namespace Duplicati.Server.Serialization.Interface
{
public interface IQueuedTask
{
long TaskID { get; }
string BackupID { get; }
Duplicati.Server.Serialization.DuplicatiOperation Operation { get; }
}
}
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Threading.Tasks;
namespace Duplicati.Server.Serialization.Interface;
/// <summary>
/// Represents a queued task.
/// </summary>
public interface IQueuedTask
{
/// <summary>
/// The task ID.
/// </summary>
long TaskID { get; }
/// <summary>
/// The backup ID, if applicable.
/// </summary>
string? BackupID { get; }
/// <summary>
/// The operation type of the task.
/// </summary>
DuplicatiOperation Operation { get; }
/// <summary>
/// Callback to be executed when the task is starting.
/// </summary>
Func<Task>? OnStarting { get; set; }
/// <summary>
/// Callback to be executed when the task is finished.
/// If the task completes successfully, the exception parameter will be null.
/// </summary>
Func<Exception?, Task>? OnFinished { get; set; }
/// <summary>
/// Updates the throttle speeds for the task.
/// </summary>
/// <param name="uploadSpeed">The upload speed to set.</param>
/// <param name="downloadSpeed">The download speed to set.</param>
void UpdateThrottleSpeeds(string? uploadSpeed, string? downloadSpeed);
/// <summary>
/// The time when the task was starting to execute.
/// </summary>
DateTime? TaskStarted { get; set; }
/// <summary>
/// The time when the task was finished executing.
/// </summary>
DateTime? TaskFinished { get; set; }
/// <summary>
/// Stops the task.
/// </summary>
void Stop();
/// <summary>
/// Aborts the task.
/// </summary>
void Abort();
/// <summary>
/// Pauses the task.
/// </summary>
/// <param name="alsoTransfers">If true, also pauses transfers.</param>
void Pause(bool alsoTransfers);
/// <summary>
/// Resumes the task.
/// </summary>
void Resume();
}
Duplicati/Server/Program.cs
+135 -247
View File
@@ -32,11 +32,11 @@ using Duplicati.Library.Interface;
using Duplicati.Library.Logging;
using Duplicati.Library.Main;
using Duplicati.Library.Main.Database;
using Duplicati.Library.RestAPI;
using Duplicati.Library.Utility;
using Duplicati.Server.Database;
using Duplicati.WebserverCore;
using Duplicati.WebserverCore.Abstractions;
using Duplicati.WebserverCore.Services;
using Microsoft.Extensions.DependencyInjection;
namespace Duplicati.Server
@@ -84,61 +84,21 @@ namespace Duplicati.Server
/// </summary>
private static readonly string ENV_NAME_PREFIX = AutoUpdateSettings.AppName.ToUpperInvariant();
/// <summary>
/// Gets the folder where Duplicati data is stored
/// </summary>
public static string DataFolder { get => FIXMEGlobal.DataFolder; private set => FIXMEGlobal.DataFolder = value; }
/// <summary>
/// The single instance
/// </summary>
public static SingleInstance ApplicationInstance = null;
/// <summary>
/// This is the only access to the database
/// </summary>
public static Database.Connection DataConnection { get => FIXMEGlobal.DataConnection; set => FIXMEGlobal.DataConnection = value; }
/// <summary>
/// This is the lock to be used before manipulating the shared resources
/// </summary>
public static object MainLock { get => FIXMEGlobal.MainLock; }
/// <summary>
/// This is the scheduling thread
/// </summary>
public static IScheduler Scheduler { get => FIXMEGlobal.Scheduler; }
/// <summary>
/// List of completed task results
/// </summary>
public static List<KeyValuePair<long, Exception>> TaskResultCache { get => FIXMEGlobal.TaskResultCache; }
/// <summary>
/// The maximum number of completed task results to keep in memory
/// </summary>
private static readonly int MAX_TASK_RESULT_CACHE_SIZE = 100;
/// <summary>
/// The thread running the ping-pong handler
/// </summary>
private static System.Threading.Thread PingPongThread;
/// <summary>
/// The path to the file that contains the current database
/// </summary>
private static string DatabasePath;
private static Thread PingPongThread;
/// <summary>
/// The controller interface for pause/resume and throttle options
/// </summary>
public static LiveControls LiveControl { get => DuplicatiWebserver.Provider.GetRequiredService<LiveControls>(); }
/// <summary>
/// The application exit event
/// </summary>
public static System.Threading.ManualResetEvent ApplicationExitEvent { get => FIXMEGlobal.ApplicationExitEvent; set => FIXMEGlobal.ApplicationExitEvent = value; }
/// <summary>
/// Duplicati webserver instance
/// </summary>
@@ -152,68 +112,38 @@ namespace Duplicati.Server
DuplicatiWebserver.Stop().GetAwaiter().GetResult();
}
/// <summary>
/// The update poll thread.
/// </summary>
public static UpdatePollThread UpdatePoller => FIXMEGlobal.UpdatePoller;
/// <summary>
/// An event that is set once the server is ready to respond to requests
/// </summary>
public static readonly System.Threading.ManualResetEvent ServerStartedEvent = new System.Threading.ManualResetEvent(false);
public static readonly ManualResetEvent ServerStartedEvent = new ManualResetEvent(false);
/// <summary>
/// The status event signaler, used to control long polling of status updates
/// Timer for purging temp files and log data
/// </summary>
public static EventPollNotify StatusEventNotifyer => FIXMEGlobal.Provider.GetRequiredService<EventPollNotify>();
/// <summary>
/// A delegate method for creating a copy of the current progress state
/// </summary>
public static Func<Duplicati.Server.Serialization.Interface.IProgressEventData> GenerateProgressState { get => FIXMEGlobal.GenerateProgressState; set => FIXMEGlobal.GenerateProgressState = value; }
/// <summary>
/// The log redirect handler
/// </summary>
public static LogWriteHandler LogHandler { get => FIXMEGlobal.LogHandler; }
private static System.Threading.Timer PurgeTempFilesTimer = null;
public static int ServerPort
{
get
{
return DuplicatiWebserver.Port;
}
}
public static bool IsFirstRun
{
get { return DataConnection.ApplicationSettings.IsFirstRun; }
set { DataConnection.ApplicationSettings.IsFirstRun = value; }
}
public static bool ServerPortChanged
{
get { return DataConnection.ApplicationSettings.ServerPortChanged; }
set { DataConnection.ApplicationSettings.ServerPortChanged = value; }
}
static Program()
{
FIXMEGlobal.StartOrStopUsageReporter = Program.StartOrStopUsageReporter;
}
/// <summary>
/// The main entry point for the application.
/// </summary>
/// <param name="_args"> The command line arguments</param>
[STAThread]
public static int Main(string[] _args)
=> Main(null, _args);
/// <summary>
/// The main entry point for the application.
/// </summary>
/// <param name="applicationSettings"> The application settings</param>
/// <param name="_args"> The command line arguments</param>
[STAThread]
public static int Main(IApplicationSettings applicationSettings, string[] _args)
{
PreloadSettingsLoader.ConfigurePreloadSettings(ref _args, PackageHelper.NamedExecutable.Server, out var preloadDbSettings);
applicationSettings ??= new ApplicationSettings();
//If this executable is invoked directly, write to console, otherwise throw exceptions
var writeToConsoleOnException = FIXMEGlobal.Origin == "Server";
var writeToConsoleOnException = applicationSettings.Origin == "Server";
// Prepared for the future, where we might want to have a silent console mode
var silentConsole = false;
@@ -238,7 +168,7 @@ namespace Duplicati.Server
SystemContextSettings.StartSession();
ApplyEnvironmentVariables(commandlineOptions);
ApplySecretProvider(commandlineOptions, CancellationToken.None).Await();
ApplySecretProvider(applicationSettings, commandlineOptions, CancellationToken.None).Await();
var parameterFileOption = PARAMETERS_FILE_OPTION_EXTRAS.Prepend(PARAMETERS_FILE_OPTION)
.FirstOrDefault(x => commandlineOptions.ContainsKey(x));
@@ -251,62 +181,70 @@ namespace Duplicati.Server
return 100;
}
ConfigureLogging(commandlineOptions);
var logHandler = new LogWriteHandler();
using var logScope = ConfigureLogging(logHandler, commandlineOptions);
// Validate after logging is configured
CommandLineArgumentValidator.ValidateArguments(SupportedCommands, commandlineOptions, KnownDuplicateOptions, ValidationIgnoredOptions);
var crashed = false;
var terminated = false;
IQueueRunnerService queueRunner = null;
UpdatePollThread updatePollThread = null;
EventPollNotify eventPollNotify = null;
ISchedulerService scheduler = null;
try
{
DataConnection = GetDatabaseConnection(commandlineOptions, silentConsole);
var connection = GetDatabaseConnection(applicationSettings, commandlineOptions, silentConsole);
if (!DataConnection.ApplicationSettings.FixedInvalidBackupId)
DataConnection.FixInvalidBackupId();
if (!connection.ApplicationSettings.FixedInvalidBackupId)
connection.FixInvalidBackupId();
DataConnection.ApplicationSettings.UpgradePasswordToKBDF();
CreateApplicationInstance(writeToConsoleOnException);
connection.ApplicationSettings.UpgradePasswordToKBDF();
CreateApplicationInstance(applicationSettings.DataFolder, writeToConsoleOnException);
StartOrStopUsageReporter();
applicationSettings.StartOrStopUsageReporter = () => StartOrStopUsageReporter(connection);
applicationSettings.StartOrStopUsageReporter?.Invoke();
AdjustApplicationSettings(commandlineOptions);
ApplicationExitEvent = new System.Threading.ManualResetEvent(false);
AdjustApplicationSettings(connection, commandlineOptions);
UpdaterManager.OnError += obj =>
{
DataConnection.LogError(null, "Error in updater", obj);
connection.LogError(null, "Error in updater", obj);
};
DuplicatiWebserver = StartWebServer(commandlineOptions, DataConnection).Await();
DuplicatiWebserver = StartWebServer(commandlineOptions, connection, logHandler, applicationSettings).Await();
UpdatePoller.Init(Library.Utility.Utility.ParseBoolOption(commandlineOptions, DISABLE_UPDATE_CHECK_OPTION));
connection.SetServiceProvider(DuplicatiWebserver.Provider);
queueRunner = DuplicatiWebserver.Provider.GetRequiredService<IQueueRunnerService>();
updatePollThread = DuplicatiWebserver.Provider.GetRequiredService<UpdatePollThread>();
eventPollNotify = DuplicatiWebserver.Provider.GetRequiredService<EventPollNotify>();
scheduler = DuplicatiWebserver.Provider.GetRequiredService<ISchedulerService>();
SetPurgeTempFilesTimer(commandlineOptions);
updatePollThread.Init(Library.Utility.Utility.ParseBoolOption(commandlineOptions, DISABLE_UPDATE_CHECK_OPTION));
LiveControl.StateChanged = LiveControl_StateChanged;
SetPurgeTempFilesTimer(connection, commandlineOptions);
SetWorkerThread();
LiveControl.StateChanged = (e) => { LiveControl_StateChanged(queueRunner, connection, eventPollNotify, e); };
if (Library.Utility.Utility.ParseBoolOption(commandlineOptions, PING_PONG_KEEPALIVE_OPTION))
{
PingPongThread = new System.Threading.Thread(PingPongMethod) { IsBackground = true };
PingPongThread = new Thread(() => PingPongMethod(applicationSettings)) { IsBackground = true };
PingPongThread.Start();
}
DataConnection.ReWriteAllFieldsIfEncryptionChanged();
DataConnection.SetPreloadSettingsIfChanged(preloadDbSettings);
EmitWarningsForConfigurationIssues(commandlineOptions);
connection.ReWriteAllFieldsIfEncryptionChanged();
connection.SetPreloadSettingsIfChanged(preloadDbSettings);
EmitWarningsForConfigurationIssues(connection, applicationSettings, commandlineOptions);
Log.WriteInformationMessage(LOGTAG, "ServerStarted", Strings.Program.ServerStarted(DuplicatiWebserver.Interface, DuplicatiWebserver.Port));
logMessageToConsole(Strings.Program.ServerStarted(DuplicatiWebserver.Interface, DuplicatiWebserver.Port));
if (FIXMEGlobal.Origin == "Server" && DataConnection.ApplicationSettings.AutogeneratedPassphrase)
if (applicationSettings.Origin == "Server" && connection.ApplicationSettings.AutogeneratedPassphrase)
{
var signinToken = DuplicatiWebserver.Provider.GetRequiredService<IJWTTokenProvider>().CreateSigninToken("server-cli");
var hostname = (DataConnection.ApplicationSettings.AllowedHostnames ?? string.Empty).Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries).FirstOrDefault(x => x != "*") ?? "localhost";
var scheme = DataConnection.ApplicationSettings.UseHTTPS ? "https" : "http";
var hostname = (connection.ApplicationSettings.AllowedHostnames ?? string.Empty).Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries).FirstOrDefault(x => x != "*") ?? "localhost";
var scheme = connection.ApplicationSettings.UseHTTPS ? "https" : "http";
var url = $"{scheme}://{hostname}:{DuplicatiWebserver.Port}/signin.html?token={signinToken}";
Log.WriteWarningMessage(LOGTAG, "ServerStartedSignin", null, Strings.Program.ServerStartedSignin(url));
@@ -322,7 +260,7 @@ namespace Duplicati.Server
}
terminated = true;
ApplicationExitEvent.Set();
applicationSettings.ApplicationExitEvent.Set();
});
var stopCounter = 0;
@@ -344,7 +282,7 @@ namespace Duplicati.Server
};
ServerStartedEvent.Set();
ApplicationExitEvent.WaitOne();
applicationSettings.ApplicationExitEvent.WaitOne();
}
catch (SingleInstance.MultipleInstanceException mex)
{
@@ -374,19 +312,19 @@ namespace Duplicati.Server
Log.WriteInformationMessage(LOGTAG, "ServerStopping", Strings.Program.ServerStopping);
var steps = new Action[] {
() => StatusEventNotifyer.SignalNewEvent(),
() => { if (ShutdownModernWebserver != null) ShutdownModernWebserver(); },
() => UpdatePoller?.Terminate(),
() => Scheduler?.Terminate(true),
() => FIXMEGlobal.WorkThread?.Terminate(true),
() => eventPollNotify?.SignalNewEvent(),
() => ShutdownModernWebserver(),
() => updatePollThread?.Terminate(),
() => scheduler?.Terminate(true),
() => queueRunner?.Terminate(true),
() => ApplicationInstance?.Dispose(),
() => PurgeTempFilesTimer?.Dispose(),
() => Library.UsageReporter.Reporter.ShutDown(),
() => PingPongThread?.Interrupt(),
() =>
{
Library.Logging.Log.WriteInformationMessage(LOGTAG, "ServerStopped", Strings.Program.ServerStopped);
LogHandler?.Dispose();
Log.WriteInformationMessage(LOGTAG, "ServerStopped", Strings.Program.ServerStopped);
logHandler?.Dispose();
}
};
@@ -412,7 +350,7 @@ namespace Duplicati.Server
return 0;
}
private static async Task<DuplicatiWebserver> StartWebServer(IReadOnlyDictionary<string, string> options, Connection connection)
private static async Task<DuplicatiWebserver> StartWebServer(IReadOnlyDictionary<string, string> options, Connection connection, ILogWriteHandler logWriteHandler, IApplicationSettings applicationSettings)
{
var server = await WebServerLoader.TryRunServer(options, connection, async parsedOptions =>
{
@@ -421,7 +359,6 @@ namespace Duplicati.Server
parsedOptions.Port,
parsedOptions.Interface,
parsedOptions.Certificate,
parsedOptions.Servername,
parsedOptions.AllowedHostnames,
parsedOptions.DisableStaticFiles,
parsedOptions.SPAPaths,
@@ -429,7 +366,7 @@ namespace Duplicati.Server
parsedOptions.PreAuthTokens
);
var server = DuplicatiWebserver.CreateWebServer(mappedSettings, connection);
var server = DuplicatiWebserver.CreateWebServer(mappedSettings, connection, logWriteHandler, applicationSettings);
// Start the server, but catch any configuration issues
var task = server.Start();
@@ -440,57 +377,13 @@ namespace Duplicati.Server
return server;
}).ConfigureAwait(false);
FIXMEGlobal.Provider = server.Provider;
ServerPortChanged |= server.Port != DataConnection.ApplicationSettings.LastWebserverPort;
DataConnection.ApplicationSettings.LastWebserverPort = server.Port;
connection.ApplicationSettings.ServerPortChanged |= server.Port != connection.ApplicationSettings.LastWebserverPort;
connection.ApplicationSettings.LastWebserverPort = server.Port;
return server;
}
private static void SetWorkerThread()
{
FIXMEGlobal.WorkerThreadsManager.Spawn(x => { Runner.Run(x, true); });
FIXMEGlobal.WorkThread.StartingWork += (worker, task) => { SignalNewEvent(null, null); };
FIXMEGlobal.WorkThread.CompletedWork += (worker, task) => { SignalNewEvent(null, null); };
FIXMEGlobal.WorkThread.WorkQueueChanged += (worker) => { SignalNewEvent(null, null); };
FIXMEGlobal.Scheduler.SubScribeToNewSchedule(() => SignalNewEvent(null, null));
FIXMEGlobal.WorkThread.OnError += (worker, task, exception) =>
{
DataConnection.LogError(task?.BackupID, "Error in worker", exception);
};
var lastScheduleId = FIXMEGlobal.NotificationUpdateService.LastDataUpdateId;
StatusEventNotifyer.NewEvent += (sender, e) =>
{
if (lastScheduleId == FIXMEGlobal.NotificationUpdateService.LastDataUpdateId) return;
lastScheduleId = FIXMEGlobal.NotificationUpdateService.LastDataUpdateId;
Scheduler.Reschedule();
};
void RegisterTaskResult(long id, Exception ex)
{
lock (MainLock)
{
// If the new results says it crashed, we store that instead of success
if (TaskResultCache.Count > 0 && TaskResultCache.Last().Key == id)
{
if (ex != null && TaskResultCache.Last().Value == null)
TaskResultCache.RemoveAt(TaskResultCache.Count - 1);
else
return;
}
TaskResultCache.Add(new KeyValuePair<long, Exception>(id, ex));
while (TaskResultCache.Count > MAX_TASK_RESULT_CACHE_SIZE)
TaskResultCache.RemoveAt(0);
}
}
FIXMEGlobal.WorkThread.CompletedWork += (worker, task) => { RegisterTaskResult(task.TaskID, null); };
FIXMEGlobal.WorkThread.OnError += (worker, task, exception) => { RegisterTaskResult(task.TaskID, exception); };
}
private static void SetPurgeTempFilesTimer(Dictionary<string, string> commandlineOptions)
private static void SetPurgeTempFilesTimer(Connection connection, Dictionary<string, string> commandlineOptions)
{
var lastPurge = new DateTime(0);
@@ -505,7 +398,7 @@ namespace Duplicati.Server
lastPurge = DateTime.Now;
foreach (var e in DataConnection.GetTempFiles().Where((f) => f.Expires < DateTime.Now))
foreach (var e in connection.GetTempFiles().Where((f) => f.Expires < DateTime.Now))
{
try
{
@@ -514,16 +407,16 @@ namespace Duplicati.Server
}
catch (Exception ex)
{
DataConnection.LogError(null, $"Failed to delete temp file: {e.Path}", ex);
connection.LogError(null, $"Failed to delete temp file: {e.Path}", ex);
}
DataConnection.DeleteTempFile(e.ID);
connection.DeleteTempFile(e.ID);
}
Library.Utility.TempFile.RemoveOldApplicationTempFiles((path, ex) =>
{
DataConnection.LogError(null, $"Failed to delete temp file: {path}", ex);
connection.LogError(null, $"Failed to delete temp file: {path}", ex);
});
if (!commandlineOptions.TryGetValue("log-retention", out string pts))
@@ -531,11 +424,11 @@ namespace Duplicati.Server
pts = DEFAULT_LOG_RETENTION;
}
DataConnection.PurgeLogData(Library.Utility.Timeparser.ParseTimeInterval(pts, DateTime.Now, true));
connection.PurgeLogData(Library.Utility.Timeparser.ParseTimeInterval(pts, DateTime.Now, true));
}
catch (Exception ex)
{
DataConnection.LogError(null, "Failed during temp file cleanup", ex);
connection.LogError(null, "Failed during temp file cleanup", ex);
}
};
@@ -545,37 +438,37 @@ namespace Duplicati.Server
DEBUG_MODE ? TimeSpan.FromHours(1) : TimeSpan.FromDays(1));
}
private static void AdjustApplicationSettings(Dictionary<string, string> commandlineOptions)
private static void AdjustApplicationSettings(Connection connection, Dictionary<string, string> commandlineOptions)
{
// This clears the JWT config, and a new will be generated, invalidating all existing tokens
if (Library.Utility.Utility.ParseBoolOption(commandlineOptions, WebServerLoader.OPTION_WEBSERVICE_RESET_JWT_CONFIG))
{
DataConnection.ApplicationSettings.JWTConfig = null;
connection.ApplicationSettings.JWTConfig = null;
// Clean up stored tokens as they are now invalid
DataConnection.ExecuteWithCommand((con) => con.ExecuteNonQuery("DELETE FROM TokenFamily"));
connection.ExecuteWithCommand((con) => con.ExecuteNonQuery("DELETE FROM TokenFamily"));
}
if (Library.Utility.Utility.ParseBoolOption(commandlineOptions, WebServerLoader.OPTION_WEBSERVICE_ENABLE_FOREVER_TOKEN))
DataConnection.ApplicationSettings.EnableForeverTokens();
connection.ApplicationSettings.EnableForeverTokens();
if (commandlineOptions.ContainsKey(WebServerLoader.OPTION_WEBSERVICE_DISABLE_VISUAL_CAPTCHA))
DataConnection.ApplicationSettings.DisableVisualCaptcha = Library.Utility.Utility.ParseBool(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_DISABLE_VISUAL_CAPTCHA], true);
connection.ApplicationSettings.DisableVisualCaptcha = Library.Utility.Utility.ParseBool(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_DISABLE_VISUAL_CAPTCHA], true);
if (commandlineOptions.ContainsKey(WebServerLoader.OPTION_WEBSERVICE_DISABLE_SIGNIN_TOKENS))
DataConnection.ApplicationSettings.DisableSigninTokens = Library.Utility.Utility.ParseBool(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_DISABLE_SIGNIN_TOKENS], true);
connection.ApplicationSettings.DisableSigninTokens = Library.Utility.Utility.ParseBool(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_DISABLE_SIGNIN_TOKENS], true);
if (commandlineOptions.ContainsKey(WebServerLoader.OPTION_WEBSERVICE_PASSWORD))
DataConnection.ApplicationSettings.SetWebserverPassword(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_PASSWORD]);
connection.ApplicationSettings.SetWebserverPassword(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_PASSWORD]);
if (commandlineOptions.ContainsKey(WebServerLoader.OPTION_WEBSERVICE_ALLOWEDHOSTNAMES))
DataConnection.ApplicationSettings.SetAllowedHostnames(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_ALLOWEDHOSTNAMES]);
connection.ApplicationSettings.SetAllowedHostnames(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_ALLOWEDHOSTNAMES]);
else if (commandlineOptions.ContainsKey(WebServerLoader.OPTION_WEBSERVICE_ALLOWEDHOSTNAMES_ALT))
DataConnection.ApplicationSettings.SetAllowedHostnames(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_ALLOWEDHOSTNAMES_ALT]);
connection.ApplicationSettings.SetAllowedHostnames(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_ALLOWEDHOSTNAMES_ALT]);
if (commandlineOptions.ContainsKey(WebServerLoader.OPTION_WEBSERVICE_TIMEZONE) && !string.IsNullOrEmpty(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_TIMEZONE]))
try
{
DataConnection.ApplicationSettings.Timezone = TimeZoneHelper.FindTimeZone(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_TIMEZONE]);
connection.ApplicationSettings.Timezone = TimeZoneHelper.FindTimeZone(commandlineOptions[WebServerLoader.OPTION_WEBSERVICE_TIMEZONE]);
}
catch (Exception ex)
{
@@ -583,33 +476,33 @@ namespace Duplicati.Server
}
// The database has recorded a new version
if (DataConnection.ApplicationSettings.UpdatedVersion != null)
if (connection.ApplicationSettings.UpdatedVersion != null)
{
// Check if the running version is newer than the recorded version
if (UpdaterManager.TryParseVersion(DataConnection.ApplicationSettings.UpdatedVersion.Version) <= UpdaterManager.TryParseVersion(UpdaterManager.SelfVersion.Version))
if (UpdaterManager.TryParseVersion(connection.ApplicationSettings.UpdatedVersion.Version) <= UpdaterManager.TryParseVersion(UpdaterManager.SelfVersion.Version))
{
// Clean up lingering update notifications
var updateNotifications = DataConnection.GetNotifications().Where(x => x.Action == "update:new").ToList();
var updateNotifications = connection.GetNotifications().Where(x => x.Action == "update:new").ToList();
foreach (var n in updateNotifications)
DataConnection.DismissNotification(n.ID);
connection.DismissNotification(n.ID);
// Clear up the recorded version
DataConnection.ApplicationSettings.UpdatedVersion = null;
connection.ApplicationSettings.UpdatedVersion = null;
}
}
}
private static void EmitWarningsForConfigurationIssues(Dictionary<string, string> commandlineOptions)
private static void EmitWarningsForConfigurationIssues(Connection connection, IApplicationSettings applicationSettings, Dictionary<string, string> commandlineOptions)
{
if (DataConnection.ApplicationSettings.LastConfigIssueCheckVersion != UpdaterManager.SelfVersion.Version)
if (connection.ApplicationSettings.LastConfigIssueCheckVersion != UpdaterManager.SelfVersion.Version)
{
var updateNotifications = DataConnection.GetNotifications().Where(x => x.Action.StartsWith("config:issue:")).ToList();
var updateNotifications = connection.GetNotifications().Where(x => x.Action.StartsWith("config:issue:")).ToList();
foreach (var n in updateNotifications)
DataConnection.DismissNotification(n.ID);
connection.DismissNotification(n.ID);
if (!DataConnection.IsEncryptingFields && !Library.Utility.Utility.ParseBoolOption(commandlineOptions, DISABLE_DB_ENCRYPTION_OPTION))
if (!connection.IsEncryptingFields && !Library.Utility.Utility.ParseBoolOption(commandlineOptions, DISABLE_DB_ENCRYPTION_OPTION))
{
DataConnection.RegisterNotification(
connection.RegisterNotification(
Serialization.NotificationType.Warning,
"Unencrypted database",
"The database is not encrypted. This is a security risk and should be fixed as soon as possible.",
@@ -626,9 +519,9 @@ namespace Duplicati.Server
);
}
if (OperatingSystem.IsWindows() && DataFolder.StartsWith(Util.AppendDirSeparator(Environment.GetFolderPath(Environment.SpecialFolder.Windows)), StringComparison.OrdinalIgnoreCase))
if (OperatingSystem.IsWindows() && applicationSettings.DataFolder.StartsWith(Util.AppendDirSeparator(Environment.GetFolderPath(Environment.SpecialFolder.Windows)), StringComparison.OrdinalIgnoreCase))
{
DataConnection.RegisterNotification(
connection.RegisterNotification(
Serialization.NotificationType.Warning,
"Incorrect storage folder",
"The server configuraion is stored inside the Windows folder. Please move the configuration to a different location, or it may be deleted on Windows version upgrades.",
@@ -645,16 +538,16 @@ namespace Duplicati.Server
);
}
DataConnection.ApplicationSettings.LastConfigIssueCheckVersion = UpdaterManager.SelfVersion.Version;
connection.ApplicationSettings.LastConfigIssueCheckVersion = UpdaterManager.SelfVersion.Version;
}
}
private static void CreateApplicationInstance(bool writeToConsoleOnExceptionw)
private static void CreateApplicationInstance(string dataFolder, bool writeToConsoleOnExceptionw)
{
try
{
//This will also create DATAFOLDER if it does not exist
ApplicationInstance = new SingleInstance(DataFolder);
ApplicationInstance = new SingleInstance(dataFolder);
}
catch (Exception ex)
{
@@ -698,8 +591,8 @@ namespace Duplicati.Server
commandlineOptions[SETTINGS_ENCRYPTION_KEY_OPTION] = Environment.GetEnvironmentVariable(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME);
}
private static async Task ApplySecretProvider(Dictionary<string, string> commandlineOptions, CancellationToken cancellationToken)
=> FIXMEGlobal.SecretProvider = await SecretProviderHelper.ApplySecretProviderAsync([], [], commandlineOptions, TempFolder.SystemTempPath, FIXMEGlobal.SecretProvider, cancellationToken).ConfigureAwait(false);
private static async Task ApplySecretProvider(IApplicationSettings applicationSettings, Dictionary<string, string> commandlineOptions, CancellationToken cancellationToken)
=> applicationSettings.SecretProvider = await SecretProviderHelper.ApplySecretProviderAsync([], [], commandlineOptions, TempFolder.SystemTempPath, applicationSettings.SecretProvider, cancellationToken).ConfigureAwait(false);
private class ConsoleLogDestination(LogMessageType level) : ILogDestination
{
@@ -710,8 +603,9 @@ namespace Duplicati.Server
}
}
private static void ConfigureLogging(Dictionary<string, string> commandlineOptions)
private static IDisposable ConfigureLogging(ILogWriteHandler logWriteHandler, Dictionary<string, string> commandlineOptions)
{
IDisposable logScope;
//Log various information in the logfile
if (DEBUG_MODE && !commandlineOptions.ContainsKey(LOG_FILE_OPTION))
{
@@ -722,7 +616,7 @@ namespace Duplicati.Server
System.IO.File.Delete(commandlineOptions[LOG_FILE_OPTION]);
}
Log.StartScope(LogHandler, null);
logScope = Log.StartScope(logWriteHandler, null);
if (commandlineOptions.ContainsKey(LOG_FILE_OPTION))
{
@@ -730,7 +624,7 @@ namespace Duplicati.Server
if (commandlineOptions.ContainsKey(LOG_LEVEL_OPTION))
Enum.TryParse(commandlineOptions[LOG_LEVEL_OPTION], true, out loglevel);
LogHandler.SetServerFile(commandlineOptions[LOG_FILE_OPTION], loglevel);
logWriteHandler.SetServerFile(commandlineOptions[LOG_FILE_OPTION], loglevel);
}
if (Library.Utility.Utility.ParseBoolOption(commandlineOptions, LOG_CONSOLE_OPTION))
@@ -739,7 +633,7 @@ namespace Duplicati.Server
if (commandlineOptions.ContainsKey(LOG_LEVEL_OPTION))
Enum.TryParse(commandlineOptions[LOG_LEVEL_OPTION], true, out loglevel);
LogHandler.AppendLogDestination(new ConsoleLogDestination(loglevel), loglevel);
logWriteHandler.AppendLogDestination(new ConsoleLogDestination(loglevel), loglevel);
}
if (commandlineOptions.TryGetValue(WINDOWS_EVENTLOG_OPTION, out var source) && !string.IsNullOrEmpty(source))
@@ -765,16 +659,17 @@ namespace Duplicati.Server
if (WindowsEventLogSource.SourceExists(source))
{
var loglevel = Library.Logging.LogMessageType.Information;
var loglevel = LogMessageType.Information;
if (commandlineOptions.ContainsKey(WINDOWS_EVENTLOG_LEVEL_OPTION))
Enum.TryParse(commandlineOptions[WINDOWS_EVENTLOG_LEVEL_OPTION], true, out loglevel);
LogHandler.AppendLogDestination(new WindowsEventLogSource(source), loglevel);
logWriteHandler.AppendLogDestination(new WindowsEventLogSource(source), loglevel);
}
}
}
CrashlogHelper.OnUnobservedTaskException += (ex) => LogHandler.WriteMessage(new Library.Logging.LogEntry(ex.Message, null, Library.Logging.LogMessageType.Error, LOGTAG, "UnobservedTaskException", ex));
CrashlogHelper.OnUnobservedTaskException += (ex) => logWriteHandler.WriteMessage(new LogEntry(ex.Message, null, Library.Logging.LogMessageType.Error, LOGTAG, "UnobservedTaskException", ex));
return logScope;
}
private static int ShowHelp(bool writeToConsoleOnExceptionw)
@@ -792,15 +687,13 @@ namespace Duplicati.Server
throw new Exception("Server invoked with --help");
}
public static Connection GetDatabaseConnection(Dictionary<string, string> commandlineOptions, bool silentConsole)
public static Connection GetDatabaseConnection(IApplicationSettings applicationSettings, Dictionary<string, string> commandlineOptions, bool silentConsole)
{
DataFolder = DataFolderManager.GetDataFolder(DataFolderManager.AccessMode.ReadWritePermissionSet);
// Emit a warning if the database is stored in the Windows folder
if (Util.IsPathUnderWindowsFolder(DataFolder))
if (Util.IsPathUnderWindowsFolder(applicationSettings.DataFolder))
Log.WriteWarningMessage(LOGTAG, "DatabaseInWindowsFolder", null, "The database is stored in the Windows folder, this is not recommended as it will be deleted on Windows upgrades.");
CrashlogHelper.DefaultLogDir = DataFolder;
CrashlogHelper.DefaultLogDir = applicationSettings.DataFolder;
var sqliteVersion = new Version(Duplicati.Library.SQLiteHelper.SQLiteLoader.SQLiteVersion);
if (sqliteVersion < new Version(3, 6, 3))
@@ -814,15 +707,15 @@ namespace Duplicati.Server
try
{
DatabasePath = System.IO.Path.Combine(DataFolder, DataFolderManager.SERVER_DATABASE_FILENAME);
var databasePath = System.IO.Path.Combine(applicationSettings.DataFolder, DataFolderManager.SERVER_DATABASE_FILENAME);
if (!System.IO.Directory.Exists(System.IO.Path.GetDirectoryName(DatabasePath)))
System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(DatabasePath));
if (!System.IO.Directory.Exists(System.IO.Path.GetDirectoryName(databasePath)))
System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(databasePath));
// Attempt to open the database, removing any encryption present
Library.SQLiteHelper.SQLiteLoader.OpenDatabase(con, DatabasePath, Library.SQLiteHelper.SQLiteRC4Decrypter.GetEncryptionPassword(commandlineOptions));
Library.SQLiteHelper.SQLiteLoader.OpenDatabase(con, databasePath, Library.SQLiteHelper.SQLiteRC4Decrypter.GetEncryptionPassword(commandlineOptions));
Library.SQLiteHelper.DatabaseUpgrader.UpgradeDatabase(con, DatabasePath, typeof(Library.RestAPI.Database.DatabaseSchemaMarker));
Library.SQLiteHelper.DatabaseUpgrader.UpgradeDatabase(con, databasePath, typeof(Library.RestAPI.Database.DatabaseSchemaMarker));
}
catch (Exception ex)
{
@@ -839,7 +732,7 @@ namespace Duplicati.Server
var usingBlacklistedKey = encKey?.IsBlacklisted ?? false;
var hasValidEncryptionKey = encKey != null;
FIXMEGlobal.SettingsEncryptionKeyProvidedExternally = hasValidEncryptionKey;
applicationSettings.SettingsEncryptionKeyProvidedExternally = hasValidEncryptionKey;
if (requireDbEncryptionKey && !(hasValidEncryptionKey || disableDbEncryption))
throw new UserInformationException(Strings.Program.DatabaseEncryptionKeyRequired(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION), "RequireDbEncryptionKey");
@@ -856,7 +749,7 @@ namespace Duplicati.Server
if (hasEncryptedFields)
{
Library.Logging.Log.WriteWarningMessage(LOGTAG, "EncryptionKeyMissing", null, Strings.Program.EncryptionKeyMissing(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME));
Log.WriteWarningMessage(LOGTAG, "EncryptionKeyMissing", null, Strings.Program.EncryptionKeyMissing(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME));
if (!silentConsole)
Console.WriteLine(Strings.Program.EncryptionKeyMissing(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME));
}
@@ -871,61 +764,55 @@ namespace Duplicati.Server
if (!hasValidEncryptionKey && !disableDbEncryption)
{
disableDbEncryption = true;
Log.WriteWarningMessage(LOGTAG, "MissingEncryptionKey", null, Strings.Program.NoEncryptionKeySpecified(Library.Encryption.EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION));
Log.WriteWarningMessage(LOGTAG, "MissingEncryptionKey", null, Strings.Program.NoEncryptionKeySpecified(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION));
if (!silentConsole)
Console.WriteLine(Strings.Program.NoEncryptionKeySpecified(Library.Encryption.EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION));
Console.WriteLine(Strings.Program.NoEncryptionKeySpecified(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION));
}
if (usingBlacklistedKey && !disableDbEncryption)
{
disableDbEncryption = true;
Log.WriteErrorMessage(LOGTAG, "BlacklistedEncryptionKey", null, Strings.Program.BlacklistedEncryptionKey(Library.Encryption.EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION));
Log.WriteErrorMessage(LOGTAG, "BlacklistedEncryptionKey", null, Strings.Program.BlacklistedEncryptionKey(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION));
if (!silentConsole)
Console.WriteLine(Strings.Program.BlacklistedEncryptionKey(Library.Encryption.EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION));
Console.WriteLine(Strings.Program.BlacklistedEncryptionKey(EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME, DISABLE_DB_ENCRYPTION_OPTION));
}
return new Connection(con, disableDbEncryption, encKey);
return new Connection(con, disableDbEncryption, encKey, applicationSettings.DataFolder, applicationSettings.StartOrStopUsageReporter);
}
public static void StartOrStopUsageReporter()
private static void StartOrStopUsageReporter(Connection connection)
{
var disableUsageReporter =
string.Equals(DataConnection.ApplicationSettings.UsageReporterLevel, "none", StringComparison.OrdinalIgnoreCase)
string.Equals(connection.ApplicationSettings.UsageReporterLevel, "none", StringComparison.OrdinalIgnoreCase)
||
string.Equals(DataConnection.ApplicationSettings.UsageReporterLevel, "disabled", StringComparison.OrdinalIgnoreCase);
string.Equals(connection.ApplicationSettings.UsageReporterLevel, "disabled", StringComparison.OrdinalIgnoreCase);
if (!Enum.TryParse<Library.UsageReporter.ReportType>(DataConnection.ApplicationSettings.UsageReporterLevel, true, out var reportLevel))
if (!Enum.TryParse<Library.UsageReporter.ReportType>(connection.ApplicationSettings.UsageReporterLevel, true, out var reportLevel))
Library.UsageReporter.Reporter.SetReportLevel(null, disableUsageReporter);
else
Library.UsageReporter.Reporter.SetReportLevel(reportLevel, disableUsageReporter);
}
private static void SignalNewEvent(object sender, EventArgs e)
{
StatusEventNotifyer.SignalNewEvent();
}
/// <summary>
/// This event handler updates the trayicon menu with the current state of the runner.
/// </summary>
/// <exception cref="ArgumentOutOfRangeException"></exception>
private static void LiveControl_StateChanged(LiveControls.LiveControlEvent e)
private static void LiveControl_StateChanged(IQueueRunnerService queueRunnerService, Connection connection, EventPollNotify eventPollNotify, LiveControls.LiveControlEvent e)
{
var worker = FIXMEGlobal.WorkThread;
var appSettings = FIXMEGlobal.DataConnection.ApplicationSettings;
var appSettings = connection.ApplicationSettings;
switch (e.State)
{
case LiveControls.LiveControlState.Paused:
{
worker.Pause();
worker.CurrentTask?.Pause(e.TransfersPaused);
queueRunnerService.Pause();
queueRunnerService.GetCurrentTask()?.Pause(e.TransfersPaused);
appSettings.PausedUntil = e.WaitTimeExpiration;
break;
}
case LiveControls.LiveControlState.Running:
{
worker.Resume();
worker.CurrentTask?.Resume();
queueRunnerService.Resume();
queueRunnerService.GetCurrentTask()?.Resume();
appSettings.PausedUntil = null;
break;
}
@@ -934,14 +821,15 @@ namespace Duplicati.Server
break;
}
StatusEventNotifyer.SignalNewEvent();
eventPollNotify.SignalNewEvent();
}
/// <summary>
/// Simple method for tracking if the server has crashed
/// </summary>
private static void PingPongMethod()
/// <param name="applicationSettings">The application settings</param>
private static void PingPongMethod(IApplicationSettings applicationSettings)
{
var rd = new System.IO.StreamReader(Console.OpenStandardInput());
var wr = new System.IO.StreamWriter(Console.OpenStandardOutput());
@@ -952,7 +840,7 @@ namespace Duplicati.Server
{
// TODO: All calls to ApplicationExitEvent and TrayIcon->Quit
// should check if we are running something
ApplicationExitEvent.Set();
applicationSettings.ApplicationExitEvent.Set();
}
else
{
Duplicati/Server/WebServerLoader.cs
-3
View File
@@ -158,7 +158,6 @@ public static class WebServerLoader
/// <param name="Port">The listining port</param>
/// <param name="Interface">The listening interface</param>
/// <param name="Certificate">SSL certificate, if any</param>
/// <param name="Servername">The servername to report</param>
/// <param name="AllowedHostnames">The allowed hostnames</param>
/// <param name="DisableStaticFiles">If static files should be disabled</param>
/// <param name="SPAPaths">The paths to serve as SPAs</param>
@@ -168,7 +167,6 @@ public static class WebServerLoader
int Port,
System.Net.IPAddress Interface,
X509Certificate2Collection? Certificate,
string Servername,
IEnumerable<string> AllowedHostnames,
bool DisableStaticFiles,
IEnumerable<string> SPAPaths,
@@ -264,7 +262,6 @@ public static class WebServerLoader
-1,
listenInterface,
connection.ApplicationSettings.UseHTTPS ? connection.ApplicationSettings.ServerSSLCertificate : null,
string.Format("{0} v{1}", Library.AutoUpdater.AutoUpdateSettings.AppName, Library.AutoUpdater.UpdaterManager.SelfVersion.Version),
(connection.ApplicationSettings.AllowedHostnames ?? string.Empty).Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries),
Utility.ParseBoolOption(options, OPTION_WEBSERVICE_API_ONLY),
spaPathsString.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries),
Duplicati/UnitTest/ImportExportTests.cs
+4 -13
View File
@@ -29,7 +29,7 @@ using Duplicati.Server.Database;
using Duplicati.Server.Serializable;
using Duplicati.Server.Serialization;
using Duplicati.Server.Serialization.Interface;
using Microsoft.Extensions.DependencyInjection;
using Duplicati.WebserverCore.Services;
using NUnit.Framework;
using Backup = Duplicati.Server.Database.Backup;
@@ -107,10 +107,8 @@ namespace Duplicati.UnitTest
}
byte[] jsonByteArray;
using (Program.DataConnection = Program.GetDatabaseConnection(advancedOptions, true))
{
jsonByteArray = BackupImportExportHandler.ExportToJSON(Program.DataConnection, backup, null);
}
using (var con = Program.GetDatabaseConnection(new ApplicationSettings(), advancedOptions, true))
jsonByteArray = BackupImportExportHandler.ExportToJSON(con, backup, null);
// The username should not have the '%40' converted to '@' since the import code
// cannot handle it (see issue #3619).
@@ -130,13 +128,6 @@ namespace Duplicati.UnitTest
var metadata = new Dictionary<string, string> { { "SourceFilesCount", "1" } };
var advancedOptions = new Dictionary<string, string> { { "server-datafolder", this.serverDatafolder } };
// Mock the setup enough to get the import/export working
var serviceCollection = new ServiceCollection();
serviceCollection.AddSingleton<INotificationUpdateService, NotificationUpdateService>();
serviceCollection.AddSingleton(new EventPollNotify());
FIXMEGlobal.Provider = new DefaultServiceProviderFactory().CreateServiceProvider(serviceCollection);
FIXMEGlobal.DataFolder = this.serverDatafolder;
var dbpath = Path.Combine(this.serverDatafolder, Library.AutoUpdater.DataFolderManager.SERVER_DATABASE_FILENAME);
if (File.Exists(dbpath))
File.Delete(dbpath);
@@ -145,7 +136,7 @@ namespace Duplicati.UnitTest
SQLiteLoader.OpenDatabase(con, dbpath, null);
DatabaseUpgrader.UpgradeDatabase(con, dbpath, typeof(Library.RestAPI.Database.DatabaseSchemaMarker));
using (var connection = new Connection(con, true, null))
using (var connection = new Connection(con, true, null, this.serverDatafolder, null))
{
// Unencrypted file, don't import metadata.
string unencryptedWithoutMetadata = Path.Combine(this.serverDatafolder, Path.GetRandomFileName());
Duplicati/WebserverCore/Dto/GetTaskStateDto.cs
+3 -1
View File
@@ -25,6 +25,8 @@ namespace Duplicati.WebserverCore.Dto;
/// </summary>
/// <param name="Status">The status</param>
/// <param name="ID">The ID</param>
/// <param name="TaskStarted">When the task started</param>
/// <param name="TaskFinished">When the task finished</param>
/// <param name="ErrorMessage">The error message</param>
/// <param name="Exception">The exception</param>
public record GetTaskStateDto(string Status, long ID, string? ErrorMessage = null, string? Exception = null);
public record GetTaskStateDto(string Status, long ID, DateTime? TaskStarted, DateTime? TaskFinished, string? ErrorMessage = null, string? Exception = null);
Duplicati/WebserverCore/Dto/LogEntry.cs
+4 -4
View File
@@ -19,7 +19,7 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Logging;
using Duplicati.Server;
using Duplicati.WebserverCore.Abstractions;
namespace Duplicati.WebserverCore.Dto;
@@ -56,7 +56,7 @@ public sealed record LogEntry
/// <summary>
/// The message ID
/// </summary>
public required string ExceptionID { get; init; }
public required string? ExceptionID { get; init; }
/// <summary>
/// The message type
@@ -71,7 +71,7 @@ public sealed record LogEntry
/// <summary>
/// The backup ID, if any
/// </summary>
public required string BackupID { get; init; }
public required string? BackupID { get; init; }
/// <summary>
/// The task ID, if any
@@ -83,7 +83,7 @@ public sealed record LogEntry
/// </summary>
/// <param name="entry">The internal record</param>
/// <returns>The DTO record</returns>
public static LogEntry FromInternalEntry(LogWriteHandler.LogEntry entry)
public static LogEntry FromInternalEntry(ILogWriteHandler.LiveLogEntry entry)
{
return new LogEntry
{
Duplicati/WebserverCore/Dto/ServerStatusDto.cs
+2 -2
View File
@@ -30,7 +30,7 @@ public sealed record ServerStatusDto
/// <summary>
/// Gets or sets the active task.
/// </summary>
public required Tuple<long, string>? ActiveTask { get; init; }
public required Tuple<long, string?>? ActiveTask { get; init; }
/// <summary>
/// Gets or sets the state of the program.
@@ -40,7 +40,7 @@ public sealed record ServerStatusDto
/// <summary>
/// Gets the IDs of the tasks in the scheduler queue.
/// </summary>
public required IList<Tuple<long, string>> SchedulerQueueIds { get; init; } = [];
public required IList<Tuple<long, string?>> SchedulerQueueIds { get; init; } = [];
/// <summary>
/// Gets or sets the proposed schedule.
Duplicati/WebserverCore/DuplicatiWebserver.cs
+13 -6
View File
@@ -108,7 +108,6 @@ public class DuplicatiWebserver
/// <param name="Port">The listining port</param>
/// <param name="Interface">The listening interface</param>
/// <param name="Certificate">The certificate, if using SSL</param>
/// <param name="Servername">The servername to report</param>
/// <param name="AllowedHostnames">The allowed hostnames</param>
/// <param name="DisableStaticFiles">If static files should be disabled</param>
/// <param name="SPAPaths">The paths to serve as SPAs</param>
@@ -119,7 +118,6 @@ public class DuplicatiWebserver
int Port,
System.Net.IPAddress Interface,
X509Certificate2Collection? Certificate,
string Servername,
IEnumerable<string> AllowedHostnames,
bool DisableStaticFiles,
IEnumerable<string> SPAPaths,
@@ -132,8 +130,10 @@ public class DuplicatiWebserver
/// </summary>
/// <param name="settings">The settings for the server</param>
/// <param name="connection">The connection to the database</param>
/// <param name="logWriteHandler">The log write handler</param>
/// <param name="applicationSettings">The application settings</param>
/// <returns>The new webserver instance</returns>
public static DuplicatiWebserver CreateWebServer(InitSettings settings, Connection connection)
public static DuplicatiWebserver CreateWebServer(InitSettings settings, Connection connection, ILogWriteHandler logWriteHandler, IApplicationSettings applicationSettings)
{
var builder = WebApplication.CreateBuilder(new WebApplicationOptions()
{
@@ -279,7 +279,7 @@ public class DuplicatiWebserver
builder.Services.AddHealthChecks()
.AddCheck("Basic", () => HealthCheckResult.Healthy("Service is running"));
builder.Services.AddDuplicati(connection);
builder.Services.AddDuplicati(connection, logWriteHandler, applicationSettings);
// Prevent logs from spamming the console, but allow enabling for debugging
if (Environment.GetEnvironmentVariable("DUPLICATI_WEBSERVER_LOGGING") != "1")
@@ -331,6 +331,13 @@ public class DuplicatiWebserver
if (!settings.DisableStaticFiles)
app.UseDefaultStaticFiles(settings.WebRoot, settings.SPAPaths);
app.Use(async (context, next) =>
{
// Set up the log scope to capture any log messages for the livelog / server log
using (var scope = Library.Logging.Log.StartScope(logWriteHandler))
await next();
});
app.UseExceptionHandler(app =>
{
app.Run(async context =>
@@ -387,9 +394,9 @@ public class DuplicatiWebserver
var _ = Task.Run(() => app.Services.GetRequiredService<ISystemInfoProvider>().GetSystemInfo(null));
// Get a string description of the listen interface used
var listenInterface = settings.Interface == System.Net.IPAddress.Any
var listenInterface = settings.Interface == IPAddress.Any
? "*"
: settings.Interface == System.Net.IPAddress.Loopback
: settings.Interface == IPAddress.Loopback
? "localhost"
: settings.Interface.ToString();
Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs
+25 -28
View File
@@ -21,7 +21,7 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using Duplicati.Library.Interface;
using Duplicati.Library.RestAPI.Abstractions;
using Duplicati.Library.RestAPI;
using Duplicati.Server;
using Duplicati.Server.Database;
using Duplicati.Server.Serialization;
@@ -42,8 +42,8 @@ public class BackupGet : IEndpointV1
=> ExecuteGet(connection, GetBackup(connection, id)))
.RequireAuthorization();
group.MapGet("/backup/{id}/files", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery] string? filter, [FromQuery] string? time, [FromQuery(Name = "all-versions")] bool? allVersions, [FromQuery(Name = "prefix-only")] bool? prefixOnly, [FromQuery(Name = "folder-contents")] bool? folderContents)
=> ExecuteGetFiles(GetBackup(connection, id), filter, time, allVersions ?? false, prefixOnly ?? false, folderContents ?? false, new Dictionary<string, string>()))
group.MapGet("/backup/{id}/files", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id, [FromQuery] string? filter, [FromQuery] string? time, [FromQuery(Name = "all-versions")] bool? allVersions, [FromQuery(Name = "prefix-only")] bool? prefixOnly, [FromQuery(Name = "folder-contents")] bool? folderContents)
=> ExecuteGetFiles(queueRunnerService, GetBackup(connection, id), filter, time, allVersions ?? false, prefixOnly ?? false, folderContents ?? false, new Dictionary<string, string>()))
.RequireAuthorization();
group.MapGet("/backup/{id}/log", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery] long? offset, [FromQuery] long? pagesize)
@@ -54,16 +54,16 @@ public class BackupGet : IEndpointV1
=> ExecuteGetRemotelog(connection, GetBackup(connection, id), offset, pagesize ?? 100))
.RequireAuthorization();
group.MapGet("/backup/{id}/filesets", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery(Name = "include-metadata")] bool? includeMetadata, [FromQuery(Name = "from-remote-only")] bool? fromRemoteOnly)
=> ExecuteGetFilesets(GetBackup(connection, id), includeMetadata ?? false, fromRemoteOnly ?? false))
group.MapGet("/backup/{id}/filesets", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id, [FromQuery(Name = "include-metadata")] bool? includeMetadata, [FromQuery(Name = "from-remote-only")] bool? fromRemoteOnly)
=> ExecuteGetFilesets(queueRunnerService, GetBackup(connection, id), includeMetadata ?? false, fromRemoteOnly ?? false))
.RequireAuthorization();
group.MapGet("/backup/{id}/export-argsonly", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery(Name = "export-passwords")] bool? exportPasswords, [FromQuery] string? passphrase)
=> ExecuteGetExportArgsOnly(GetBackup(connection, id), exportPasswords ?? false))
=> ExecuteGetExportArgsOnly(connection, GetBackup(connection, id), exportPasswords ?? false))
.RequireAuthorization();
group.MapGet("/backup/{id}/export-cmdline", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery(Name = "export-passwords")] bool? exportPasswords, [FromQuery] string? passphrase)
=> ExecuteGetExportCmdline(GetBackup(connection, id), exportPasswords ?? false))
=> ExecuteGetExportCmdline(connection, GetBackup(connection, id), exportPasswords ?? false))
.RequireAuthorization();
group.MapGet("/backup/{id}/export", ([FromServices] Connection connection, [FromServices] IHttpContextAccessor httpContextAccessor, [FromServices] IJWTTokenProvider jWTTokenProvider, [FromRoute] string id, [FromQuery(Name = "export-passwords")] bool? exportPasswords, [FromQuery] string? passphrase, [FromQuery] string token, CancellationToken ct) =>
@@ -86,8 +86,8 @@ public class BackupGet : IEndpointV1
=> ExecuteGetIsdbUsedElsewhere(GetBackup(connection, id)))
.RequireAuthorization();
group.MapGet("/backup/{id}/isactive", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id)
=> ExecuteGetIsActive(workerThreadsManager, GetBackup(connection, id)))
group.MapGet("/backup/{id}/isactive", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id)
=> ExecuteGetIsActive(queueRunnerService, GetBackup(connection, id)))
.RequireAuthorization();
}
@@ -143,7 +143,7 @@ public class BackupGet : IEndpointV1
);
}
private static Dictionary<string, object> SearchFiles(IBackup backup, string? filter, string? timestring, bool allVersions, bool prefixOnly, bool folderContents, Dictionary<string, string> extraValues)
private static Dictionary<string, object> SearchFiles(IQueueRunnerService queueRunnerService, IBackup backup, string? filter, string? timestring, bool allVersions, bool prefixOnly, bool folderContents, Dictionary<string, string> extraValues)
{
if (string.IsNullOrWhiteSpace(timestring) && !allVersions)
throw new BadRequestException("Invalid or missing time");
@@ -152,7 +152,7 @@ public class BackupGet : IEndpointV1
if (!allVersions)
time = Library.Utility.Timeparser.ParseTimeInterval(timestring, DateTime.Now);
var r = Runner.Run(Runner.CreateListTask(backup, [filter], prefixOnly, allVersions, folderContents, time), false) as Duplicati.Library.Interface.IListResults;
var r = queueRunnerService.RunImmediately(Runner.CreateListTask(backup, filter == null ? null : [filter], prefixOnly, allVersions, folderContents, time)) as IListResults;
if (r == null)
throw new ServerErrorException("No result from list operation");
@@ -171,8 +171,8 @@ public class BackupGet : IEndpointV1
return result;
}
private static Dictionary<string, object> ExecuteGetFiles(IBackup bk, string? filter, string? timestring, bool allVersions, bool prefixOnly, bool folderContents, Dictionary<string, string> extraValues)
=> SearchFiles(bk, filter, timestring, allVersions, prefixOnly, folderContents, extraValues);
private static Dictionary<string, object> ExecuteGetFiles(IQueueRunnerService queueRunnerService, IBackup bk, string? filter, string? timestring, bool allVersions, bool prefixOnly, bool folderContents, Dictionary<string, string> extraValues)
=> SearchFiles(queueRunnerService, bk, filter, timestring, allVersions, prefixOnly, folderContents, extraValues);
private static List<Dictionary<string, object>> ExecuteGetLog(Connection connection, IBackup bk, long? offset, long pagesize)
{
@@ -203,9 +203,9 @@ public class BackupGet : IEndpointV1
}
}
private static IEnumerable<IListResultFileset> ExecuteGetFilesets(IBackup bk, bool includeMetadata, bool fromRemoteOnly)
private static IEnumerable<IListResultFileset> ExecuteGetFilesets(IQueueRunnerService queueRunnerService, IBackup bk, bool includeMetadata, bool fromRemoteOnly)
{
var extra = new Dictionary<string, string>
var extra = new Dictionary<string, string?>
{
["list-sets-only"] = "true"
};
@@ -219,7 +219,7 @@ public class BackupGet : IEndpointV1
try
{
var r = Runner.Run(Runner.CreateTask(DuplicatiOperation.List, bk, extra), false) as IListResults;
var r = queueRunnerService.RunImmediately(Runner.CreateTask(DuplicatiOperation.List, bk, extra)) as IListResults;
if (r == null)
throw new ServerErrorException("No result from list operation");
@@ -240,20 +240,20 @@ public class BackupGet : IEndpointV1
backup.SanitizeTargetUrl();
}
private static Dto.ExportCommandlineDto ExecuteGetExportCmdline(IBackup backup, bool exportPasswords)
private static Dto.ExportCommandlineDto ExecuteGetExportCmdline(Connection connection, IBackup backup, bool exportPasswords)
{
if (!exportPasswords)
RemovePasswords(backup);
return new Dto.ExportCommandlineDto(Runner.GetCommandLine(Runner.CreateTask(DuplicatiOperation.Backup, backup)));
return new Dto.ExportCommandlineDto(Runner.GetCommandLine(connection, Runner.CreateTask(DuplicatiOperation.Backup, backup)));
}
private static Dto.ExportArgsOnlyDto ExecuteGetExportArgsOnly(IBackup backup, bool exportPasswords)
private static Dto.ExportArgsOnlyDto ExecuteGetExportArgsOnly(Connection connection, IBackup backup, bool exportPasswords)
{
if (!exportPasswords)
RemovePasswords(backup);
var parts = Runner.GetCommandLineParts(Runner.CreateTask(DuplicatiOperation.Backup, backup));
var parts = Runner.GetCommandLineParts(connection, Runner.CreateTask(DuplicatiOperation.Backup, backup));
return new Dto.ExportArgsOnlyDto(
parts.First(),
parts.Skip(1).Where(x => !x.StartsWith("--", StringComparison.Ordinal)),
@@ -304,17 +304,14 @@ public class BackupGet : IEndpointV1
private static Dto.IsDbUsedElsewhereDto ExecuteGetIsdbUsedElsewhere(IBackup bk)
=> new Dto.IsDbUsedElsewhereDto(Library.Main.CLIDatabaseLocator.IsDatabasePathInUse(bk.DBPath));
private static Dto.IsBackupActiveDto ExecuteGetIsActive(IWorkerThreadsManager workerThreadsManager, IBackup bk)
private static Dto.IsBackupActiveDto ExecuteGetIsActive(IQueueRunnerService queueRunnerService, IBackup bk)
{
if (workerThreadsManager.WorkerThread == null)
throw new InvalidOperationException("Worker thread not available");
var t = workerThreadsManager.WorkerThread.CurrentTask;
var bt = t?.Backup;
if (bt != null && bk.ID == bt.ID)
var t = queueRunnerService.GetCurrentTask();
var bt = t?.BackupID;
if (bt != null && bk.ID == bt)
return new Dto.IsBackupActiveDto("OK", true);
if (workerThreadsManager.WorkerThread.CurrentTasks.Any(x => x?.Backup == null || x.Backup.ID == bk.ID))
if (queueRunnerService.GetCurrentTasks().Any(x => x?.BackupID == null || x.BackupID == bk.ID))
return new Dto.IsBackupActiveDto("OK", true);
return new Dto.IsBackupActiveDto("OK", false);
Duplicati/WebserverCore/Endpoints/V1/Backup/BackupPost.cs
+45 -46
View File
@@ -18,7 +18,6 @@
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.RestAPI.Abstractions;
using Duplicati.Server;
using Duplicati.Server.Database;
using Duplicati.Server.Serialization;
@@ -45,44 +44,44 @@ public class BackupPost : IEndpointV1
=> UpdateDatabasePath(connection, GetBackup(connection, id), input.path, false))
.RequireAuthorization();
group.MapPost("/backup/{id}/restore", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id, [FromBody] Dto.RestoreInputDto input)
=> ExecuteRestore(GetBackup(connection, id), workerThreadsManager, input))
group.MapPost("/backup/{id}/restore", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id, [FromBody] Dto.RestoreInputDto input)
=> ExecuteRestore(GetBackup(connection, id), queueRunnerService, input))
.RequireAuthorization();
group.MapPost("/backup/{id}/createreport", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id)
=> ExecuteCreateReport(GetBackup(connection, id), workerThreadsManager))
group.MapPost("/backup/{id}/createreport", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id)
=> ExecuteCreateReport(GetBackup(connection, id), queueRunnerService))
.RequireAuthorization();
group.MapPost("/backup/{id}/repair", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id, Dto.RepairInputDto? input)
=> ExecuteRepair(GetBackup(connection, id), workerThreadsManager, input))
group.MapPost("/backup/{id}/repair", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id, Dto.RepairInputDto? input)
=> ExecuteRepair(GetBackup(connection, id), queueRunnerService, input))
.RequireAuthorization();
group.MapPost("/backup/{id}/repairupdate", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id, Dto.RepairInputDto? input)
=> ExecuteRepairUpdate(GetBackup(connection, id), workerThreadsManager, input))
group.MapPost("/backup/{id}/repairupdate", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id, Dto.RepairInputDto? input)
=> ExecuteRepairUpdate(GetBackup(connection, id), queueRunnerService, input))
.RequireAuthorization();
group.MapPost("/backup/{id}/vacuum", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id)
=> ExecuteVacuum(GetBackup(connection, id), workerThreadsManager))
group.MapPost("/backup/{id}/vacuum", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id)
=> ExecuteVacuum(GetBackup(connection, id), queueRunnerService))
.RequireAuthorization();
group.MapPost("/backup/{id}/verify", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id)
=> ExecuteVerify(GetBackup(connection, id), workerThreadsManager))
group.MapPost("/backup/{id}/verify", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id)
=> ExecuteVerify(GetBackup(connection, id), queueRunnerService))
.RequireAuthorization();
group.MapPost("/backup/{id}/compact", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id)
=> ExecuteCompact(GetBackup(connection, id), workerThreadsManager))
group.MapPost("/backup/{id}/compact", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id)
=> ExecuteCompact(GetBackup(connection, id), queueRunnerService))
.RequireAuthorization();
group.MapPost("/backup/{id}/start", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id)
=> ExecuteRunBackup(GetBackup(connection, id), workerThreadsManager))
group.MapPost("/backup/{id}/start", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id)
=> ExecuteRunBackup(GetBackup(connection, id), queueRunnerService))
.RequireAuthorization();
group.MapPost("/backup/{id}/run", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id)
=> ExecuteRunBackup(GetBackup(connection, id), workerThreadsManager))
group.MapPost("/backup/{id}/run", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id)
=> ExecuteRunBackup(GetBackup(connection, id), queueRunnerService))
.RequireAuthorization();
group.MapPost("/backup/{id}/report-remote-size", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromRoute] string id)
=> ExecuteReportRemoteSize(GetBackup(connection, id), workerThreadsManager))
group.MapPost("/backup/{id}/report-remote-size", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id)
=> ExecuteReportRemoteSize(GetBackup(connection, id), queueRunnerService))
.RequireAuthorization();
group.MapPost("/backup/{id}/copytotemp", ([FromServices] Connection connection, [FromRoute] string id)
@@ -114,8 +113,8 @@ public class BackupPost : IEndpointV1
private static Dto.TaskStartedDto ExecuteRestore(IBackup backup, IWorkerThreadsManager workerThreadsManager, Dto.RestoreInputDto input)
=> new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateRestoreTask(
private static Dto.TaskStartedDto ExecuteRestore(IBackup backup, IQueueRunnerService queueRunnerService, Dto.RestoreInputDto input)
=> new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateRestoreTask(
backup,
input.paths ?? [],
Library.Utility.Timeparser.ParseTimeInterval(input.time, DateTime.Now),
@@ -125,31 +124,31 @@ public class BackupPost : IEndpointV1
input.skip_metadata ?? false,
string.IsNullOrWhiteSpace(input.passphrase) ? null : input.passphrase)));
private static Dto.TaskStartedDto ExecuteCreateReport(IBackup backup, IWorkerThreadsManager workerThreadsManager)
=> new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateTask(DuplicatiOperation.CreateReport, backup)));
private static Dto.TaskStartedDto ExecuteCreateReport(IBackup backup, IQueueRunnerService queueRunnerService)
=> new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateTask(DuplicatiOperation.CreateReport, backup)));
private static Dto.TaskStartedDto ExecuteReportRemoteSize(IBackup backup, IWorkerThreadsManager workerThreadsManager)
=> new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateTask(DuplicatiOperation.ListRemote, backup)));
private static Dto.TaskStartedDto ExecuteReportRemoteSize(IBackup backup, IQueueRunnerService queueRunnerService)
=> new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateTask(DuplicatiOperation.ListRemote, backup)));
private static Dto.TaskStartedDto ExecuteRepair(IBackup backup, IWorkerThreadsManager workerThreadsManager, Dto.RepairInputDto? input)
=> DoRepair(backup, false, workerThreadsManager, input);
private static Dto.TaskStartedDto ExecuteRepair(IBackup backup, IQueueRunnerService queueRunnerService, Dto.RepairInputDto? input)
=> DoRepair(backup, false, queueRunnerService, input);
private static Dto.TaskStartedDto ExecuteRepairUpdate(IBackup backup, IWorkerThreadsManager workerThreadsManager, Dto.RepairInputDto? input)
=> DoRepair(backup, true, workerThreadsManager, input);
private static Dto.TaskStartedDto ExecuteRepairUpdate(IBackup backup, IQueueRunnerService queueRunnerService, Dto.RepairInputDto? input)
=> DoRepair(backup, true, queueRunnerService, input);
private static Dto.TaskStartedDto ExecuteVacuum(IBackup backup, IWorkerThreadsManager workerThreadsManager)
=> new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateTask(DuplicatiOperation.Vacuum, backup)));
private static Dto.TaskStartedDto ExecuteVacuum(IBackup backup, IQueueRunnerService queueRunnerService)
=> new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateTask(DuplicatiOperation.Vacuum, backup)));
private static Dto.TaskStartedDto ExecuteVerify(IBackup backup, IWorkerThreadsManager workerThreadsManager)
=> new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateTask(DuplicatiOperation.Verify, backup)));
private static Dto.TaskStartedDto ExecuteVerify(IBackup backup, IQueueRunnerService queueRunnerService)
=> new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateTask(DuplicatiOperation.Verify, backup)));
private static Dto.TaskStartedDto ExecuteCompact(IBackup backup, IWorkerThreadsManager workerThreadsManager)
=> new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateTask(DuplicatiOperation.Compact, backup)));
private static Dto.TaskStartedDto ExecuteCompact(IBackup backup, IQueueRunnerService queueRunnerService)
=> new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateTask(DuplicatiOperation.Compact, backup)));
private static Dto.TaskStartedDto DoRepair(IBackup backup, bool repairUpdate, IWorkerThreadsManager workerThreadsManager, Dto.RepairInputDto? input)
private static Dto.TaskStartedDto DoRepair(IBackup backup, bool repairUpdate, IQueueRunnerService queueRunnerService, Dto.RepairInputDto? input)
{
// These are all props on the input object
var extra = new Dictionary<string, string>();
var extra = new Dictionary<string, string?>();
if (input != null)
{
if (input.only_paths.HasValue)
@@ -168,23 +167,23 @@ public class BackupPost : IEndpointV1
var filters = input?.paths ?? [];
return new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateTask(repairUpdate ? DuplicatiOperation.RepairUpdate : DuplicatiOperation.Repair, backup, extra, filters)));
return new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateTask(repairUpdate ? DuplicatiOperation.RepairUpdate : DuplicatiOperation.Repair, backup, extra, filters)));
}
private static Dto.TaskStartedDto ExecuteRunBackup(IBackup backup, IWorkerThreadsManager workerThreadsManager)
private static Dto.TaskStartedDto ExecuteRunBackup(IBackup backup, IQueueRunnerService queueRunnerService)
{
var t = workerThreadsManager.WorkerThread?.CurrentTask;
var bt = t?.Backup;
var t = queueRunnerService.GetCurrentTask();
var bt = t?.BackupID;
// Already running
if (bt != null && backup.ID == bt.ID)
if (bt != null && backup.ID == bt)
return new Dto.TaskStartedDto("OK", t!.TaskID);
t = workerThreadsManager.WorkerThread?.CurrentTasks.FirstOrDefault(x => x?.Backup != null && x.Backup.ID == backup.ID);
t = queueRunnerService.GetCurrentTasks().FirstOrDefault(x => x.BackupID == backup.ID);
if (t != null)
return new Dto.TaskStartedDto("OK", t.TaskID);
return new Dto.TaskStartedDto("OK", workerThreadsManager.AddTask(Runner.CreateTask(DuplicatiOperation.Backup, backup), true));
return new Dto.TaskStartedDto("OK", queueRunnerService.AddTask(Runner.CreateTask(DuplicatiOperation.Backup, backup), true));
}
private class WrappedBackup : Server.Database.Backup
Duplicati/WebserverCore/Endpoints/V1/Backup/BackupPutDelete.cs
+36 -49
View File
@@ -18,8 +18,6 @@
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.Text.Json;
using Duplicati.Library.RestAPI.Abstractions;
using Duplicati.Server;
using Duplicati.Server.Database;
using Duplicati.Server.Serialization;
@@ -38,9 +36,9 @@ public class BackupPutDelete : IEndpointV1
=> ExecutePut(GetBackup(connection, id), connection, input))
.RequireAuthorization();
group.MapDelete("/backup/{id}", ([FromServices] Connection connection, [FromServices] IWorkerThreadsManager workerThreadsManager, [FromServices] LiveControls liveControls, [FromServices] IHttpContextAccessor httpContextAccessor, [FromRoute] string id, [FromQuery(Name = "delete-remote-files")] bool? delete_remote_files, [FromQuery(Name = "delete-local-db")] bool? delete_local_db, [FromQuery] bool? force) =>
group.MapDelete("/backup/{id}", ([FromServices] Connection connection, [FromServices] ISchedulerService schedulerService, [FromServices] IQueueRunnerService queueRunnerService, [FromServices] LiveControls liveControls, [FromServices] IHttpContextAccessor httpContextAccessor, [FromRoute] string id, [FromQuery(Name = "delete-remote-files")] bool? delete_remote_files, [FromQuery(Name = "delete-local-db")] bool? delete_local_db, [FromQuery] bool? force) =>
{
var res = ExecuteDelete(GetBackup(connection, id), workerThreadsManager, liveControls, delete_remote_files ?? false, delete_local_db, force ?? false);
var res = ExecuteDelete(GetBackup(connection, id), schedulerService, queueRunnerService, liveControls, delete_remote_files ?? false, delete_local_db, force ?? false);
if (res.Status != "OK" && httpContextAccessor.HttpContext != null)
httpContextAccessor.HttpContext.Response.StatusCode = 500;
return res;
@@ -132,67 +130,56 @@ public class BackupPutDelete : IEndpointV1
}
}
private static Dto.DeleteBackupOutputDto ExecuteDelete(IBackup backup, IWorkerThreadsManager workerThreadsManager, LiveControls liveControls, bool delete_remote_files, bool? delete_local_db, bool force)
private static Dto.DeleteBackupOutputDto ExecuteDelete(IBackup backup, ISchedulerService schedulerService, IQueueRunnerService queueRunnerService, LiveControls liveControls, bool delete_remote_files, bool? delete_local_db, bool force)
{
if (workerThreadsManager.WorkerThread!.Active)
try
{
try
var nt = queueRunnerService.GetCurrentTask();
if (backup.ID == nt?.BackupID)
{
//TODO: It's not safe to access the values like this,
//because the runner thread might interfere
var nt = workerThreadsManager.WorkerThread.CurrentTask;
if (backup.Equals(nt?.Backup))
if (!force)
return new Dto.DeleteBackupOutputDto("failed", "backup-in-progress", nt?.TaskID);
bool hasPaused = liveControls.State != LiveControls.LiveControlState.Paused;
if (hasPaused)
liveControls.Pause(true);
nt.Abort();
for (int i = 0; i < 10; i++)
{
if (!force)
return new Dto.DeleteBackupOutputDto("failed", "backup-in-progress", nt?.TaskID);
bool hasPaused = liveControls.State != LiveControls.LiveControlState.Paused;
if (hasPaused)
liveControls.Pause(true);
nt.Abort();
for (int i = 0; i < 10; i++)
if (workerThreadsManager.WorkerThread.Active)
{
var t = workerThreadsManager.WorkerThread.CurrentTask;
if (backup.Equals(t == null ? null : t.Backup))
Thread.Sleep(1000);
else
break;
}
else
break;
if (workerThreadsManager.WorkerThread.Active)
{
var t = workerThreadsManager.WorkerThread.CurrentTask;
if (backup.Equals(t == null ? null : t.Backup))
{
if (hasPaused)
liveControls.Resume();
return new Dto.DeleteBackupOutputDto("failed", "backup-unstoppable", t?.TaskID);
}
}
var tt = queueRunnerService.GetCurrentTask();
if (backup.ID == tt?.BackupID)
Thread.Sleep(1000);
else
break;
}
var t = queueRunnerService.GetCurrentTask();
if (backup.ID == t?.BackupID)
{
if (hasPaused)
liveControls.Resume();
return new Dto.DeleteBackupOutputDto("failed", "backup-unstoppable", t?.TaskID);
}
}
catch (Exception ex)
{
return new Dto.DeleteBackupOutputDto("error", ex.Message, null);
if (hasPaused)
liveControls.Resume();
}
}
catch (Exception ex)
{
return new Dto.DeleteBackupOutputDto("error", ex.Message, null);
}
var extra = new Dictionary<string, string>();
var extra = new Dictionary<string, string?>();
if (delete_local_db.HasValue)
extra["delete-local-db"] = delete_local_db.Value.ToString();
if (delete_remote_files)
extra["delete-remote-files"] = "true";
return new Dto.DeleteBackupOutputDto("OK", null, workerThreadsManager.AddTask(Runner.CreateTask(DuplicatiOperation.Delete, backup, extra)));
return new Dto.DeleteBackupOutputDto("OK", null, queueRunnerService.AddTask(Runner.CreateDeleteTask(backup, extra, (_) => schedulerService.Reschedule())));
}
}
Duplicati/WebserverCore/Endpoints/V1/BackupDefaults.cs
+1 -1
View File
@@ -60,7 +60,7 @@ public class BackupDefaults : IEndpointV1
{
// Add built-in defaults
Newtonsoft.Json.Linq.JObject? n;
using (var s = new StreamReader(typeof(FIXMEGlobal).Assembly.GetManifestResourceStream(typeof(FIXMEGlobal), "newbackup.json")!))
using (var s = new StreamReader(typeof(Server.Runner).Assembly.GetManifestResourceStream(typeof(Server.Runner), "newbackup.json")!))
n = (Newtonsoft.Json.Linq.JObject)Newtonsoft.Json.JsonConvert.DeserializeObject(s.ReadToEnd())!;
o.Merge(n);
Duplicati/WebserverCore/Endpoints/V1/LogData.cs
+4 -5
View File
@@ -19,7 +19,6 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Crashlog;
using Duplicati.Library.RestAPI;
using Duplicati.Server.Database;
using Duplicati.WebserverCore.Abstractions;
using Duplicati.WebserverCore.Dto;
@@ -31,8 +30,8 @@ public class LogData : IEndpointV1
{
public static void Map(RouteGroupBuilder group)
{
group.MapGet("/logdata/poll", ([FromServices] Connection connection, [FromQuery] Library.Logging.LogMessageType level, [FromQuery] long id, [FromQuery] long? offset, [FromQuery] int? pagesize)
=> ExecuteLogPoll(level, id, offset ?? 0, pagesize ?? 100))
group.MapGet("/logdata/poll", ([FromServices] ILogWriteHandler logWriteHandler, [FromQuery] Library.Logging.LogMessageType level, [FromQuery] long id, [FromQuery] int? pagesize)
=> ExecuteLogPoll(logWriteHandler, level, id, pagesize ?? 100))
.RequireAuthorization();
group.MapGet("/logdata/log", ([FromServices] Connection connection, [FromQuery] long? offset, [FromQuery] int? pagesize)
@@ -44,10 +43,10 @@ public class LogData : IEndpointV1
.RequireAuthorization();
}
private static LogEntry[] ExecuteLogPoll(Library.Logging.LogMessageType level, long id, long offset, int pagesize)
private static LogEntry[] ExecuteLogPoll(ILogWriteHandler logWriteHandler, Library.Logging.LogMessageType level, long id, int pagesize)
{
pagesize = Math.Max(1, Math.Min(500, pagesize));
return FIXMEGlobal.LogHandler.AfterID(id, level, pagesize).Select(x => Dto.LogEntry.FromInternalEntry(x)).ToArray();
return logWriteHandler.AfterID(id, level, pagesize).Select(x => Dto.LogEntry.FromInternalEntry(x)).ToArray();
}
private static List<Dictionary<string, object>>? ExecuteGetLog(Connection connection, long? offset, long pagesize)
Duplicati/WebserverCore/Endpoints/V1/ProgressState.cs
+6 -5
View File
@@ -18,9 +18,9 @@
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.RestAPI;
using Duplicati.WebserverCore.Abstractions;
using Duplicati.WebserverCore.Exceptions;
using Microsoft.AspNetCore.Mvc;
namespace Duplicati.WebserverCore.Endpoints.V1;
@@ -28,15 +28,16 @@ public class ProgressState : IEndpointV1
{
public static void Map(RouteGroupBuilder group)
{
group.MapGet("/progressstate", Execute)
group.MapGet("/progressstate", ([FromServices] IProgressStateProviderService progressStateProviderService) => Execute(progressStateProviderService))
.RequireAuthorization();
}
private static Server.Serialization.Interface.IProgressEventData Execute()
private static Server.Serialization.Interface.IProgressEventData Execute(IProgressStateProviderService progressStateProviderService)
{
if (FIXMEGlobal.GenerateProgressState == null)
var pgState = progressStateProviderService.GenerateProgressState;
if (pgState == null)
throw new NotFoundException("No active backup");
return FIXMEGlobal.GenerateProgressState();
return pgState();
}
}
Duplicati/WebserverCore/Endpoints/V1/RemoteControl.cs
+4 -2
View File
@@ -18,7 +18,6 @@
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.RestAPI;
using Duplicati.WebserverCore.Abstractions;
using Duplicati.WebserverCore.Exceptions;
using Microsoft.AspNetCore.Mvc;
@@ -32,9 +31,12 @@ public class RemoteControl : IEndpointV1
group.MapGet("/remotecontrol/status", ([FromServices] IRemoteControllerRegistration registration, [FromServices] IRemoteController remoteController)
=> GetStatus(registration, remoteController))
.RequireAuthorization();
}
public static void MapConditional(RouteGroupBuilder group, string origin)
{
// Don't allow these in agent-mode
if (FIXMEGlobal.Origin == "Agent")
if (origin == "Agent")
return;
group.MapPost("/remotecontrol/enable", ([FromServices] IRemoteControllerRegistration registration, [FromServices] IRemoteController remoteController)
Duplicati/WebserverCore/Endpoints/V1/RemoteOperation.cs
+10 -10
View File
@@ -20,8 +20,8 @@
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Interface;
using Duplicati.Library.Main;
using Duplicati.Library.RestAPI;
using Duplicati.Server;
using Duplicati.Server.Database;
using Duplicati.WebserverCore.Abstractions;
using Duplicati.WebserverCore.Exceptions;
using Microsoft.AspNetCore.Mvc;
@@ -38,8 +38,8 @@ namespace Duplicati.WebserverCore.Endpoints.V1
=> ExecuteDbPath(input.path))
.RequireAuthorization();
group.MapPost("/remoteoperation/test", ([FromQuery] bool? autocreate, [FromBody] RemoteOperationInput input, CancellationToken cancelToken)
=> ExecuteTest(input.path, autocreate ?? false, cancelToken))
group.MapPost("/remoteoperation/test", ([FromServices] Connection connection, [FromServices] IApplicationSettings applicationSettings, [FromQuery] bool? autocreate, [FromBody] RemoteOperationInput input, CancellationToken cancelToken)
=> ExecuteTest(connection, applicationSettings, input.path, autocreate ?? false, cancelToken))
.RequireAuthorization();
group.MapPost("/remoteoperation/create", ([FromBody] RemoteOperationInput input, CancellationToken cancelToken)
@@ -53,11 +53,11 @@ namespace Duplicati.WebserverCore.Endpoints.V1
return new Dto.GetDbPathDto(!string.IsNullOrWhiteSpace(path), path);
}
private static Dictionary<string, string?> ParseUrlOptions(Library.Utility.Uri uri)
private static Dictionary<string, string?> ParseUrlOptions(Connection connection, Library.Utility.Uri uri)
{
var qp = uri.QueryParameters;
var opts = Runner.GetCommonOptions();
var opts = Runner.GetCommonOptions(connection);
foreach (var k in qp.Keys.Cast<string>())
opts[k] = qp[k];
@@ -87,13 +87,13 @@ namespace Duplicati.WebserverCore.Endpoints.V1
}
}
private static async Task<TupleDisposeWrapper> GetBackend(string url, CancellationToken cancelToken)
private static async Task<TupleDisposeWrapper> GetBackend(Connection connection, IApplicationSettings applicationSettings, string url, CancellationToken cancelToken)
{
var uri = new Library.Utility.Uri(url);
var opts = ParseUrlOptions(uri);
var opts = ParseUrlOptions(connection, uri);
var tmp = new[] { uri };
await SecretProviderHelper.ApplySecretProviderAsync([], tmp, opts, Library.Utility.TempFolder.SystemTempPath, FIXMEGlobal.SecretProvider, cancelToken);
await SecretProviderHelper.ApplySecretProviderAsync([], tmp, opts, Library.Utility.TempFolder.SystemTempPath, applicationSettings.SecretProvider, cancelToken);
url = tmp[0].ToString();
var modules = ConfigureModules(opts);
@@ -101,13 +101,13 @@ namespace Duplicati.WebserverCore.Endpoints.V1
return new TupleDisposeWrapper(backend, modules);
}
private static async Task ExecuteTest(string url, bool autoCreate, CancellationToken cancelToken)
private static async Task ExecuteTest(Connection connection, IApplicationSettings applicationSettings, string url, bool autoCreate, CancellationToken cancelToken)
{
TupleDisposeWrapper? wrapper = null;
try
{
wrapper = await GetBackend(url, cancelToken);
wrapper = await GetBackend(connection, applicationSettings, url, cancelToken);
using (var b = wrapper.Backend)
{
Duplicati/WebserverCore/Endpoints/V1/Tasks.cs
+23 -25
View File
@@ -34,57 +34,55 @@ public class Tasks : IEndpointV1
}
public static void Map(RouteGroupBuilder group)
{
group.MapGet("/tasks", Execute).RequireAuthorization();
group.MapGet("/task/{taskid}", ([FromRoute] long taskId) => ExecuteGet(taskId)).RequireAuthorization();
group.MapPost("/task/{taskid}/stop", ([FromRoute] long taskId) => ExecutePost(taskId, TaskStopState.Stop)).RequireAuthorization();
group.MapPost("/task/{taskid}/abort", ([FromRoute] long taskId) => ExecutePost(taskId, TaskStopState.Abort)).RequireAuthorization();
group.MapGet("/tasks", ([FromServices] IQueueRunnerService queueRunnerService) => Execute(queueRunnerService)).RequireAuthorization();
group.MapGet("/task/{taskid}", ([FromRoute] long taskId, [FromServices] IQueueRunnerService queueRunnerService) => ExecuteGet(queueRunnerService, taskId)).RequireAuthorization();
group.MapPost("/task/{taskid}/stop", ([FromRoute] long taskId, [FromServices] IQueueRunnerService queueRunnerService) => ExecutePost(queueRunnerService, taskId, TaskStopState.Stop)).RequireAuthorization();
group.MapPost("/task/{taskid}/abort", ([FromRoute] long taskId, [FromServices] IQueueRunnerService queueRunnerService) => ExecutePost(queueRunnerService, taskId, TaskStopState.Abort)).RequireAuthorization();
}
private static IEnumerable<Server.Runner.IRunnerData> Execute()
private static IEnumerable<Server.Runner.IRunnerData> Execute(IQueueRunnerService queueRunnerService)
{
var cur = FIXMEGlobal.WorkThread.CurrentTask;
var n = FIXMEGlobal.WorkThread.CurrentTasks;
var cur = queueRunnerService.GetCurrentTask();
var n = queueRunnerService.GetCurrentTasks();
if (cur != null)
n.Insert(0, cur);
return n;
return n.OfType<Server.Runner.IRunnerData>();
}
private static Dto.GetTaskStateDto ExecuteGet(long taskid)
private static Dto.GetTaskStateDto ExecuteGet(IQueueRunnerService queueRunnerService, long taskid)
{
var task = FIXMEGlobal.WorkThread.CurrentTask;
var tasks = FIXMEGlobal.WorkThread.CurrentTasks;
var task = queueRunnerService.GetCurrentTask();
var tasks = queueRunnerService.GetCurrentTasks();
if (task != null && task.TaskID == taskid)
return new Dto.GetTaskStateDto("Running", taskid);
return new Dto.GetTaskStateDto("Running", taskid, task.TaskStarted, task.TaskFinished);
if (tasks.FirstOrDefault(x => x.TaskID == taskid) == null)
{
KeyValuePair<long, Exception>[] matches;
lock (FIXMEGlobal.MainLock)
matches = FIXMEGlobal.TaskResultCache.Where(x => x.Key == taskid).ToArray();
if (matches.Length == 0)
var res = queueRunnerService.GetCachedTaskResults(taskid);
if (res == null)
throw new NotFoundException("No such task found");
return new Dto.GetTaskStateDto(
Status: matches[0].Value == null ? "Completed" : "Failed",
Status: res.Exception == null ? "Completed" : "Failed",
ID: taskid,
ErrorMessage: matches[0].Value == null ? null : matches[0].Value.Message,
Exception: matches[0].Value == null ? null : matches[0].Value.ToString()
TaskStarted: res?.TaskStarted,
TaskFinished: res?.TaskFinished,
ErrorMessage: res?.Exception?.Message,
Exception: res?.Exception?.ToString()
);
}
return new Dto.GetTaskStateDto("Waiting", taskid);
return new Dto.GetTaskStateDto("Waiting", taskid, null, null);
}
private static void ExecutePost(long taskid, TaskStopState stopState)
private static void ExecutePost(IQueueRunnerService queueRunnerService, long taskid, TaskStopState stopState)
{
var task = FIXMEGlobal.WorkThread.CurrentTask;
var tasks = FIXMEGlobal.WorkThread.CurrentTasks;
var task = queueRunnerService.GetCurrentTask();
var tasks = queueRunnerService.GetCurrentTasks();
if (task != null)
tasks.Insert(0, task);
Duplicati/WebserverCore/Endpoints/V1/Updates.cs
+7 -4
View File
@@ -18,8 +18,9 @@
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.RestAPI;
using Duplicati.Server;
using Duplicati.WebserverCore.Abstractions;
using Microsoft.AspNetCore.Mvc;
namespace Duplicati.WebserverCore.Endpoints.V1;
@@ -27,9 +28,11 @@ public class Updates : IEndpointV1
{
public static void Map(RouteGroupBuilder group)
{
group.MapPost("/updates/check", Execute).RequireAuthorization();
group.MapPost("/updates/check",
([FromServices] UpdatePollThread updatePollThread) => Execute(updatePollThread))
.RequireAuthorization();
}
private static void Execute()
=> FIXMEGlobal.UpdatePoller.CheckNow();
private static void Execute(UpdatePollThread updatePollThread)
=> updatePollThread.CheckNow();
}
Duplicati/WebserverCore/Endpoints/V1/WebModules.cs
+7 -5
View File
@@ -20,8 +20,8 @@
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Interface;
using Duplicati.Library.Main;
using Duplicati.Library.RestAPI;
using Duplicati.Server;
using Duplicati.Server.Database;
using Duplicati.WebserverCore.Abstractions;
using Duplicati.WebserverCore.Exceptions;
using Microsoft.AspNetCore.Mvc;
@@ -33,22 +33,24 @@ public record WebModules : IEndpointV1
public static void Map(RouteGroupBuilder group)
{
group.MapGet("/webmodules", ExecuteGet).RequireAuthorization();
group.MapPost("/webmodule/{modulekey}", ([FromRoute] string modulekey, [FromBody] Dictionary<string, string> options, CancellationToken cancellationToken) => ExecutePost(modulekey, options, cancellationToken)).RequireAuthorization();
group.MapPost("/webmodule/{modulekey}", ([FromServices] Connection connection, [FromServices] IApplicationSettings applicationSettings, [FromRoute] string modulekey, [FromBody] Dictionary<string, string> options, CancellationToken cancellationToken)
=> ExecutePost(connection, applicationSettings, modulekey, options, cancellationToken))
.RequireAuthorization();
}
private static IEnumerable<IWebModule> ExecuteGet()
=> Library.DynamicLoader.WebLoader.Modules;
private static async Task<Dto.WebModuleOutputDto> ExecutePost(string modulekey, Dictionary<string, string> inputOptions, CancellationToken cancellationToken)
private static async Task<Dto.WebModuleOutputDto> ExecutePost(Connection connection, IApplicationSettings applicationSettings, string modulekey, Dictionary<string, string> inputOptions, CancellationToken cancellationToken)
{
var m = Library.DynamicLoader.WebLoader.Modules.FirstOrDefault(x => x.Key.Equals(modulekey, StringComparison.OrdinalIgnoreCase))
?? throw new NotFoundException("No such module found");
var options = Runner.GetCommonOptions();
var options = Runner.GetCommonOptions(connection);
foreach (var k in inputOptions.Keys)
options[k] = inputOptions[k];
await SecretProviderHelper.ApplySecretProviderAsync([], [], options, Library.Utility.TempFolder.SystemTempPath, FIXMEGlobal.SecretProvider, cancellationToken);
await SecretProviderHelper.ApplySecretProviderAsync([], [], options, Library.Utility.TempFolder.SystemTempPath, applicationSettings.SecretProvider, cancellationToken);
return new Dto.WebModuleOutputDto(
Status: "OK",
Duplicati/WebserverCore/Endpoints/V2/BackupListing.cs
+19 -17
View File
@@ -20,11 +20,13 @@
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Interface;
using Duplicati.Library.RestAPI;
using Duplicati.Server;
using Duplicati.Server.Database;
using Duplicati.Server.Serialization.Interface;
using Duplicati.WebserverCore.Abstractions;
using Duplicati.WebserverCore.Exceptions;
using Duplicati.WebserverCore.Services;
using Microsoft.AspNetCore.Mvc;
namespace Duplicati.WebserverCore.Endpoints.V2.Backup;
@@ -33,33 +35,33 @@ public class BackupListing : IEndpointV2
{
public static void Map(RouteGroupBuilder group)
{
group.MapPost("/backup/list-filesets", ([FromServices] Connection connection, [FromBody] Dto.V2.ListFilesetsRequestDto input)
=> ExecuteGetFilesets(GetBackup(connection, input.BackupId)))
group.MapPost("/backup/list-filesets", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromBody] Dto.V2.ListFilesetsRequestDto input)
=> ExecuteGetFilesets(queueRunnerService, GetBackup(connection, input.BackupId)))
.RequireAuthorization();
group.MapPost("/backup/list-folder", ([FromServices] Connection connection, [FromBody] Dto.V2.ListFolderContentRequestDto input)
=> ExecuteListFolder(GetBackup(connection, input.BackupId), input))
group.MapPost("/backup/list-folder", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromBody] Dto.V2.ListFolderContentRequestDto input)
=> ExecuteListFolder(queueRunnerService, GetBackup(connection, input.BackupId), input))
.RequireAuthorization();
group.MapPost("/backup/list-versions", ([FromServices] Connection connection, [FromBody] Dto.V2.ListFileVersionsRequestDto input)
=> ExecuteListVersions(GetBackup(connection, input.BackupId), input))
group.MapPost("/backup/list-versions", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromBody] Dto.V2.ListFileVersionsRequestDto input)
=> ExecuteListVersions(queueRunnerService, GetBackup(connection, input.BackupId), input))
.RequireAuthorization();
group.MapPost("/backup/search", ([FromServices] Connection connection, [FromBody] Dto.V2.SearchEntriesRequestDto input)
=> ExecuteSearch(GetBackup(connection, input.BackupId), input))
group.MapPost("/backup/search", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromBody] Dto.V2.SearchEntriesRequestDto input)
=> ExecuteSearch(queueRunnerService, GetBackup(connection, input.BackupId), input))
.RequireAuthorization();
}
private static IBackup GetBackup(Connection connection, string id)
=> connection.GetBackup(id) ?? throw new NotFoundException("Backup not found");
private static Dto.V2.ListFolderContentResponseDto ExecuteListFolder(IBackup bk, Dto.V2.ListFolderContentRequestDto input)
private static Dto.V2.ListFolderContentResponseDto ExecuteListFolder(IQueueRunnerService queueRunnerService, IBackup bk, Dto.V2.ListFolderContentRequestDto input)
{
var time = string.IsNullOrWhiteSpace(input.Time)
? new DateTime(0)
: Library.Utility.Timeparser.ParseTimeInterval(input.Time, DateTime.Now);
var r = Runner.Run(Runner.CreateListFolderContents(bk, input.Paths, time, input.PageSize ?? 1000, input.Page ?? 0), false) as IListFolderResults;
var r = queueRunnerService.RunImmediately(Runner.CreateListFolderContents(bk, input.Paths, time, input.PageSize ?? 1000, input.Page ?? 0)) as IListFolderResults;
if (r == null)
throw new ServerErrorException("No result from list operation");
@@ -78,16 +80,16 @@ public class BackupListing : IEndpointV2
r.Entries.TotalCount);
}
private static Dto.V2.ListFilesetsResponseDto ExecuteGetFilesets(IBackup bk)
private static Dto.V2.ListFilesetsResponseDto ExecuteGetFilesets(IQueueRunnerService queueRunnerService, IBackup bk)
{
var extra = new Dictionary<string, string>();
var extra = new Dictionary<string, string?>();
// Retries will hang the http request
extra["number-of-retries"] = "0";
try
{
var r = Runner.Run(Runner.CreateListFilesetsTask(bk, extra), false) as IListFilesetResults;
var r = queueRunnerService.RunImmediately(Runner.CreateListFilesetsTask(bk, extra)) as IListFilesetResults;
if (r == null)
throw new ServerErrorException("No result from list operation");
@@ -112,9 +114,9 @@ public class BackupListing : IEndpointV2
}
}
private static Dto.V2.ListFileVersionsOutputDto ExecuteListVersions(IBackup bk, Dto.V2.ListFileVersionsRequestDto input)
private static Dto.V2.ListFileVersionsOutputDto ExecuteListVersions(IQueueRunnerService queueRunnerService, IBackup bk, Dto.V2.ListFileVersionsRequestDto input)
{
var r = Runner.Run(Runner.ListFileVersionsTask(bk, input.Paths, input.PageSize ?? 1000, input.Page ?? 0), false) as IListFileVersionsResults;
var r = queueRunnerService.RunImmediately(Runner.ListFileVersionsTask(bk, input.Paths, input.PageSize ?? 1000, input.Page ?? 0)) as IListFileVersionsResults;
if (r == null)
throw new ServerErrorException("No result from list operation");
@@ -135,13 +137,13 @@ public class BackupListing : IEndpointV2
r.FileVersions.TotalCount);
}
private static Dto.V2.SearchEntriesResponseDto ExecuteSearch(IBackup bk, Dto.V2.SearchEntriesRequestDto input)
private static Dto.V2.SearchEntriesResponseDto ExecuteSearch(IQueueRunnerService queueRunnerService, IBackup bk, Dto.V2.SearchEntriesRequestDto input)
{
var time = string.IsNullOrWhiteSpace(input.Time)
? new DateTime(0)
: Library.Utility.Timeparser.ParseTimeInterval(input.Time, DateTime.Now);
var r = Runner.Run(Runner.CreateSearchEntriesTask(bk, input.Filters, input.Paths, time, input.PageSize ?? 1000, input.Page ?? 0), false) as ISearchFilesResults;
var r = queueRunnerService.RunImmediately(Runner.CreateSearchEntriesTask(bk, input.Filters, input.Paths, time, input.PageSize ?? 1000, input.Page ?? 0)) as ISearchFilesResults;
if (r == null)
throw new ServerErrorException("No result from list operation");
Duplicati/WebserverCore/Endpoints/V2/DestinationVerify.cs
+9 -10
View File
@@ -22,7 +22,6 @@
using Duplicati.Library.Interface;
using Duplicati.Library.Main;
using Duplicati.Library.Main.Volumes;
using Duplicati.Library.RestAPI;
using Duplicati.Server;
using Duplicati.Server.Database;
using Duplicati.WebserverCore.Abstractions;
@@ -35,8 +34,8 @@ public class DestinationVerify : IEndpointV2
{
public static void Map(RouteGroupBuilder group)
{
group.MapPost("/destination/test", ([FromServices] Connection connection, [FromBody] Dto.V2.DestinationTestRequestDto input, CancellationToken cancelToken)
=> ExecuteTest(input, cancelToken))
group.MapPost("/destination/test", ([FromServices] Connection connection, [FromServices] IApplicationSettings applicationSettings, [FromBody] Dto.V2.DestinationTestRequestDto input, CancellationToken cancelToken)
=> ExecuteTest(connection, applicationSettings, input, cancelToken))
.RequireAuthorization();
}
@@ -51,11 +50,11 @@ public class DestinationVerify : IEndpointV2
}
}
private static Dictionary<string, string?> ParseUrlOptions(Library.Utility.Uri uri)
private static Dictionary<string, string?> ParseUrlOptions(Connection connection, Library.Utility.Uri uri)
{
var qp = uri.QueryParameters;
var opts = Runner.GetCommonOptions();
var opts = Runner.GetCommonOptions(connection);
foreach (var k in qp.Keys.Cast<string>())
opts[k] = qp[k];
@@ -75,13 +74,13 @@ public class DestinationVerify : IEndpointV2
return modules;
}
private static async Task<TupleDisposeWrapper> GetBackend(string url, CancellationToken cancelToken)
private static async Task<TupleDisposeWrapper> GetBackend(Connection connection, IApplicationSettings applicationSettings, string url, CancellationToken cancelToken)
{
var uri = new Library.Utility.Uri(url);
var opts = ParseUrlOptions(uri);
var opts = ParseUrlOptions(connection, uri);
var tmp = new[] { uri };
await SecretProviderHelper.ApplySecretProviderAsync([], tmp, opts, Library.Utility.TempFolder.SystemTempPath, FIXMEGlobal.SecretProvider, cancelToken);
await SecretProviderHelper.ApplySecretProviderAsync([], tmp, opts, Library.Utility.TempFolder.SystemTempPath, applicationSettings.SecretProvider, cancelToken);
url = tmp[0].ToString();
var modules = ConfigureModules(opts);
@@ -90,13 +89,13 @@ public class DestinationVerify : IEndpointV2
}
private static async Task<DestinationTestResponseDto> ExecuteTest(DestinationTestRequestDto input, CancellationToken cancelToken)
private static async Task<DestinationTestResponseDto> ExecuteTest(Connection connection, IApplicationSettings applicationSettings, DestinationTestRequestDto input, CancellationToken cancelToken)
{
TupleDisposeWrapper? wrapper = null;
try
{
wrapper = await GetBackend(input.DestinationUrl, cancelToken);
wrapper = await GetBackend(connection, applicationSettings, input.DestinationUrl, cancelToken);
using (var b = wrapper.Backend)
{
Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs
+7 -6
View File
@@ -20,7 +20,6 @@
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.IO;
using Duplicati.Library.RestAPI;
using Duplicati.Library.RestAPI.Abstractions;
using Duplicati.Server;
using Duplicati.Server.Database;
using Duplicati.Server.Serialization;
@@ -36,7 +35,7 @@ namespace Duplicati.WebserverCore.Extensions;
public static class ServiceCollectionsExtensions
{
public static IServiceCollection AddDuplicati(this IServiceCollection services, Connection connection)
public static IServiceCollection AddDuplicati(this IServiceCollection services, Connection connection, ILogWriteHandler logWriteHandler, IApplicationSettings applicationSettings)
{
//old part
services
@@ -44,7 +43,6 @@ public static class ServiceCollectionsExtensions
.AddSingleton(Serializer.JsonSettings)
.AddSingleton<UpdatePollThread>()
.AddSingleton<EventPollNotify>()
.AddSingleton<Scheduler>()
.AddSingleton(connection);
@@ -57,8 +55,7 @@ public static class ServiceCollectionsExtensions
.AddTransient<IStatusService, StatusService>()
.AddTransient<IUpdateService, UpdateService>()
.AddSingleton<INotificationUpdateService, NotificationUpdateService>()
.AddSingleton<IWorkerThreadsManager, WorkerThreadsManager>()
.AddSingleton<IScheduler, SchedulerService>()
.AddSingleton<ISchedulerService, SchedulerService>()
.AddSingleton<IWebsocketAccessor, WebsocketAccessor>()
.AddTransient<ILanguageService, LanguageService>()
.AddSingleton<ICommandlineRunService, CommandlineRunService>()
@@ -68,7 +65,11 @@ public static class ServiceCollectionsExtensions
.AddSingleton<IRemoteController, RemoteControllerService>()
.AddSingleton<IRemoteControllerHandler, RemoteControllerHandler>()
.AddSingleton<IRemoteControllerRegistration, RemoteControllerRegistrationService>()
.AddSingleton<ISystemInfoProvider, SystemInfoProvider>();
.AddSingleton<ISystemInfoProvider, SystemInfoProvider>()
.AddSingleton<IQueueRunnerService, QueueRunnerService>()
.AddSingleton<IProgressStateProviderService, ProgressStateProviderService>()
.AddSingleton(logWriteHandler)
.AddSingleton(applicationSettings);
return services;
}
Duplicati/WebserverCore/Extensions/WebApplicationExtensions.cs
+4
View File
@@ -56,6 +56,10 @@ public static class WebApplicationExtensions
methodMap!.Invoke(null, [group]);
}
// Special handling for endpoints that are not supported in all modes
var applicationSettings = application.Services.GetRequiredService<IApplicationSettings>();
Endpoints.V1.RemoteControl.MapConditional(group, applicationSettings.Origin);
return application;
}
Duplicati/WebserverCore/Services/ApplicationSettings.cs
+46
View File
@@ -0,0 +1,46 @@
using Duplicati.Library.AutoUpdater;
using Duplicati.Library.Interface;
using Duplicati.WebserverCore.Abstractions;
namespace Duplicati.WebserverCore.Services;
/// <summary>
/// Application settings for the Duplicati server
/// </summary>
public class ApplicationSettings : IApplicationSettings
{
/// <summary>
/// The application exit event
/// </summary>
private readonly ManualResetEvent _applicationExitEvent = new ManualResetEvent(false);
/// <summary>
/// The folder where Duplicati data is stored
/// </summary>
private readonly string _dataFolder;
/// <summary>
/// Initializes a new instance of the <see cref="ApplicationSettings"/> class.
/// </summary>
public ApplicationSettings()
{
_dataFolder = DataFolderManager.GetDataFolder(DataFolderManager.AccessMode.ReadWritePermissionSet);
}
/// <inheritdoc />
public bool SettingsEncryptionKeyProvidedExternally { get; set; }
/// <inheritdoc />
public Action? StartOrStopUsageReporter { get; set; }
/// <inheritdoc />
public string DataFolder => _dataFolder;
/// <inheritdoc />
public string Origin { get; set; } = "Server";
/// <inheritdoc />
public ManualResetEvent ApplicationExitEvent => _applicationExitEvent;
/// <inheritdoc />
public ISecretProvider? SecretProvider { get; set; }
}
Duplicati/WebserverCore/Services/CommandlineRunService.cs
+16 -24
View File
@@ -19,35 +19,27 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.Text;
using Duplicati.Library.RestAPI;
using Duplicati.Library.RestAPI.Abstractions;
using Duplicati.Server;
using Duplicati.WebserverCore.Abstractions;
namespace Duplicati.WebserverCore.Services;
public class CommandlineRunService(IWorkerThreadsManager workerThreadsManager) : ICommandlineRunService
public class CommandlineRunService(IQueueRunnerService queueRunnerService, ILogWriteHandler logWriteHandler) : ICommandlineRunService
{
private static readonly string LOGTAG = Library.Logging.Log.LogTagFromType<CommandlineRunService>();
private class LogWriter : TextWriter
private class LogWriter(ActiveRun target, ILogWriteHandler logWriteHandler) : TextWriter
{
private readonly ActiveRun m_target;
private readonly StringBuilder m_sb = new StringBuilder();
private readonly StringBuilder sb = new StringBuilder();
private int m_newlinechars = 0;
public LogWriter(ActiveRun target)
{
m_target = target;
}
public override Encoding Encoding => Encoding.UTF8;
public override void Write(char value)
{
lock (m_target.Lock)
lock (target.Lock)
{
m_sb.Append(value);
sb.Append(value);
if (NewLine[m_newlinechars] == value)
{
m_newlinechars++;
@@ -62,34 +54,34 @@ public class CommandlineRunService(IWorkerThreadsManager workerThreadsManager) :
public override void WriteLine(string? value)
{
value ??= string.Empty;
lock (m_target.Lock)
lock (target.Lock)
{
m_target.LastAccess = DateTime.Now;
target.LastAccess = DateTime.Now;
//Avoid writing the log if it does not exist
if (m_target.IsLogDisposed)
if (target.IsLogDisposed)
{
FIXMEGlobal.LogHandler.WriteMessage(new Library.Logging.LogEntry("Attempted to write message after closing: {0}", new object[] { value }, Library.Logging.LogMessageType.Warning, LOGTAG, "CommandLineOutputAfterLogClosed", null));
logWriteHandler.WriteMessage(new Library.Logging.LogEntry("Attempted to write message after closing: {0}", new object[] { value }, Library.Logging.LogMessageType.Warning, LOGTAG, "CommandLineOutputAfterLogClosed", null));
return;
}
try
{
if (m_sb.Length != 0)
if (sb.Length != 0)
{
m_target.Log.Add(m_sb + value);
m_sb.Length = 0;
target.Log.Add(sb + value);
sb.Length = 0;
m_newlinechars = 0;
}
else
{
m_target.Log.Add(value);
target.Log.Add(value);
}
}
catch (Exception ex)
{
// This can happen on a very unlucky race where IsLogDisposed is set right after the check
FIXMEGlobal.LogHandler.WriteMessage(new Library.Logging.LogEntry("Failed to forward commandline message: {0}", new object[] { value }, Library.Logging.LogMessageType.Warning, LOGTAG, "CommandLineOutputAfterLogClosed", ex));
logWriteHandler.WriteMessage(new Library.Logging.LogEntry("Failed to forward commandline message: {0}", new object[] { value }, Library.Logging.LogMessageType.Warning, LOGTAG, "CommandLineOutputAfterLogClosed", ex));
}
}
}
@@ -141,7 +133,7 @@ public class CommandlineRunService(IWorkerThreadsManager workerThreadsManager) :
public string StartTask(string[] args)
{
var k = new ActiveRun();
k.Writer = new LogWriter(k);
k.Writer = new LogWriter(k, logWriteHandler);
m_activeItems[k.ID] = k;
StartCleanupTask();
@@ -180,7 +172,7 @@ public class CommandlineRunService(IWorkerThreadsManager workerThreadsManager) :
}
});
workerThreadsManager.AddTask(k.Task);
queueRunnerService.AddTask(k.Task);
return k.ID;
}
Duplicati/WebserverCore/Services/WorkerThreadsManager.cs → Duplicati/WebserverCore/Services/LoginProvider copy.cs
+10 -36
View File
@@ -18,44 +18,18 @@
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using Duplicati.Library.IO;
using Duplicati.Library.RestAPI;
using Duplicati.Library.RestAPI.Abstractions;
using Duplicati.Library.Utility;
using Duplicati.Server;
using Duplicati.Server.Serialization.Interface;
using Duplicati.WebserverCore.Abstractions;
namespace Duplicati.WebserverCore.Services;
public class WorkerThreadsManager(ILiveControls liveControls, IScheduler scheduler) : IWorkerThreadsManager
/// <summary>
/// Provides a way to generate progress state.
/// </summary>
public class ProgressStateProviderService : IProgressStateProviderService
{
public WorkerThread<Runner.IRunnerData>? WorkerThread { get; private set; }
public void Spawn(Action<Runner.IRunnerData> item)
{
WorkerThread = new WorkerThread<Runner.IRunnerData>(item, liveControls.IsPaused);
scheduler.Init(WorkerThread);
}
public Tuple<long, string>? CurrentTask
{
get
{
var t = WorkerThread?.CurrentTask;
return t == null ? null : new Tuple<long, string>(t.TaskID, t.Backup.ID);
}
}
public void UpdateThrottleSpeeds(string? uploadSpeed, string? downloadSpeed)
{
WorkerThread?.CurrentTask?.UpdateThrottleSpeed(uploadSpeed, downloadSpeed);
}
public long AddTask(Runner.IRunnerData data, bool skipQueue = false)
{
WorkerThread!.AddTask(data, skipQueue);
FIXMEGlobal.StatusEventNotifyer.SignalNewEvent();
return data.TaskID;
}
}
/// <summary>
/// The current progress state generator function.
/// </summary>
public Func<IProgressEventData>? GenerateProgressState { get; set; }
}
Duplicati/WebserverCore/Services/QueueRunnerService.cs
+216
View File
@@ -0,0 +1,216 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Interface;
using Duplicati.Library.RestAPI;
using Duplicati.Library.Utility;
using Duplicati.Server;
using Duplicati.Server.Database;
using Duplicati.Server.Serialization.Interface;
using Duplicati.WebserverCore.Abstractions;
namespace Duplicati.WebserverCore.Services;
/// <summary>
/// Simple queue that will run the given task
/// </summary>
public class QueueRunnerService(
Connection connection,
EventPollNotify eventPollNotify,
INotificationUpdateService notificationUpdateService,
IProgressStateProviderService progressStateProviderService,
IApplicationSettings applicationSettings) : IQueueRunnerService
{
private readonly object _lock = new();
/// <summary>
/// A thread-safe dictionary to store cached task results.
/// </summary>
private readonly Dictionary<long, CachedTaskResult> _taskCache = new();
/// <summary>
/// The maximum number of completed task results to keep in memory
/// </summary>
private static readonly int MAX_TASK_RESULT_CACHE_SIZE = 100;
private readonly List<IQueuedTask> _tasks = new();
private (Task? Task, IQueuedTask? QueuedTask) _current;
private bool _isPaused;
private bool _isTerminated;
public long AddTask(IQueuedTask task)
=> AddTask(task, false);
public long AddTask(IQueuedTask task, bool skipQueue)
{
lock (_lock)
if (skipQueue)
_tasks.Insert(0, task);
else
_tasks.Add(task);
eventPollNotify.SignalNewEvent();
StartNextTask();
return task.TaskID;
}
public bool GetIsActive()
=> _current.Task != null;
public IQueuedTask? GetCurrentTask()
=> _current.QueuedTask;
public List<IQueuedTask> GetCurrentTasks()
{
lock (_lock)
return [.. _tasks];
}
public void Pause()
{
lock (_lock)
_isPaused = true;
}
public void Resume()
{
lock (_lock)
_isPaused = false;
StartNextTask();
}
public void Terminate(bool wait)
{
_isTerminated = true;
if (wait)
{
var task = _current.Task;
if (task != null)
task.Await();
}
}
private void StartNextTask()
{
lock (_lock)
{
if (_isTerminated || _isPaused || (_current.Task != null && !_current.Task.IsCompleted))
return;
// Clean up completed tasks
if (_current.Task != null && _current.Task.IsCompleted)
_current = (null, null);
if (_tasks.Count == 0)
return;
var nextTask = _tasks[0];
_tasks.RemoveAt(0);
_current = (Task.Run(() => RunTask(nextTask), CancellationToken.None), nextTask);
}
}
private async Task RunTask(IQueuedTask task)
{
var completed = false;
try
{
eventPollNotify.SignalNewEvent();
task.TaskStarted = DateTime.UtcNow;
if (task.OnStarting != null)
await task.OnStarting().ConfigureAwait(false);
Runner.Run(connection, eventPollNotify, notificationUpdateService, progressStateProviderService, applicationSettings, task, true);
// If the task is completed, don't call OnFinished again
completed = true;
AddTaskResult(new CachedTaskResult(task.TaskID, task.BackupID, task.TaskStarted, task.TaskFinished ?? DateTime.Now, null));
if (task.OnFinished != null)
await task.OnFinished(null).ConfigureAwait(false);
}
catch (Exception ex)
{
connection.LogError(task.BackupID, "Error in worker", ex);
if (!completed)
{
AddTaskResult(new CachedTaskResult(task.TaskID, task.BackupID, task.TaskStarted, task.TaskFinished ?? DateTime.Now, ex));
if (task.OnFinished != null)
await task.OnFinished(ex).ConfigureAwait(false);
}
}
finally
{
task.TaskFinished = DateTime.UtcNow;
lock (_lock)
_current = (null, null);
eventPollNotify.SignalNewEvent();
StartNextTask();
}
}
public IList<Tuple<long, string?>> GetQueueWithIds()
{
return (from n in GetCurrentTasks()
where n.BackupID != null
select new Tuple<long, string?>(n.TaskID, n.BackupID)).ToList();
}
/// <inheritdoc/>
public CachedTaskResult? GetCachedTaskResults(long taskID)
{
lock (_lock)
{
_taskCache.TryGetValue(taskID, out var result);
return result;
}
}
private void AddTaskResult(CachedTaskResult taskResult)
{
lock (_lock)
{
// If the task result is already in the cache, remove it
if (_taskCache.TryGetValue(taskResult.TaskID, out var existingResult))
{
// If the stored task result has an exception, do not overwrite it
if (existingResult.Exception != null)
return;
}
// Add/update the new task result in the cache
_taskCache[taskResult.TaskID] = taskResult;
// If the cache size exceeds the maximum, remove the oldest entry
while (_taskCache.Count >= MAX_TASK_RESULT_CACHE_SIZE)
{
var oldestTaskID = _taskCache.Keys.Min();
_taskCache.Remove(oldestTaskID);
}
}
}
/// <inheritdoc/>
public IBasicResults? RunImmediately(IQueuedTask task)
{
return Runner.Run(connection, eventPollNotify, notificationUpdateService, progressStateProviderService, applicationSettings, task, false);
}
}
Duplicati/WebserverCore/Services/RemoteControllerHandler.cs
+4 -5
View File
@@ -22,7 +22,6 @@
using System.Net.Http.Headers;
using Duplicati.Library.Logging;
using Duplicati.Library.RemoteControl;
using Duplicati.Library.RestAPI;
using Duplicati.Server.Database;
using Duplicati.WebserverCore.Abstractions;
using Newtonsoft.Json;
@@ -35,7 +34,7 @@ namespace Duplicati.WebserverCore.Services;
/// <param name="connection">The connection to the database</param>
/// <param name="httpClientFactory">The HTTP client factory</param>
/// <param name="jwtTokenProvider">The JWT token provider</param>
public class RemoteControllerHandler(Connection connection, IHttpClientFactory httpClientFactory, IJWTTokenProvider jwtTokenProvider) : IRemoteControllerHandler
public class RemoteControllerHandler(Connection connection, IHttpClientFactory httpClientFactory, IJWTTokenProvider jwtTokenProvider, IApplicationSettings applicationSettings) : IRemoteControllerHandler
{
/// <summary>
/// The log tag for this class.
@@ -64,7 +63,7 @@ public class RemoteControllerHandler(Connection connection, IHttpClientFactory h
CertificateUrl = data.CertificateUrl
});
if (!FIXMEGlobal.SettingsEncryptionKeyProvidedExternally)
if (!applicationSettings.SettingsEncryptionKeyProvidedExternally)
{
// TODO: Implement changing the encryption key
// if (!string.IsNullOrWhiteSpace(data.LocalEncryptionKey) && data.LocalEncryptionKey != connection.ApplicationSettings.SettingsEncryptionKey)
@@ -86,10 +85,10 @@ public class RemoteControllerHandler(Connection connection, IHttpClientFactory h
try
{
if (string.Equals(message.ControlRequestMessage.Command, ControlRequestMessage.ConfigureReportUrlSet, StringComparison.OrdinalIgnoreCase))
FIXMEGlobal.DataConnection.ApplicationSettings.AdditionalReportUrl = message.ControlRequestMessage.Parameters.GetValueOrDefault(ControlRequestMessage.ConfigureReportUrlParameter);
connection.ApplicationSettings.AdditionalReportUrl = message.ControlRequestMessage.Parameters.GetValueOrDefault(ControlRequestMessage.ConfigureReportUrlParameter);
if (string.Equals(message.ControlRequestMessage.Command, ControlRequestMessage.ConfigureReportUrlGet, StringComparison.OrdinalIgnoreCase))
result = new Dictionary<string, string?> { { ControlRequestMessage.ConfigureReportUrlParameter, FIXMEGlobal.DataConnection.ApplicationSettings.AdditionalReportUrl } };
result = new Dictionary<string, string?> { { ControlRequestMessage.ConfigureReportUrlParameter, connection.ApplicationSettings.AdditionalReportUrl } };
}
catch (Exception ex)
{
Duplicati/WebserverCore/Services/SchedulerService.cs
+15 -16
View File
@@ -19,31 +19,33 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Utility;
using Duplicati.Library.RestAPI;
using Duplicati.Server;
using Duplicati.Server.Database;
using Duplicati.Server.Serialization.Interface;
using Duplicati.WebserverCore.Abstractions;
namespace WebserverCore.Services;
public class SchedulerService : IScheduler
public class SchedulerService : ISchedulerService
{
private readonly Duplicati.Server.Scheduler scheduler;
public SchedulerService(Duplicati.Server.Scheduler scheduler)
private readonly Scheduler scheduler;
public SchedulerService(Connection connection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IQueueRunnerService queueRunnerService)
{
this.scheduler = scheduler;
this.scheduler = new Scheduler(connection, queueRunnerService);
var lastScheduleId = notificationUpdateService.LastDataUpdateId;
eventPollNotify.NewEvent += (sender, e) =>
{
if (lastScheduleId != notificationUpdateService.LastDataUpdateId)
{
lastScheduleId = notificationUpdateService.LastDataUpdateId;
Reschedule();
}
};
}
public List<KeyValuePair<DateTime, ISchedule>> Schedule => scheduler.Schedule;
public List<Runner.IRunnerData> WorkerQueue => scheduler.WorkerQueue;
public void SubScribeToNewSchedule(Action handler)
=> scheduler.NewSchedule += (_, _) => handler();
public IList<Tuple<long, string>> GetSchedulerQueueIds()
=> scheduler.GetSchedulerQueueIds();
public IList<Tuple<string, DateTime>> GetProposedSchedule()
=> scheduler.GetProposedSchedule();
@@ -52,7 +54,4 @@ public class SchedulerService : IScheduler
public void Terminate(bool wait)
=> scheduler.Terminate(wait);
public void Init(WorkerThread<Runner.IRunnerData> worker)
=> scheduler.Init(worker);
}
Duplicati/WebserverCore/Services/StatusService.cs
+8 -6
View File
@@ -19,7 +19,6 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.RestAPI;
using Duplicati.Library.RestAPI.Abstractions;
using Duplicati.Server;
using Duplicati.Server.Serialization;
using Duplicati.WebserverCore.Abstractions;
@@ -31,22 +30,24 @@ public class StatusService(
LiveControls liveControls,
UpdatePollThread updatePollThread,
IUpdateService updateService,
IWorkerThreadsManager workerThreadsManager,
IQueueRunnerService queueRunnerService,
ISettingsService settingsService,
IScheduler scheduler,
ISchedulerService scheduler,
EventPollNotify eventPollNotify,
INotificationUpdateService notificationUpdateService)
: IStatusService
{
public ServerStatusDto GetStatus()
{
var task = queueRunnerService.GetCurrentTask();
var status = new ServerStatusDto
{
UpdatedVersion = GetUpdatedVersion(),
UpdaterState = updatePollThread.ThreadState,
UpdateDownloadProgress = updatePollThread.DownloadProgess,
ActiveTask = workerThreadsManager.CurrentTask,
SchedulerQueueIds = scheduler.GetSchedulerQueueIds(),
ActiveTask = task == null ? null : new Tuple<long, string?>(task.TaskID, task.BackupID),
SchedulerQueueIds = queueRunnerService.GetQueueWithIds(),
ProposedSchedule = scheduler.GetProposedSchedule(),
LastEventID = eventPollNotify.EventNo,
LastDataUpdateID = notificationUpdateService.LastDataUpdateId,
@@ -72,7 +73,8 @@ public class StatusService(
private SuggestedStatusIcon MapStateToIcon()
{
if (workerThreadsManager.CurrentTask == null)
var task = queueRunnerService.GetCurrentTask();
if (task == null)
{
if (liveControls.State == LiveControls.LiveControlState.Paused)
return SuggestedStatusIcon.Paused;
Duplicati/WebserverCore/Services/SystemInfoProvider.cs
+7 -11
View File
@@ -21,8 +21,8 @@
using System.Globalization;
using Duplicati.Library.AutoUpdater;
using Duplicati.Library.Localization;
using Duplicati.Library.RestAPI;
using Duplicati.Server;
using Duplicati.Server.Database;
using Duplicati.Server.Serialization.Interface;
using Duplicati.WebserverCore.Abstractions;
using Duplicati.WebserverCore.Dto;
@@ -32,7 +32,7 @@ namespace Duplicati.WebserverCore.Services;
/// <summary>
/// Produces system information.
/// </summary>
public class SystemInfoProvider : ISystemInfoProvider
public class SystemInfoProvider(IApplicationSettings applicationSettings) : ISystemInfoProvider
{
/// <summary>
/// The API extensions that are available
@@ -42,7 +42,9 @@ public class SystemInfoProvider : ISystemInfoProvider
"v2:backup:list-folder",
"v2:backup:list-versions",
"v2:backup:search",
"v2:destination:test"
"v2:destination:test",
"v1:gettask:taskstarted",
"v1:gettask:taskfinished",
];
/// <summary>
@@ -85,11 +87,6 @@ public class SystemInfoProvider : ISystemInfoProvider
/// </summary>
public required string RemoteControlRegistrationUrl { get; init; }
/// <summary>
/// Gets or sets the started by.
/// </summary>
public required string StartedBy { get; init; }
/// <summary>
/// Gets or sets the default update channel.
/// </summary>
@@ -229,12 +226,11 @@ public class SystemInfoProvider : ISystemInfoProvider
=> new StaticSystemInformation
{
APIVersion = 1,
PasswordPlaceholder = FIXMEGlobal.PASSWORD_PLACEHOLDER,
PasswordPlaceholder = Connection.PASSWORD_PLACEHOLDER,
ServerVersion = UpdaterManager.SelfVersion.Version,
ServerVersionName = License.VersionNumbers.VERSION_NAME,
ServerVersionType = UpdaterManager.SelfVersion.ReleaseType,
RemoteControlRegistrationUrl = Library.RemoteControl.RegisterForRemote.DefaultRegisterationUrl,
StartedBy = FIXMEGlobal.Origin,
DefaultUpdateChannel = AutoUpdateSettings.DefaultUpdateChannel.ToString(),
DefaultUsageReportLevel = Library.UsageReporter.Reporter.DefaultReportLevel,
OSType = UpdaterManager.OperatingSystemName,
@@ -313,7 +309,7 @@ public class SystemInfoProvider : ISystemInfoProvider
ServerVersionName = systeminfo.ServerVersionName,
ServerVersionType = systeminfo.ServerVersionType,
RemoteControlRegistrationUrl = systeminfo.RemoteControlRegistrationUrl,
StartedBy = systeminfo.StartedBy,
StartedBy = applicationSettings.Origin,
DefaultUpdateChannel = systeminfo.DefaultUpdateChannel,
DefaultUsageReportLevel = systeminfo.DefaultUsageReportLevel,
ServerTime = DateTime.Now,
LiveTests/Duplicati.Backend.Tests/OneDrive/OneDriveTests.cs
+46
View File
@@ -0,0 +1,46 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
namespace Duplicati.Backend.Tests.OneDrive;
/// <summary>
/// OneDrive Tests
/// </summary>
[TestClass]
public sealed class OneDriveTests : BaseTest
{
[TestMethod]
public Task TestOneDrive()
{
CheckRequiredEnvironment(["TESTCREDENTIAL_ONEDRIVE_FOLDER", "TESTCREDENTIAL_ONEDRIVE_AUTHID"]);
var exitCode = CommandLine.BackendTester.Program.Main(
new[]
{
$"onedrivev2://{Environment.GetEnvironmentVariable("TESTCREDENTIAL_ONEDRIVE_FOLDER")}/?authid={Uri.EscapeDataString(Environment.GetEnvironmentVariable("TESTCREDENTIAL_ONEDRIVE_AUTHID")!)}",
}.Concat(Parameters.GlobalTestParameters).ToArray());
if (exitCode != 0) Assert.Fail("BackendTester is returning non-zero exit code, check logs for details");
return Task.CompletedTask;
}
}