haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge commit 'b1f705f71a39e4ea79c7dae56bfd0fb345af5110' into feature/support-glacier-style-non-readable-files

Browse Source
This commit is contained in:
Kenneth Skovhede
2025-03-18 20:44:05 +01:00
parent 14ea722e53 b1f705f71a
commit dd07dabee1
352 changed files with 18753 additions and 9638 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/backendtests.yml
+142
View File
@@ -13,6 +13,8 @@ jobs:
testcontainers_secrets_available: "${{ steps.check_testcontainers.outputs.available }}"
google_secrets_available: "${{ steps.check_google.outputs.available }}"
pcloud_secrets_available: "${{ steps.check_pcloud.outputs.available }}"
b2_secrets_available: "${{ steps.check_b2.outputs.available }}"
box_secrets_available: "${{ steps.check_box.outputs.available }}"
steps:
- id: check_pcloud
name: Check pCloud secrets
@@ -103,6 +105,36 @@ jobs:
fi
echo "Google check completed"
- id: check_b2
name: Check B2 secrets
shell: bash
run: |
echo "Starting B2 secrets check..."
if [[ -n "${{ secrets.TESTCREDENTIAL_B2_BUCKET }}" ]] && \
[[ -n "${{ secrets.TESTCREDENTIAL_B2_FOLDER }}" ]] && \
[[ -n "${{ secrets.TESTCREDENTIAL_B2_USERNAME }}" ]] && \
[[ -n "${{ secrets.TESTCREDENTIAL_B2_PASSWORD }}" ]]; then
echo "All B2 secrets found"
echo "available=true" >> $GITHUB_OUTPUT
else
echo "Missing some B2 secrets"
echo "available=false" >> $GITHUB_OUTPUT
fi
echo "B2 check completed"
- id: check_box
name: Check Box.com secrets
shell: bash
run: |
echo "Starting Box.com secrets check..."
if [[ -n "${{ secrets.TESTCREDENTIAL_BOX_FOLDER }}" ]] && \
[[ -n "${{ secrets.TESTCREDENTIAL_BOX_AUTHID }}" ]]; then
echo "All Box.com secrets found"
echo "available=true" >> $GITHUB_OUTPUT
else
echo "Missing some Box.com secrets"
echo "available=false" >> $GITHUB_OUTPUT
fi
echo "Box.com check completed"
- name: Debug Output
shell: bash
run: |
@@ -110,6 +142,10 @@ jobs:
echo "Dropbox Available: ${{ steps.check_dropbox.outputs.available }}"
echo "TestContainers Available: ${{ steps.check_testcontainers.outputs.available }}"
echo "Google Available: ${{ steps.check_google.outputs.available }}"
echo "pCloud Available: ${{ steps.check_pcloud.outputs.available }}"
echo "Azure Available: ${{ steps.check_azure.outputs.available }}"
echo "B2 Available: ${{ steps.check_b2.outputs.available }}"
echo "Box.com Available: ${{ steps.check_box.outputs.available }}"
test_ftp:
needs: check_secrets
if: needs.check_secrets.outputs.testcontainers_secrets_available == 'true'
@@ -257,6 +293,7 @@ jobs:
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
- name: Run Azure tests
env:
READ_WRITE_TIMEOUT_SECONDS: 18000
TESTCREDENTIAL_AZURE_ACCOUNTNAME: "${{ secrets.TESTCREDENTIAL_AZURE_ACCOUNTNAME }}"
TESTCREDENTIAL_AZURE_ACCESSKEY: "${{ secrets.TESTCREDENTIAL_AZURE_ACCESSKEY }}"
TESTCREDENTIAL_AZURE_CONTAINERNAME: "${{ secrets.TESTCREDENTIAL_AZURE_CONTAINERNAME }}"
@@ -415,3 +452,108 @@ jobs:
dotnet test --no-build --filter="ClassName~GoogleDriveTests"
--logger:"console;verbosity=detailed"
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
test_cifs:
needs: check_secrets
if: needs.check_secrets.outputs.testcontainers_secrets_available == 'true'
name: CIFS Tests
runs-on: "${{ matrix.os }}"
strategy:
max-parallel: 1
fail-fast: false
matrix:
os:
- ubuntu-latest
steps:
- name: Set up .NET
uses: actions/setup-dotnet@v4
with:
dotnet-version: 8.x
- name: Checkout source
uses: actions/checkout@v4
- name: Restore NuGet dependencies
run: >-
dotnet restore
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
- name: Build project
run: >-
dotnet build --no-restore
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
- name: Run CIFS tests
run: >-
dotnet test --no-build --filter="ClassName~CIFSTests"
--logger:"console;verbosity=detailed"
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
test_b2:
needs: check_secrets
if: needs.check_secrets.outputs.b2_secrets_available == 'true'
name: Backblaze B2 Tests
runs-on: "${{ matrix.os }}"
strategy:
max-parallel: 1
fail-fast: false
matrix:
os:
- ubuntu-latest
- windows-latest
- macos-latest
steps:
- name: Set up .NET
uses: actions/setup-dotnet@v4
with:
dotnet-version: 8.x
- name: Checkout source
uses: actions/checkout@v4
- name: Restore NuGet dependencies
run: >-
dotnet restore
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
- name: Build project
run: >-
dotnet build --no-restore
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
- name: Run B2 tests
env:
TESTCREDENTIAL_B2_BUCKET: "${{ secrets.TESTCREDENTIAL_B2_BUCKET }}"
TESTCREDENTIAL_B2_FOLDER: "${{ secrets.TESTCREDENTIAL_B2_FOLDER }}"
TESTCREDENTIAL_B2_USERNAME: "${{ secrets.TESTCREDENTIAL_B2_USERNAME }}"
TESTCREDENTIAL_B2_PASSWORD: "${{ secrets.TESTCREDENTIAL_B2_PASSWORD }}"
run: >-
dotnet test --no-build --filter="ClassName~B2Tests"
--logger:"console;verbosity=detailed"
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
test_box:
needs: check_secrets
if: needs.check_secrets.outputs.box_secrets_available == 'true'
name: Box.com Tests
runs-on: "${{ matrix.os }}"
strategy:
max-parallel: 1
fail-fast: false
matrix:
os:
- ubuntu-latest
- windows-latest
- macos-latest
steps:
- name: Set up .NET
uses: actions/setup-dotnet@v4
with:
dotnet-version: 8.x
- name: Checkout source
uses: actions/checkout@v4
- name: Restore NuGet dependencies
run: >-
dotnet restore
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
- name: Build project
run: >-
dotnet build --no-restore
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
- name: Run Box.com tests
env:
TESTCREDENTIAL_BOX_FOLDER: "${{ secrets.TESTCREDENTIAL_BOX_FOLDER }}"
TESTCREDENTIAL_BOX_AUTHID: "${{ secrets.TESTCREDENTIAL_BOX_AUTHID }}"
run: >-
dotnet test --no-build --filter="ClassName~Box.BoxTests"
--logger:"console;verbosity=detailed"
LiveTests/Duplicati.Backend.Tests/Duplicati.Backend.Tests.sln
.github/workflows/tests.yml
+7 -6
View File
@@ -29,9 +29,10 @@ jobs:
- name: Run unit tests
run: dotnet test --no-build --verbosity minimal Duplicati.sln
selenium:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Selenium
run: pipeline/selenium/test.sh
# Disabled, as a new test needs to be written for the new UI
# selenium:
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v4
# - name: Selenium
# run: pipeline/selenium/test.sh
.vscode/launch.json
Vendored
+8 -1
View File
@@ -114,6 +114,13 @@
"stopAtEntry": false,
"console": "internalConsole"
},
{
"name": "Launch SourceTool executable",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "build",
"program": "${workspaceFolder}/Executables/net8/Duplicati.CommandLine.SourceTool/bin/Debug/net8.0/Duplicati.CommandLine.SourceTool",
"args": []
}
]
}
Duplicati.sln
+45
View File
@@ -166,6 +166,22 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Crashlog"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Backend.CIFS", "Duplicati\Library\Backend\CIFS\Duplicati.Library.Backend.CIFS.csproj", "{836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.SourceProviders", "Duplicati\Library\SourceProviders\Duplicati.Library.SourceProviders.csproj", "{8A2C7A9F-3EC8-4DE5-A9FA-9E4BCF955EF3}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SourceProviders", "SourceProviders", "{424CEF73-4984-430E-9C8B-E61CC0F22074}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.SourceProvider.Builtin", "Duplicati\Library\SourceProvider\Builtin\Duplicati.Library.SourceProvider.Builtin.csproj", "{6D9742A7-6F32-4571-8276-79716168B525}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tools", "Tools", "{15388C37-9218-4818-972E-738EEA8F1602}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RemoteSynchronization", "Tools\RemoteSynchronization\RemoteSynchronization.csproj", "{D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.CommandLine.SyncTool", "Executables\net8\Duplicati.CommandLine.SyncTool\Duplicati.CommandLine.SyncTool.csproj", "{48D32674-0FE9-4407-B102-4AE46D93595F}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.CommandLine.SourceTool", "Duplicati\CommandLine\SourceTool\Duplicati.CommandLine.SourceTool.csproj", "{2873BCE2-2EA6-419D-BFFA-693FDB34926E}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.CommandLine.SourceTool", "Executables\net8\Duplicati.CommandLine.SourceTool\Duplicati.CommandLine.SourceTool.csproj", "{4003BF90-6681-4155-9D06-639ED23FE3AB}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -472,6 +488,30 @@ Global
{836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{836E0557-B40C-4DC7-9A2A-5C062F9ACC6B}.Release|Any CPU.Build.0 = Release|Any CPU
{8A2C7A9F-3EC8-4DE5-A9FA-9E4BCF955EF3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8A2C7A9F-3EC8-4DE5-A9FA-9E4BCF955EF3}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8A2C7A9F-3EC8-4DE5-A9FA-9E4BCF955EF3}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8A2C7A9F-3EC8-4DE5-A9FA-9E4BCF955EF3}.Release|Any CPU.Build.0 = Release|Any CPU
{6D9742A7-6F32-4571-8276-79716168B525}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6D9742A7-6F32-4571-8276-79716168B525}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6D9742A7-6F32-4571-8276-79716168B525}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6D9742A7-6F32-4571-8276-79716168B525}.Release|Any CPU.Build.0 = Release|Any CPU
{D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}.Release|Any CPU.Build.0 = Release|Any CPU
{48D32674-0FE9-4407-B102-4AE46D93595F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{48D32674-0FE9-4407-B102-4AE46D93595F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{48D32674-0FE9-4407-B102-4AE46D93595F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{48D32674-0FE9-4407-B102-4AE46D93595F}.Release|Any CPU.Build.0 = Release|Any CPU
{2873BCE2-2EA6-419D-BFFA-693FDB34926E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{2873BCE2-2EA6-419D-BFFA-693FDB34926E}.Debug|Any CPU.Build.0 = Debug|Any CPU
{2873BCE2-2EA6-419D-BFFA-693FDB34926E}.Release|Any CPU.ActiveCfg = Release|Any CPU
{2873BCE2-2EA6-419D-BFFA-693FDB34926E}.Release|Any CPU.Build.0 = Release|Any CPU
{4003BF90-6681-4155-9D06-639ED23FE3AB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{4003BF90-6681-4155-9D06-639ED23FE3AB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4003BF90-6681-4155-9D06-639ED23FE3AB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4003BF90-6681-4155-9D06-639ED23FE3AB}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@@ -532,6 +572,11 @@ Global
{CA1A0EE8-DFFF-42AF-B15D-EF538357A2DE} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4}
{8ACA2736-3C69-4FA5-BCF9-5EDF50CAF332} = {566EBBDA-19A4-4056-A615-D901D57D2439}
{836E0557-B40C-4DC7-9A2A-5C062F9ACC6B} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4}
{6D9742A7-6F32-4571-8276-79716168B525} = {424CEF73-4984-430E-9C8B-E61CC0F22074}
{D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D} = {15388C37-9218-4818-972E-738EEA8F1602}
{48D32674-0FE9-4407-B102-4AE46D93595F} = {6B46F6B1-1898-49B8-ADA7-5CAF68EB77E3}
{2873BCE2-2EA6-419D-BFFA-693FDB34926E} = {D19A38DD-68F1-4EF5-BF5F-8966CE0D9A5B}
{4003BF90-6681-4155-9D06-639ED23FE3AB} = {6B46F6B1-1898-49B8-ADA7-5CAF68EB77E3}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {8B40BAFE-D862-4397-9495-8F5EAF5CE80C}
Duplicati/Agent/Program.cs
+41 -46
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.CommandLine;
@@ -25,6 +25,7 @@ using System.CommandLine.NamingConventionBinder;
using System.CommandLine.Parsing;
using System.Reactive.Linq;
using System.Security.Cryptography;
using Duplicati.Library.AutoUpdater;
using Duplicati.Library.Encryption;
using Duplicati.Library.Interface;
using Duplicati.Library.Logging;
@@ -33,7 +34,7 @@ using Duplicati.Library.RemoteControl;
using Duplicati.Library.RestAPI;
using Duplicati.Library.Utility;
using Duplicati.Server;
using Duplicati.WebserverCore.Abstractions;
using Duplicati.WebserverCore.Services;
using Microsoft.Extensions.DependencyInjection;
namespace Duplicati.Agent;
@@ -45,12 +46,6 @@ public static class Program
/// </summary>
private static readonly string LogTag = Log.LogTagFromType(typeof(Program));
/// <summary>
/// Create a random pre-shared key for the agent
/// </summary>
private static string? PreSharedKey = null;
/// <summary>
/// Commandline arguments for the agent
/// </summary>
@@ -123,6 +118,8 @@ public static class Program
new Option<string?>("--secret-provider", description: "The secret provider to use", getDefaultValue: () => null),
new Option<SecretProviderHelper.CachingLevel>("--secret-provider-cache", description: "The secret provider cache level", getDefaultValue: () => SecretProviderHelper.CachingLevel.None),
new Option<string>("--secret-provider-pattern", description: "The secret provider pattern", getDefaultValue: () => SecretProviderHelper.DEFAULT_PATTERN),
new Option<bool>($"--{DataFolderManager.PORTABLE_MODE_OPTION}", description: "Use portable mode for locating the database and storing configuration", getDefaultValue: () => DataFolderManager.PORTABLE_MODE),
new Option<DirectoryInfo?>($"--{DataFolderManager.SERVER_DATAFOLDER_OPTION}", description: "The datafolder to use for locating the database and storing configuration", getDefaultValue: () => new DirectoryInfo(DataFolderManager.DATAFOLDER)),
};
runcmd.Handler = CommandHandler.Create<CommandLineArguments>(RunAgent);
@@ -166,6 +163,7 @@ public static class Program
context.ExitCode = 1;
}
})
.UseAdditionalHelpAliases()
.Build()
.InvokeAsync(args);
}
@@ -271,7 +269,7 @@ public static class Program
// Set the pre-shared key for the agent
if (!agentConfig.DisablePreSharedKey)
WebserverCore.Middlewares.PreSharedKeyFilter.PreSharedKey = PreSharedKey = RandomNumberGenerator.GetHexString(128);
WebserverCore.Middlewares.PreSharedKeyFilter.PreSharedKey = RandomNumberGenerator.GetHexString(128);
using var cts = new CancellationTokenSource();
@@ -309,8 +307,10 @@ public static class Program
settings.CertificateUrl,
settings.ServerCertificates,
cts.Token,
c => ReKey(agentConfig, c),
(m) => OnMessage(m, agentConfig)
OnConnect,
m => ReKey(m, agentConfig),
OnControl,
OnMessage
)
);
@@ -320,24 +320,19 @@ public static class Program
}
}
private static async Task OnMessage(KeepRemoteConnection.CommandMessage message, CommandLineArguments agentConfig)
{
Log.WriteMessage(LogMessageType.Verbose, LogTag, "OnMessage", "Received message: {0}", message);
var provider = FIXMEGlobal.Provider.GetRequiredService<IJWTTokenProvider>();
var token = provider.CreateAccessToken("agent", provider.TemporaryFamilyId, TimeSpan.FromMinutes(2));
using var httpClient = FIXMEGlobal.Provider.GetRequiredService<IHttpClientFactory>().CreateClient();
httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", token);
httpClient.BaseAddress = new System.Uri($"http://127.0.0.1:{agentConfig.WebservicePort}");
if (!string.IsNullOrWhiteSpace(WebserverCore.Middlewares.PreSharedKeyFilter.PreSharedKey))
httpClient.DefaultRequestHeaders.Add(WebserverCore.Middlewares.PreSharedKeyFilter.HeaderName, PreSharedKey);
await message.Handle(httpClient);
}
private static Task ReKey(CommandLineArguments agentConfig, ClaimedClientData keydata)
private static Task<Dictionary<string, string?>> OnConnect(Dictionary<string, string?> metadata)
=> FIXMEGlobal.Provider.GetRequiredService<IRemoteControllerHandler>().OnConnect(metadata);
private static Task OnControl(KeepRemoteConnection.ControlMessage message)
=> FIXMEGlobal.Provider.GetRequiredService<IRemoteControllerHandler>().OnControl(message);
private static Task OnMessage(KeepRemoteConnection.CommandMessage message)
=> FIXMEGlobal.Provider.GetRequiredService<IRemoteControllerHandler>().OnMessage(message);
private static Task ReKey(ClaimedClientData keydata, CommandLineArguments agentConfig)
{
// ReKey is handled here because we store the config outside of the database
Log.WriteMessage(LogMessageType.Verbose, LogTag, "ReKey", "Rekeying the settings");
var settings = Settings.Load(agentConfig.AgentSettingsFile.FullName, agentConfig.AgentSettingsFilePassphrase);
if (!string.IsNullOrWhiteSpace(keydata.JWT) && settings.JWT != keydata.JWT)
Duplicati/Agent/Settings.cs
+2 -4
View File
@@ -148,10 +148,8 @@ public sealed record Settings(
/// <returns>The path to the settings file</returns>
private static string GetSettingsPath()
{
var folder = DatabaseLocator.GetDefaultStorageFolderWithDebugSupport(DefaultSettingsFilename);
if (!Directory.Exists(folder))
Directory.CreateDirectory(folder);
// Ideally, this should use DataFolderManager.DATAFOLDER, but we cannot due to backwards compatibility
var folder = Library.AutoUpdater.DataFolderLocator.GetDefaultStorageFolder(DefaultSettingsFilename, true);
return Path.Combine(folder, DefaultSettingsFilename);
}
}
Duplicati/CommandLine/BackendTester/Program.cs
+69 -55
View File
@@ -29,6 +29,7 @@ using System.Globalization;
using System.Threading;
using Duplicati.Library.Utility;
using Duplicati.StreamUtil;
using System.Threading.Tasks;
namespace Duplicati.CommandLine.BackendTester
{
@@ -83,7 +84,7 @@ namespace Duplicati.CommandLine.BackendTester
List<string> args = new List<string>(_args);
Dictionary<string, string> options = Library.Utility.CommandLineParser.ExtractOptions(args);
if (args.Count != 1 || String.Equals(args[0], "help", StringComparison.OrdinalIgnoreCase) || args[0] == "?")
if (args.Count != 1 || HelpOptionExtensions.IsArgumentAnyHelpString(args))
{
Console.WriteLine("Usage: <protocol>://<username>:<password>@<path>");
Console.WriteLine("Example: ftp://user:pass@server/folder");
@@ -118,16 +119,17 @@ namespace Duplicati.CommandLine.BackendTester
for (int i = 0; i < reruns; i++)
{
Console.WriteLine("Starting run no {0}", i);
Console.WriteLine(DateTime.Now.ToShortDateString() + " " + DateTime.Now.ToLongTimeString());
Console.WriteLine(LogTimeStamp + "Starting run no {0}", i);
if (!Run(args, options, i == 0))
return 1;
}
Console.WriteLine("Unittest complete!");
Console.WriteLine(LogTimeStamp + "Unittest complete!");
return 0;
}
catch (Exception ex)
{
Console.WriteLine("Unittest failed: " + ex);
Console.WriteLine(LogTimeStamp + "Unittest failed: " + ex);
}
return 1;
@@ -172,7 +174,7 @@ namespace Duplicati.CommandLine.BackendTester
try
{
backend.TestAsync(CancellationToken.None).Await();
curlist = backend.List();
curlist = backend.ListAsync(CancellationToken.None).ToBlockingEnumerable().ToList();
}
catch (FolderMissingException)
{
@@ -181,11 +183,11 @@ namespace Duplicati.CommandLine.BackendTester
try
{
backend.CreateFolderAsync(CancellationToken.None).Await();
curlist = backend.List();
curlist = backend.ListAsync(CancellationToken.None).ToBlockingEnumerable().ToList();
}
catch (Exception ex)
{
Console.WriteLine("Autocreate folder failed with message: " + ex.Message);
Console.WriteLine(LogTimeStamp + "Autocreate folder failed with message: " + ex.Message);
}
}
@@ -199,7 +201,7 @@ namespace Duplicati.CommandLine.BackendTester
if (Library.Utility.Utility.ParseBoolOption(options, "auto-clean") && first)
if (Library.Utility.Utility.ParseBoolOption(options, "force"))
{
Console.WriteLine("Auto clean, removing file: {0}", fe.Name);
Console.WriteLine(LogTimeStamp + "Auto clean, removing file: {0}", fe.Name);
backend.DeleteAsync(fe.Name, CancellationToken.None).Await();
continue;
}
@@ -208,10 +210,10 @@ namespace Duplicati.CommandLine.BackendTester
var fileCount = curlist.Where(x => !x.IsFolder).Count();
var filenames = curlist.Where(x => !x.IsFolder).Select(x => x.Name).Take(10).ToList();
Console.WriteLine($"*** Remote folder contains {fileCount} file(s), aborting");
Console.WriteLine($"*** First {filenames.Count} file(s): {Environment.NewLine}{string.Join(Environment.NewLine, filenames)}");
Console.WriteLine(LogTimeStamp + $"*** Remote folder contains {fileCount} file(s), aborting");
Console.WriteLine(LogTimeStamp + $"*** First {filenames.Count} file(s): {Environment.NewLine}{string.Join(Environment.NewLine, filenames)}");
if (fileCount > filenames.Count)
Console.WriteLine($"*** ... and {fileCount - filenames.Count} more file(s)");
Console.WriteLine(LogTimeStamp + $"*** ... and {fileCount - filenames.Count} more file(s)");
return false;
}
@@ -232,7 +234,7 @@ namespace Duplicati.CommandLine.BackendTester
{
if (!(backend is IStreamingBackend) || disableStreaming)
{
Console.WriteLine("Warning: Throttling is only supported in this tool on streaming backends");
Console.WriteLine(LogTimeStamp + "Warning: Throttling is only supported in this tool on streaming backends");
}
throttleUpload = Duplicati.Library.Utility.Sizeparser.ParseSize(throttleUploadString, "kb");
@@ -243,15 +245,25 @@ namespace Duplicati.CommandLine.BackendTester
{
if (!(backend is IStreamingBackend) || disableStreaming)
{
Console.WriteLine("Warning: Throttling is only supported in this tool on streaming backends");
Console.WriteLine(LogTimeStamp + "Warning: Throttling is only supported in this tool on streaming backends");
}
throttleDownload = Duplicati.Library.Utility.Sizeparser.ParseSize(throttleDownloadString, "kb");
}
var readWriteTimeout = (int)TimeSpan.FromSeconds(10).TotalMilliseconds;
if (backend is ITimeoutExemptBackend)
readWriteTimeout = Timeout.Infinite;
int readWriteTimeout = backend is ITimeoutExemptBackend ||
(Environment.GetEnvironmentVariable("READ_WRITE_TIMEOUT_SECONDS") is { } timeout
&& int.TryParse(timeout, out var seconds)
&& seconds == -1)
? Timeout.Infinite
: Environment.GetEnvironmentVariable("READ_WRITE_TIMEOUT_SECONDS") is { } timeoutRetry
&& int.TryParse(timeoutRetry, out var secondsRetry)
? (int)TimeSpan.FromSeconds(secondsRetry).TotalMilliseconds
: (int)TimeSpan.FromSeconds(60).TotalMilliseconds;
Console.WriteLine(LogTimeStamp + "Read Write Timeout set to {0}{1}",
readWriteTimeout == Timeout.Infinite ? "infinite" : readWriteTimeout + " ms",
backend is ITimeoutExemptBackend ? " (because implements ITimeoutExemptBackend)" : "");
// Allow overriding the timeout for the backend here, even if timeouts are disabled
if (options.TryGetValue("read-write-timeout", out var readWriteTimeoutString))
@@ -298,7 +310,7 @@ namespace Duplicati.CommandLine.BackendTester
byte[] dummyFileHash = null;
if (!skipOverwriteTest)
{
Console.WriteLine("Uploading wrong files ...");
Console.WriteLine(LogTimeStamp + "Uploading wrong files ...");
using (var dummy = Library.Utility.TempFile.WrapExistingFile(CreateRandomFile(tf, files.Count, 1024, 2048, rnd)))
{
using (var fs = new System.IO.FileStream(dummy, System.IO.FileMode.Open, System.IO.FileAccess.Read))
@@ -313,7 +325,7 @@ namespace Duplicati.CommandLine.BackendTester
}
Console.WriteLine("Uploading files ...");
Console.WriteLine(LogTimeStamp + "Uploading files ...");
for (int i = 0; i < files.Count; i++)
Uploadfile(files[i].localfilename, i, files[i].remotefilename, backend, disableStreaming, throttleUpload, readWriteTimeout);
@@ -328,7 +340,7 @@ namespace Duplicati.CommandLine.BackendTester
renamedFileNewName = CreateRandomRemoteFileName(min_filename_size, max_filename_size, allowedChars, trimFilenameSpaces, rnd);
Console.WriteLine("Renaming file {0} from {1} to {2}", renameIndex, originalRenamedFile.remotefilename, renamedFileNewName);
Console.WriteLine(LogTimeStamp + "Renaming file {0} from {1} to {2}", renameIndex, originalRenamedFile.remotefilename, renamedFileNewName);
renameEnabledBackend.RenameAsync(originalRenamedFile.remotefilename, renamedFileNewName, CancellationToken.None).Await();
files[renameIndex] = new TempFile(renamedFileNewName, originalRenamedFile.localfilename, originalRenamedFile.hash, originalRenamedFile.length);
@@ -336,13 +348,13 @@ namespace Duplicati.CommandLine.BackendTester
if (waitAfterUpload > TimeSpan.Zero)
{
Console.WriteLine("Waiting {0} after upload", waitAfterUpload);
Console.WriteLine(LogTimeStamp + "Waiting {0} after upload", waitAfterUpload);
Thread.Sleep(waitAfterUpload);
}
Console.WriteLine("Verifying file list ...");
Console.WriteLine(LogTimeStamp + "Verifying file list ...");
curlist = backend.List();
curlist = backend.ListAsync(CancellationToken.None).ToBlockingEnumerable().ToList();
foreach (var fe in curlist)
if (!fe.IsFolder)
{
@@ -351,12 +363,12 @@ namespace Duplicati.CommandLine.BackendTester
if (tx.remotefilename == fe.Name)
{
if (tx.found)
Console.WriteLine("*** File with name {0} was found more than once", tx.remotefilename);
Console.WriteLine(LogTimeStamp + "*** File with name {0} was found more than once", tx.remotefilename);
found = true;
tx.found = true;
if (fe.Size > 0 && tx.length != fe.Size)
Console.WriteLine("*** File with name {0} has size {1} but the size was reported as {2}", tx.remotefilename, tx.length, fe.Size);
Console.WriteLine(LogTimeStamp + "*** File with name {0} has size {1} but the size was reported as {2}", tx.remotefilename, tx.length, fe.Size);
break;
}
@@ -364,26 +376,26 @@ namespace Duplicati.CommandLine.BackendTester
if (!found)
if (originalRenamedFile != null && renamedFileNewName != null && originalRenamedFile.remotefilename == fe.Name)
{
Console.WriteLine("*** File with name {0} was found on server but was supposed to have been renamed to {1}!", fe.Name, renamedFileNewName);
Console.WriteLine(LogTimeStamp + "*** File with name {0} was found on server but was supposed to have been renamed to {1}!", fe.Name, renamedFileNewName);
}
else
{
Console.WriteLine("*** File with name {0} was found on server but not uploaded!", fe.Name);
Console.WriteLine(LogTimeStamp + "*** File with name {0} was found on server but not uploaded!", fe.Name);
}
}
foreach (var tx in files)
if (!tx.found)
Console.WriteLine("*** File with name {0} was uploaded but not found afterwards", tx.remotefilename);
Console.WriteLine(LogTimeStamp + "*** File with name {0} was uploaded but not found afterwards", tx.remotefilename);
Console.WriteLine("Downloading files");
Console.WriteLine(LogTimeStamp + "Downloading files");
for (int i = 0; i < files.Count; i++)
{
using (var cf = new Duplicati.Library.Utility.TempFile())
{
Exception e = null;
Console.Write("Downloading file {0} ... ", i);
Console.Write(LogTimeStamp + "Downloading file {0} ... ", i);
var sw = Stopwatch.StartNew();
@@ -410,22 +422,22 @@ namespace Duplicati.CommandLine.BackendTester
if (e != null)
{
failAfterFinished = true;
Console.WriteLine($"failed\n*** Error: {e} after {sw.ElapsedMilliseconds} ms");
Console.WriteLine(LogTimeStamp + $"failed\n*** Error: {e} after {sw.ElapsedMilliseconds} ms");
}
else
Console.WriteLine($"done in {sw.ElapsedMilliseconds} ms");
Console.WriteLine($" done in {sw.ElapsedMilliseconds} ms");
Console.Write("Checking hash ... ");
Console.Write(LogTimeStamp + "Checking hash ... ");
using (var fs = new System.IO.FileStream(cf, System.IO.FileMode.Open, System.IO.FileAccess.Read))
if (Convert.ToBase64String(sha.ComputeHash(fs)) != Convert.ToBase64String(files[i].hash))
{
if (dummyFileHash != null && Convert.ToBase64String(sha.ComputeHash(fs)) == Convert.ToBase64String(dummyFileHash))
Console.WriteLine("failed\n*** Downloaded file was the dummy file"); // Should this be failed?
Console.WriteLine(LogTimeStamp + "failed\n*** Downloaded file was the dummy file"); // Should this be failed?
else
{
failAfterFinished = true;
Console.WriteLine("failed\n*** Downloaded file was corrupt");
Console.WriteLine(LogTimeStamp + "failed\n*** Downloaded file was corrupt");
}
}
else
@@ -433,31 +445,31 @@ namespace Duplicati.CommandLine.BackendTester
}
}
Console.WriteLine("Deleting files...");
Console.WriteLine(LogTimeStamp + "Deleting files...");
foreach (var tx in files)
try { backend.DeleteAsync(tx.remotefilename, CancellationToken.None).Await(); }
catch (Exception ex)
{
failAfterFinished = true;
Console.WriteLine("*** Failed to delete file {0}, message: {1}", tx.remotefilename, ex);
Console.WriteLine(LogTimeStamp + "*** Failed to delete file {0}, message: {1}", tx.remotefilename, ex);
}
if (waitAfterDelete > TimeSpan.Zero)
{
Console.WriteLine("Waiting {0} after delete", waitAfterDelete);
Console.WriteLine(LogTimeStamp + "Waiting {0} after delete", waitAfterDelete);
Thread.Sleep(waitAfterDelete);
}
curlist = backend.List();
curlist = backend.ListAsync(CancellationToken.None).ToBlockingEnumerable().ToList();
foreach (var fe in curlist)
if (!fe.IsFolder)
{
Console.WriteLine("*** Remote folder contains {0} after cleanup", fe.Name);
Console.WriteLine(LogTimeStamp + "*** Remote folder contains {0} after cleanup", fe.Name);
}
// Test some error cases
Console.WriteLine("Checking retrieval of non-existent file...");
Console.WriteLine(LogTimeStamp + "Checking retrieval of non-existent file...");
var caughtExpectedException = false;
try
{
@@ -468,25 +480,25 @@ namespace Duplicati.CommandLine.BackendTester
}
catch (FileMissingException)
{
Console.WriteLine("Caught expected FileMissingException");
Console.WriteLine(LogTimeStamp + "Caught expected FileMissingException");
caughtExpectedException = true;
}
catch (Exception ex)
{
Console.WriteLine("*** Retrieval of non-existent file failed: {0}", ex);
Console.WriteLine(LogTimeStamp + "*** Retrieval of non-existent file failed: {0}", ex);
}
if (!caughtExpectedException)
{
failAfterFinished = true;
Console.WriteLine("*** Retrieval of non-existent file should have failed with FileMissingException");
Console.WriteLine(LogTimeStamp + "*** Retrieval of non-existent file should have failed with FileMissingException");
}
}
// Test quota retrieval
if (backend is IQuotaEnabledBackend quotaEnabledBackend)
{
Console.WriteLine("Checking quota...");
Console.WriteLine(LogTimeStamp + "Checking quota...");
IQuotaInfo quota = null;
bool noException;
try
@@ -496,7 +508,7 @@ namespace Duplicati.CommandLine.BackendTester
}
catch (Exception ex)
{
Console.WriteLine("*** Checking quota information failed: {0}", ex);
Console.WriteLine(LogTimeStamp + "*** Checking quota information failed: {0}", ex);
noException = false;
}
@@ -504,18 +516,18 @@ namespace Duplicati.CommandLine.BackendTester
{
if (quota != null)
{
Console.WriteLine("Free Space: {0}", Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace));
Console.WriteLine("Total Space: {0}", Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace));
Console.WriteLine(LogTimeStamp + "Free Space: {0}", Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace));
Console.WriteLine(LogTimeStamp + "Total Space: {0}", Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace));
}
else
{
Console.WriteLine("Unable to retrieve quota information");
Console.WriteLine(LogTimeStamp + "Unable to retrieve quota information");
}
}
}
// Test DNSName lookup
Console.WriteLine("Checking DNS names used by this backend...");
Console.WriteLine(LogTimeStamp + "Checking DNS names used by this backend...");
try
{
var dnsNames = backend.GetDNSNamesAsync(CancellationToken.None).Await();
@@ -528,12 +540,12 @@ namespace Duplicati.CommandLine.BackendTester
}
else
{
Console.WriteLine("No DNS names reported");
Console.WriteLine(LogTimeStamp + "No DNS names reported");
}
}
catch (Exception ex)
{
Console.WriteLine("*** Checking DNSName failed: {0}", ex);
Console.WriteLine(LogTimeStamp + "*** Checking DNSName failed: {0}", ex);
}
}
finally
@@ -548,7 +560,7 @@ namespace Duplicati.CommandLine.BackendTester
private static void Uploadfile(string localfilename, int i, string remotefilename, IBackend backend, bool disableStreaming, long throttle, int readWriteTimeout)
{
Console.Write("Uploading file {0}, {1} ... ", i, Duplicati.Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(localfilename).Length));
Console.Write(LogTimeStamp + "Uploading file {0}, {1} ... ", i, Duplicati.Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(localfilename).Length));
Exception e = null;
var sw = Stopwatch.StartNew();
@@ -575,11 +587,11 @@ namespace Duplicati.CommandLine.BackendTester
if (e != null)
{
Console.WriteLine($"Failed to upload file {i}, error message: {e}, remote name: {remotefilename} after {sw.ElapsedMilliseconds} ms");
Console.WriteLine(LogTimeStamp + $"Failed to upload file {i}, error message: {e}, remote name: {remotefilename} after {sw.ElapsedMilliseconds} ms");
while (e.InnerException != null)
{
e = e.InnerException;
Console.WriteLine(" Inner exception: {0}", e);
Console.WriteLine(LogTimeStamp + " Inner exception: {0}", e);
}
}
else
@@ -604,7 +616,7 @@ namespace Duplicati.CommandLine.BackendTester
private static string CreateRandomFile(Library.Utility.TempFolder tf, int i, int min_file_size, int max_file_size, Random rnd)
{
Console.Write("Generating file {0}", i);
Console.Write(LogTimeStamp + "Generating file {0}", i);
string filename = System.IO.Path.Combine(tf, i.ToString());
using (System.IO.FileStream fs = new System.IO.FileStream(filename, System.IO.FileMode.CreateNew, System.IO.FileAccess.Write))
{
@@ -625,6 +637,8 @@ namespace Duplicati.CommandLine.BackendTester
return filename;
}
private static string LogTimeStamp => $"[{DateTime.Now:HH:mm:ss fff}] ";
public static IList<ICommandLineArgument> SupportedCommands
{
get
Duplicati/CommandLine/BackendTool/Program.cs
+22 -21
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Interface;
@@ -26,6 +26,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.CommandLine.BackendTool
{
@@ -74,7 +75,7 @@ namespace Duplicati.CommandLine.BackendTool
}
if (args.Count < 2 || String.Equals(args[0], "help", StringComparison.OrdinalIgnoreCase) || args[0] == "?" || command == null)
if (args.Count < 2 || HelpOptionExtensions.IsArgumentAnyHelpString(args) || command == null)
{
if (command == null && args.Count >= 2)
{
@@ -124,7 +125,7 @@ namespace Duplicati.CommandLine.BackendTool
throw new UserInformationException(string.Format("too many arguments: {0}", string.Join(",", args)), "BackendToolTooManyArguments");
Console.WriteLine("{0}\t{1}\t{2}\t{3}", "Name", "Dir/File", "LastChange", "Size");
foreach (var e in backend.List())
foreach (var e in backend.ListAsync(CancellationToken.None).ToBlockingEnumerable())
Console.WriteLine("{0}\t{1}\t{2}\t{3}", e.Name, e.IsFolder ? "Dir" : "File", e.LastModification, e.Size < 0 ? "" : Library.Utility.Utility.FormatSizeString(e.Size));
return 0;
Duplicati/CommandLine/CLI/Commands.cs
+25 -24
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
@@ -163,7 +163,7 @@ namespace Duplicati.CommandLine
options.TryGetValue("dbpath", out dbpath);
if (string.IsNullOrEmpty(dbpath))
{
dbpath = Library.Main.DatabaseLocator.GetDatabasePath(backend, new Duplicati.Library.Main.Options(options), false, true);
dbpath = Library.Main.CLIDatabaseLocator.GetDatabasePathForCLI(backend, new Library.Main.Options(options), false, true);
if (dbpath != null)
options["dbpath"] = dbpath;
}
@@ -366,7 +366,7 @@ namespace Duplicati.CommandLine
options.TryGetValue("dbpath", out dbpath);
if (string.IsNullOrEmpty(dbpath))
{
dbpath = Library.Main.DatabaseLocator.GetDatabasePath(backend, new Duplicati.Library.Main.Options(options), false, true);
dbpath = Library.Main.CLIDatabaseLocator.GetDatabasePathForCLI(backend, new Duplicati.Library.Main.Options(options), false, true);
if (dbpath != null)
options["dbpath"] = dbpath;
}
@@ -622,7 +622,7 @@ namespace Duplicati.CommandLine
switch (phase)
{
case Duplicati.Library.Main.OperationPhase.Backup_ProcessingFiles:
output.MessageEvent("Scanning local files ...");
output.MessageEvent("Scanning source files ...");
periodicOutput.SetReady();
break;
case Duplicati.Library.Main.OperationPhase.Backup_Finalize:
@@ -668,6 +668,7 @@ namespace Duplicati.CommandLine
if (parsedStats.KnownFileCount > 0)
{
output.MessageEvent(string.Format(" Remote files: {0}", parsedStats.KnownFileCount));
output.MessageEvent(string.Format(" Remote filesets: {0}", parsedStats.KnownFilesets));
output.MessageEvent(string.Format(" Remote size: {0}", Library.Utility.Utility.FormatSizeString(parsedStats.KnownFileSize)));
}
@@ -820,7 +821,7 @@ namespace Duplicati.CommandLine
if (string.IsNullOrEmpty(dbpath))
{
if (args.Count > 0)
dbpath = Library.Main.DatabaseLocator.GetDatabasePath(args[0], new Duplicati.Library.Main.Options(options), false, true);
dbpath = Library.Main.CLIDatabaseLocator.GetDatabasePathForCLI(args[0], new Duplicati.Library.Main.Options(options), false, true);
if (dbpath == null)
{
@@ -866,7 +867,7 @@ namespace Duplicati.CommandLine
options.TryGetValue("dbpath", out dbpath);
if (string.IsNullOrEmpty(dbpath))
{
dbpath = Library.Main.DatabaseLocator.GetDatabasePath(args[0], new Duplicati.Library.Main.Options(options), false, true);
dbpath = Library.Main.CLIDatabaseLocator.GetDatabasePathForCLI(args[0], new Duplicati.Library.Main.Options(options), false, true);
if (dbpath != null)
options["dbpath"] = dbpath;
}
Duplicati/CommandLine/CLI/Help.cs
+20 -20
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
@@ -116,7 +116,7 @@ namespace Duplicati.CommandLine
Library.Main.Options opts = new Library.Main.Options(new Dictionary<string, string>());
tp = tp.Replace("%CLI_EXE%", PackageHelper.GetExecutableName(PackageHelper.NamedExecutable.CommandLine));
tp = tp.Replace("%VERSION%", License.VersionNumbers.Version);
tp = tp.Replace("%VERSION%", License.VersionNumbers.VERSION_NAME);
tp = tp.Replace("%BACKENDS%", string.Join(", ", Library.DynamicLoader.BackendLoader.Keys));
tp = tp.Replace("%APP_PATH%", Path.Combine(UpdaterManager.INSTALLATIONDIR, PackageHelper.GetExecutableName(PackageHelper.NamedExecutable.CommandLine)));
tp = tp.Replace("%PATH_SEPARATOR%", System.IO.Path.PathSeparator.ToString());
Duplicati/CommandLine/CLI/Program.cs
+36 -34
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
@@ -27,11 +27,13 @@ using System.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.AutoUpdater;
using Duplicati.Library.Crashlog;
using Duplicati.Library.Utility;
namespace Duplicati.CommandLine
{
public class Program
{
private static readonly string LOGTAG = Library.Logging.Log.LogTagFromType<Program>();
public static bool FROM_COMMANDLINE = false;
@@ -163,7 +165,7 @@ namespace Duplicati.CommandLine
}
// Probe for "help" to avoid extra processing
if (cargs.Count == 0 || (string.Equals(cargs[0], "help", StringComparison.OrdinalIgnoreCase)))
if (cargs.Count == 0 || HelpOptionExtensions.IsArgumentAnyHelpString(cargs))
{
return Commands.Help(outwriter, setup, cargs, options, filter);
}
@@ -181,6 +183,10 @@ namespace Duplicati.CommandLine
}
}
// Don't pass these options into the controller
options.Remove(Library.AutoUpdater.DataFolderManager.SERVER_DATAFOLDER_OPTION);
options.Remove(Library.AutoUpdater.DataFolderManager.PORTABLE_MODE_OPTION);
if (!options.ContainsKey("passphrase"))
if (!string.IsNullOrEmpty(System.Environment.GetEnvironmentVariable("PASSPHRASE")))
options["passphrase"] = System.Environment.GetEnvironmentVariable("PASSPHRASE");
@@ -283,20 +289,16 @@ namespace Duplicati.CommandLine
}
}
public static IList<Library.Interface.ICommandLineArgument> SupportedOptions
{
get
{
return new List<Library.Interface.ICommandLineArgument>(new Library.Interface.ICommandLineArgument[] {
new Library.Interface.CommandLineArgument("parameters-file", Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.Program.ParametersFileOptionShort, Strings.Program.ParametersFileOptionLong2, "", new string[] {"parameter-file", "parameterfile"}),
new Library.Interface.CommandLineArgument("include", Library.Interface.CommandLineArgument.ArgumentType.String, Strings.Program.IncludeShort, Strings.Program.IncludeLong),
new Library.Interface.CommandLineArgument("exclude", Library.Interface.CommandLineArgument.ArgumentType.String, Strings.Program.ExcludeShort, Strings.Program.ExcludeLong),
new Library.Interface.CommandLineArgument("control-files", Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.ControlFilesOptionShort, Strings.Program.ControlFilesOptionLong, "false"),
new Library.Interface.CommandLineArgument("quiet-console", Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.QuietConsoleOptionShort, Strings.Program.QuietConsoleOptionLong, "false"),
new Library.Interface.CommandLineArgument("auto-update", Library.Interface.CommandLineArgument.ArgumentType.Boolean, LC.L("Toggle automatic updates"), LC.L("Set this option if you prefer to have the commandline version automatically update"), "false"),
});
}
}
public static IList<ICommandLineArgument> SupportedOptions => [
new CommandLineArgument("parameters-file", CommandLineArgument.ArgumentType.Path, Strings.Program.ParametersFileOptionShort, Strings.Program.ParametersFileOptionLong2, "", new string[] {"parameter-file", "parameterfile"}),
new CommandLineArgument("include", CommandLineArgument.ArgumentType.String, Strings.Program.IncludeShort, Strings.Program.IncludeLong),
new CommandLineArgument("exclude", CommandLineArgument.ArgumentType.String, Strings.Program.ExcludeShort, Strings.Program.ExcludeLong),
new CommandLineArgument("control-files", CommandLineArgument.ArgumentType.Boolean, Strings.Program.ControlFilesOptionShort, Strings.Program.ControlFilesOptionLong, "false"),
new CommandLineArgument("quiet-console", CommandLineArgument.ArgumentType.Boolean, Strings.Program.QuietConsoleOptionShort, Strings.Program.QuietConsoleOptionLong, "false"),
new CommandLineArgument("auto-update", CommandLineArgument.ArgumentType.Boolean, Strings.Program.AutoUpdateOptionShort, Strings.Program.AutoUpdateOptionLong, "false"),
new CommandLineArgument(DataFolderManager.PORTABLE_MODE_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.Program.PortableModeOptionShort, Strings.Program.PortableModeOptionLong, DataFolderManager.PORTABLE_MODE.ToString().ToLowerInvariant()),
new CommandLineArgument(DataFolderManager.SERVER_DATAFOLDER_OPTION, CommandLineArgument.ArgumentType.Path, Strings.Program.DataFolderOptionShort, Strings.Program.DataFolderOptionLong, DataFolderManager.DATAFOLDER),
];
private static bool ReadOptionsFromFile(TextWriter outwriter, string filename, ref Library.Utility.IFilter filter, List<string> cargs, Dictionary<string, string> options)
{
Duplicati/CommandLine/CLI/Strings.cs
+25 -19
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Localization.Short;
@@ -54,6 +54,12 @@ namespace Duplicati.CommandLine.Strings
public static string QuietConsoleOptionLong { get { return LC.L(@"If this option is set, progress reports and other messages that would normally go to the console will be redirected to the log."); } }
public static string QuietConsoleOptionShort { get { return LC.L(@"Disable console output"); } }
public static string SkippingSourceArgumentsOnNonBackupOperation { get { return @"The --source argument was specified in the parameter file, but the current operation is not a backup operation, so the argument is ignored."; } }
public static string AutoUpdateOptionShort { get { return LC.L(@"Toggle automatic updates"); } }
public static string AutoUpdateOptionLong { get { return LC.L(@"Set this option if you prefer to have the commandline version automatically update"); } }
public static string PortableModeOptionShort { get { return LC.L(@"Use portable mode"); } }
public static string PortableModeOptionLong { get { return LC.L(@"If this option is set, the configuration files will be stored in a data subfolder of the Duplicati installation folder. This is useful for running from a USB stick or other portable media."); } }
public static string DataFolderOptionShort { get { return LC.L(@"Data folder"); } }
public static string DataFolderOptionLong { get { return LC.L(@"The folder where data is stored. This is the folder where the database and other files are stored."); } }
// ReSharper disable once UnusedMember.Global
// This is a placeholder message that is intended to be used with the code
Duplicati/CommandLine/RecoveryTool/Download.cs
+21 -20
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
@@ -25,6 +25,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Threading;
using Duplicati.Library.Utility;
using System.Threading.Tasks;
namespace Duplicati.CommandLine.RecoveryTool
{
@@ -56,7 +57,7 @@ namespace Duplicati.CommandLine.RecoveryTool
Console.WriteLine("Listing files on backend: {0} ...", backend.ProtocolKey);
var lst = backend.List().ToList();
var lst = backend.ListAsync(CancellationToken.None).ToBlockingEnumerable().ToList();
Console.WriteLine("Found {0} files", lst.Count);
Duplicati/CommandLine/RecoveryTool/Recompress.cs
+2 -2
View File
@@ -29,7 +29,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Security.Cryptography;
using System.Threading.Tasks;
namespace Duplicati.CommandLine.RecoveryTool
{
@@ -71,7 +71,7 @@ namespace Duplicati.CommandLine.RecoveryTool
Console.WriteLine("Listing files on backend: {0} ...", backend.ProtocolKey);
var rawlist = backend.List().ToList();
var rawlist = backend.ListAsync(CancellationToken.None).ToBlockingEnumerable().ToList();
Console.WriteLine("Found {0} files at remote storage", rawlist.Count);
Duplicati/CommandLine/SecretTool/Program.cs
+5 -1
View File
@@ -25,6 +25,8 @@ using System.CommandLine.NamingConventionBinder;
using System.CommandLine.Parsing;
using Duplicati.Library.DynamicLoader;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using Uri = System.Uri;
namespace Duplicati.CommandLine.SecretTool;
@@ -74,7 +76,9 @@ public static class Program
Console.WriteLine("Exception: " + ex);
context.ExitCode = 1;
}
}).Build()
})
.UseAdditionalHelpAliases()
.Build()
.InvokeAsync(args);
}
Duplicati/CommandLine/ServerUtil/Connection.cs
+6 -6
View File
@@ -21,6 +21,7 @@
using System.Net.Http.Json;
using System.Net.Security;
using System.Text.Json;
using Duplicati.Library.AutoUpdater;
namespace Duplicati.CommandLine.ServerUtil;
@@ -185,10 +186,9 @@ public class Connection
try
{
var opts = new Dictionary<string, string>();
if (!string.IsNullOrWhiteSpace(settings.ServerDatafolder))
opts.Add("server-datafolder", settings.ServerDatafolder);
if (File.Exists(Path.Combine(Server.Program.GetDataFolderPath(opts), Server.Program.SERVER_DATABASE_FILENAME)))
if (settings.Key != null)
opts["settings-encryption-key"] = settings.Key.Key;
if (File.Exists(Path.Combine(DataFolderManager.DATAFOLDER, DataFolderManager.SERVER_DATABASE_FILENAME)))
{
string? cfg = null;
using (var connection = Duplicati.Server.Program.GetDatabaseConnection(opts, true))
@@ -220,9 +220,9 @@ public class Connection
return CreateConnectionWithClient(client, accessToken);
}
}
else if (!string.IsNullOrWhiteSpace(settings.ServerDatafolder))
else if (!string.IsNullOrWhiteSpace(DataFolderManager.DATAFOLDER))
{
Console.WriteLine($"No database found in {settings.ServerDatafolder}");
Console.WriteLine($"No database found in {DataFolderManager.DATAFOLDER}");
}
}
catch (Exception ex)
Duplicati/CommandLine/ServerUtil/Program.cs
+2
View File
@@ -22,6 +22,7 @@ using System.CommandLine;
using System.CommandLine.Builder;
using System.CommandLine.Parsing;
using Duplicati.CommandLine.ServerUtil.Commands;
using Duplicati.Library.Utility;
namespace Duplicati.CommandLine.ServerUtil;
@@ -80,6 +81,7 @@ public static class Program
await next(context);
})
.UseAdditionalHelpAliases()
.Build()
.InvokeAsync(args);
}
Duplicati/CommandLine/ServerUtil/Settings.cs
+13 -17
View File
@@ -19,11 +19,13 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.Text.Json;
using Duplicati.Library.AutoUpdater;
using Duplicati.Library.DynamicLoader;
using Duplicati.Library.Encryption;
using Duplicati.Library.Interface;
using Duplicati.Library.Main;
using Duplicati.Library.Utility;
using Utility = Duplicati.Library.Main.Utility;
namespace Duplicati.CommandLine.ServerUtil;
@@ -33,7 +35,6 @@ namespace Duplicati.CommandLine.ServerUtil;
/// <param name="Password">The commandline password</param>
/// <param name="RefreshToken">The saved refresh token</param>
/// <param name="HostUrl">The host url to connect to</param>
/// <param name="ServerDatafolder">The server datafolder for password-free connections</param>
/// <param name="SettingsFile">The settings file where data is loaded/saved</param>
/// <param name="Insecure">Whether to disable TLS/SSL certificate trust check</param>
/// <param name="Key">The encryption key to use for the settings file</param>
@@ -44,7 +45,6 @@ public sealed record Settings(
string? Password,
string? RefreshToken,
System.Uri HostUrl,
string? ServerDatafolder,
string SettingsFile,
bool Insecure,
EncryptedFieldHelper.KeyInstance? Key,
@@ -64,21 +64,21 @@ public sealed record Settings(
System.Uri HostUrl,
string? ServerDatafolder
);
private static string GetDefaultStorageFolder(string filename)
{
var folder = DatabaseLocator.GetDefaultStorageFolderWithDebugSupport(filename);
if (!Directory.Exists(folder))
Directory.CreateDirectory(folder);
return folder;
}
/// <summary>
/// Gets the default data folder for the settings file
/// </summary>
/// <param name="filename">The filename to use</param>
/// <returns>The default storage folder</returns>
private static string GetDefaultStorageFolder(string filename)
// Ideally, this should use DataFolderManager.DATAFOLDER, but we cannot due to backwards compatibility
=> DataFolderLocator.GetDefaultStorageFolder(filename, true);
/// <summary>
/// Loads the settings from the settings file
/// </summary>
/// <param name="password">The password to use</param>
/// <param name="hostUrl">The host URL to use</param>
/// <param name="serverDataFolder">The server data folder to use</param>
/// <param name="settingsFile">The settings file to use</param>
/// <param name="insecure">Whether to disable TLS/SSL certificate trust check</param>
/// <param name="settingsPassphrase">The encryption key to use</param>
@@ -87,9 +87,9 @@ public sealed record Settings(
/// <param name="secretProviderPattern">The secret provider pattern to use</param>
/// <param name="acceptedHostCertificate">The SHA1 hash of the host certificate to accept</param>
/// <returns>The loaded settings</returns>
public static Settings Load(string? password, System.Uri? hostUrl, string? serverDataFolder, string settingsFile, bool insecure, string? settingsPassphrase, string? secretProvider, SecretProviderHelper.CachingLevel secretProviderCache, string secretProviderPattern, string? acceptedHostCertificate)
public static Settings Load(string? password, System.Uri? hostUrl, string settingsFile, bool insecure, string? settingsPassphrase, string? secretProvider, SecretProviderHelper.CachingLevel secretProviderCache, string secretProviderPattern, string? acceptedHostCertificate)
{
hostUrl ??= new System.Uri("http://localhost:8200");
hostUrl ??= new System.Uri($"http://{Library.Utility.Utility.IpVersionCompatibleLoopback}:8200");
ISecretProvider? secretInstance = null;
if (!string.IsNullOrWhiteSpace(secretProvider))
@@ -115,9 +115,6 @@ public sealed record Settings(
settingsPassphrase = opts["settings-encryption-key"];
}
if (string.IsNullOrWhiteSpace(serverDataFolder))
serverDataFolder = GetDefaultStorageFolder("Duplicati-server.sqlite");
if (!string.IsNullOrWhiteSpace(settingsFile) && !Path.IsPathRooted(settingsFile))
settingsFile = Path.Combine(GetDefaultStorageFolder(settingsFile), settingsFile);
@@ -129,7 +126,6 @@ public sealed record Settings(
password,
persistedSettings?.RefreshToken,
hostUrl,
serverDataFolder,
settingsFile,
insecure,
key,
@@ -175,7 +171,7 @@ public sealed record Settings(
File.WriteAllText(SettingsFile, JsonSerializer.Serialize(LoadSettings(SettingsFile, thisKey)
.Where(x => x.HostUrl != HostUrl)
.Append(new PersistedSettings(RefreshToken, HostUrl, ServerDatafolder))
.Append(new PersistedSettings(RefreshToken, HostUrl, DataFolderManager.DATAFOLDER))
.Select(x => x with
{
RefreshToken = string.IsNullOrWhiteSpace(x.RefreshToken) || thisKey == null
Duplicati/CommandLine/ServerUtil/SettingsBinder.cs
+8 -3
View File
@@ -20,6 +20,7 @@
// DEALINGS IN THE SOFTWARE.
using System.CommandLine;
using System.CommandLine.Binding;
using Duplicati.Library.AutoUpdater;
using Duplicati.Library.Main;
namespace Duplicati.CommandLine.ServerUtil;
@@ -36,11 +37,15 @@ public class SettingsBinder : BinderBase<Settings>
/// <summary>
/// The host URL option.
/// </summary>
public static readonly Option<Uri> hostUrlOption = new Option<Uri>("--hosturl", description: "The host URL to use", getDefaultValue: () => new Uri("http://localhost:8200"));
public static readonly Option<Uri> hostUrlOption = new Option<Uri>("--hosturl", description: "The host URL to use", getDefaultValue: () => new Uri($"http://{Library.Utility.Utility.IpVersionCompatibleLoopback}:8200"));
/// <summary>
/// The server datafolder option.
/// </summary>
public static readonly Option<DirectoryInfo?> serverDatafolderOption = new Option<DirectoryInfo?>("--server-datafolder", description: "The server datafolder to use for locating the database", getDefaultValue: () => null);
public static readonly Option<DirectoryInfo?> serverDatafolderOption = new Option<DirectoryInfo?>($"--{DataFolderManager.SERVER_DATAFOLDER_OPTION}", description: "The server datafolder to use for locating the database and storing configuration", getDefaultValue: () => new DirectoryInfo(DataFolderManager.DATAFOLDER));
/// <summary>
/// The server portable mode option.
/// </summary>
public static readonly Option<bool> portableModeOption = new Option<bool>($"--{DataFolderManager.PORTABLE_MODE_OPTION}", description: "Use portable mode for locating the database and storing configuration", getDefaultValue: () => DataFolderManager.PORTABLE_MODE);
/// <summary>
/// The settings file option.
/// </summary>
@@ -84,6 +89,7 @@ public class SettingsBinder : BinderBase<Settings>
rootCommand.AddGlobalOption(passwordOption);
rootCommand.AddGlobalOption(hostUrlOption);
rootCommand.AddGlobalOption(serverDatafolderOption);
rootCommand.AddGlobalOption(portableModeOption);
rootCommand.AddGlobalOption(settingsFileOption);
rootCommand.AddGlobalOption(insecureOption);
rootCommand.AddGlobalOption(settingsEncryptionKeyOption);
@@ -103,7 +109,6 @@ public class SettingsBinder : BinderBase<Settings>
Settings.Load(
bindingContext.ParseResult.GetValueForOption(passwordOption),
bindingContext.ParseResult.GetValueForOption(hostUrlOption),
bindingContext.ParseResult.GetValueForOption(serverDatafolderOption)?.FullName,
bindingContext.ParseResult.GetValueForOption(settingsFileOption)?.FullName ?? "settings.json",
bindingContext.ParseResult.GetValueForOption(insecureOption),
bindingContext.ParseResult.GetValueForOption(settingsEncryptionKeyOption) ?? Environment.GetEnvironmentVariable(Library.Encryption.EncryptedFieldHelper.ENVIROMENT_VARIABLE_NAME),
Duplicati/CommandLine/SourceTool/CommandExtensions.cs
+42
View File
@@ -0,0 +1,42 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.CommandLine;
using System.CommandLine.Invocation;
namespace Duplicati.CommandLine.SourceTool;
/// <summary>
/// Extensions for <see cref="Command"/>.
/// </summary>
public static class CommandExtensions
{
/// <summary>
/// Adds the missing WithHandler method to <see cref="Command"/>.
/// </summary>
/// <param name="command">The command to add the handler to.</param>
/// <param name="handler">The handler to add.</param>
/// <returns>The command with the handler added.</returns>
public static Command WithHandler(this Command command, ICommandHandler handler)
{
command.Handler = handler;
return command;
}
}
Duplicati/CommandLine/SourceTool/Commands/Download.cs
+110
View File
@@ -0,0 +1,110 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.CommandLine;
using System.CommandLine.NamingConventionBinder;
namespace Duplicati.CommandLine.SourceTool.Commands;
/// <summary>
/// The download command
/// </summary>
public static class Download
{
/// <summary>
/// Creates the download command
/// </summary>
/// <returns>The download command</returns>
public static Command Create() =>
new Command("download", "Downloads all files from the remote")
{
new Argument<string>("url", "The source URL") {
Arity = ArgumentArity.ExactlyOne
},
new Option<DirectoryInfo>("--destination", description: "The destination folder", getDefaultValue: () => new DirectoryInfo(Directory.GetCurrentDirectory())),
new Option<int>("--max-depth", description: "The maximum depth to visit", getDefaultValue: () => 0),
new Option<long>("--max-size", description: "The maximum filesize to download", getDefaultValue: () => 0),
new Option<bool>("--overwrite", description: "Overwrite existing files", getDefaultValue: () => false),
}
.WithHandler(CommandHandler.Create<string, DirectoryInfo, int, long, bool>(async (url, destination, maxdepth, maxsize, overwrite) =>
{
var token = new CancellationTokenSource().Token;
using var source = await Common.GetProvider(url);
string localPath(string path)
{
var relpath = path.Substring(source.MountedPath.Length).TrimStart(Path.DirectorySeparatorChar);
return Path.Combine(destination.FullName, relpath);
}
var started = DateTime.UtcNow;
var fileCount = 0L;
var folderCount = 0L;
var totalSize = 0L;
var downloadCount = 0L;
var downloadSize = 0L;
await Common.Visit(source, maxdepth, async (entry, level) =>
{
if (entry.IsMetaEntry)
return true;
var path = localPath(entry.Path);
if (entry.IsFolder)
{
folderCount++;
if (!Directory.Exists(path))
Directory.CreateDirectory(path);
}
else
{
fileCount++;
totalSize += entry.Size;
if (entry.Size <= maxsize || maxsize <= 0)
{
var folder = Path.GetDirectoryName(path);
if (!string.IsNullOrWhiteSpace(folder) && !Directory.Exists(folder))
Directory.CreateDirectory(folder);
if (!overwrite && File.Exists(path))
{
Console.WriteLine($"Skipping {entry.Path} as it already exists");
}
else
{
downloadCount++;
downloadSize += entry.Size;
Console.WriteLine($"Downloading {entry.Path} to {path} ({Library.Utility.Utility.FormatSizeString(entry.Size)})");
await using var stream = await entry.OpenRead(token);
await using var dest = File.OpenWrite(path);
await stream.CopyToAsync(dest, token);
}
}
}
return true;
}, token);
Console.WriteLine($"Found {fileCount} files and {folderCount} folders with a total size of {Library.Utility.Utility.FormatSizeString(totalSize)}");
Console.WriteLine($"Downloaded {downloadCount} files with a total size of {Library.Utility.Utility.FormatSizeString(downloadSize)} in {DateTime.UtcNow - started}");
}));
}
Duplicati/CommandLine/SourceTool/Commands/List.cs
+67
View File
@@ -0,0 +1,67 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.CommandLine;
using System.CommandLine.NamingConventionBinder;
namespace Duplicati.CommandLine.SourceTool.Commands;
/// <summary>
/// The list command
/// </summary>
public static class List
{
/// <summary>
/// Creates the list command
/// </summary>
/// <returns>The command</returns>
public static Command Create() =>
new Command("list", "Lists all paths on the remote")
{
new Argument<string>("url", "The source URL") {
Arity = ArgumentArity.ExactlyOne
},
new Option<int>("--max-depth", description: "The maximum depth to list", getDefaultValue: () => 0),
}
.WithHandler(CommandHandler.Create<string, int>(async (url, maxdepth) =>
{
var token = new CancellationTokenSource().Token;
var folders = 0L;
var files = 0L;
using var source = await Common.GetProvider(url);
await Common.Visit(source, maxdepth, (entry, level) =>
{
if (entry.IsFolder)
folders++;
else
files++;
if (entry.IsFolder && level > 0)
level--;
if (!entry.IsRootEntry)
Console.WriteLine($"{new string(' ', (level + 1) * 2)}{entry.Path}");
return Task.FromResult(true);
}, token);
Console.WriteLine($"Found {folders} folders and {files} files");
})
);
}
Duplicati/CommandLine/SourceTool/Duplicati.CommandLine.SourceTool.csproj
+25
View File
@@ -0,0 +1,25 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<AssemblyName>Duplicati.CommandLine.SourceTool.Implementation</AssemblyName>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<RootNamespace>Duplicati.CommandLine.Sourcetool</RootNamespace>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
<PackageReference Include="System.CommandLine.NamingConventionBinder" Version="2.0.0-beta4.22272.1" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../../Library/DynamicLoader/Duplicati.Library.DynamicLoader.csproj" />
<ProjectReference Include="../../Library/SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj" />
<ProjectReference Include="../../Library/SourceProviders/Duplicati.Library.SourceProviders.csproj" />
<ProjectReference Include="../../Library/Utility/Duplicati.Library.Utility.csproj" />
</ItemGroup>
</Project>
Duplicati/CommandLine/SourceTool/GetProvider.cs
+39
View File
@@ -0,0 +1,39 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.DynamicLoader;
using Duplicati.Library.Interface;
namespace Duplicati.CommandLine.SourceTool;
/// <summary>
/// Common methods for the source tool
/// </summary>
public static partial class Common
{
/// <summary>
/// Gets the source provider for the given URL
/// </summary>
/// <param name="url">The URL to get the provider for</param>
/// <returns>The source provider</returns>
public static async Task<ISourceProvider> GetProvider(string url) =>
(await SourceProviderLoader.GetSourceProvider(url, "", [], CancellationToken.None))
?? throw new UserInformationException(string.Format("The source \"{0}\" is not supported", url), "SourceNotSupported");
}
Duplicati/CommandLine/SourceTool/ListEnumerator.cs
+60
View File
@@ -0,0 +1,60 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Interface;
namespace Duplicati.CommandLine.SourceTool;
/// <summary>
/// Common methods for the source tool
/// </summary>
public static partial class Common
{
/// <summary>
/// Visits all entries in the source provider
/// </summary>
/// <param name="source">The source provider to visit</param>
/// <param name="maxdepth">The maximum depth to visit</param>
/// <param name="visitor">The visitor function</param>
/// <param name="token">The cancellation token</param>
/// <returns>An awaitable task</returns>
public static async Task Visit(ISourceProvider source, int maxdepth, Func<ISourceProviderEntry, int, Task<bool>> visitor, CancellationToken token)
{
var visit = new Stack<(ISourceProviderEntry Entry, int Level)>();
await foreach (var item in source.Enumerate(token))
visit.Push((item, 0));
while (visit.Count() != 0)
{
var item = visit.Pop();
var process = await visitor(item.Entry, item.Level);
if (item.Entry.IsFolder && process && (item.Level < maxdepth || maxdepth <= 0))
{
await foreach (var subitem in item.Entry.Enumerate(token))
{
if (subitem.IsFolder)
visit.Push((subitem, item.Level + 1));
else
await visitor(subitem, item.Level);
}
}
}
}
}
Duplicati/CommandLine/SourceTool/Program.cs
+68
View File
@@ -0,0 +1,68 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.CommandLine;
using System.CommandLine.Builder;
using System.CommandLine.Parsing;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
namespace Duplicati.CommandLine.SourceTool;
/// <summary>
/// The entry point of the application
/// </summary>
public static class Program
{
/// <summary>
/// Executes the program
/// </summary>
/// <param name="args">The arguments</param>
/// <returns>The return code</returns>
public static Task<int> Main(string[] args)
{
Library.AutoUpdater.PreloadSettingsLoader.ConfigurePreloadSettings(ref args, Library.AutoUpdater.PackageHelper.NamedExecutable.ServerUtil);
var rootCmd = new RootCommand("Source CLI tool for Duplicati")
{
Commands.List.Create(),
Commands.Download.Create(),
};
return new CommandLineBuilder(rootCmd)
.UseDefaults()
.UseExceptionHandler((ex, context) =>
{
if (ex is UserInformationException uie)
{
Console.WriteLine(uie.Message);
context.ExitCode = 2;
}
else
{
Console.WriteLine(ex.ToString());
context.ExitCode = 1;
}
})
.UseAdditionalHelpAliases()
.Build()
.InvokeAsync(args);
}
}
Duplicati/GUI/Duplicati.GUI.TrayIcon/Assets/icons/linux/normal-disconnected.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 10 KiB

Duplicati/GUI/Duplicati.GUI.TrayIcon/Assets/icons/macos/dark/normal-disconnected.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 10 KiB

Duplicati/GUI/Duplicati.GUI.TrayIcon/Assets/icons/macos/light/normal-disconnected.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 5.5 KiB

Duplicati/GUI/Duplicati.GUI.TrayIcon/Assets/icons/windows/normal-disconnected.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 12 KiB

Duplicati/GUI/Duplicati.GUI.TrayIcon/AvaloniaRunner.cs
+47 -15
View File
@@ -19,6 +19,8 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Collections.Generic;
using System.Linq;
@@ -40,7 +42,7 @@ namespace Duplicati.GUI.TrayIcon
public class AvaloniaRunner : TrayIconBase
{
private static readonly string LOGTAG = Log.LogTagFromType<AvaloniaRunner>();
private AvaloniaApp application;
private AvaloniaApp? application;
private ProcessBasedActionDelay actionDelayer = new ProcessBasedActionDelay();
private IEnumerable<AvaloniaMenuItem> menuItems = Enumerable.Empty<AvaloniaMenuItem>();
@@ -106,6 +108,8 @@ namespace Duplicati.GUI.TrayIcon
#endif
application = builder.Instance as AvaloniaApp;
if (application == null)
throw new InvalidOperationException("Failed to create Avalonia app");
application.SetMenu(menuItems);
application.Configure();
@@ -114,7 +118,7 @@ namespace Duplicati.GUI.TrayIcon
private async void CheckForAppInitialized(IClassicDesktopStyleApplicationLifetime lifetime)
{
Exception lastEx = null;
Exception? lastEx = null;
// Jump out of the UI thread, ot ensure that the check is done with posting to the UI thread,
// as opposed to just calling the method directly
@@ -164,7 +168,7 @@ namespace Duplicati.GUI.TrayIcon
return new AvaloniaMenuItem(this, text, icon, callback, subitems);
}
protected override void NotifyUser(string title, string message, NotificationType type)
public override void NotifyUser(string title, string message, NotificationType type)
{
//var icon = Win32NativeNotifyIcon.InfoFlags.NIIF_INFO;
@@ -180,8 +184,6 @@ namespace Duplicati.GUI.TrayIcon
//icon = Win32NativeNotifyIcon.InfoFlags.NIIF_ERROR;
break;
}
//m_ntfIcon.ShowBalloonTip(title, message, icon);
}
protected override void Exit()
@@ -209,7 +211,7 @@ namespace Duplicati.GUI.TrayIcon
internal static string GetThemePath()
{
var isDark = string.Equals(Application.Current.ActualThemeVariant.ToString(), "Dark", StringComparison.OrdinalIgnoreCase);
var isDark = string.Equals(Application.Current?.ActualThemeVariant.ToString(), "Dark", StringComparison.OrdinalIgnoreCase);
if (OperatingSystem.IsMacOS())
return isDark ? "macos/dark" : "macos/light";
@@ -237,11 +239,12 @@ namespace Duplicati.GUI.TrayIcon
public string Text { get; private set; }
public Action Callback { get; private set; }
public IList<IMenuItem> SubItems { get; private set; }
public IList<IMenuItem>? SubItems { get; private set; }
public bool Enabled { get; private set; }
public MenuIcons Icon { get; private set; }
public bool IsDefault { get; private set; }
private NativeMenuItem nativeMenuItem;
public bool Hidden { get; private set; }
private NativeMenuItem? nativeMenuItem;
private readonly AvaloniaRunner parent;
public AvaloniaMenuItem(AvaloniaRunner parent, string text, MenuIcons icon, Action callback, IList<IMenuItem> subitems)
@@ -278,6 +281,9 @@ namespace Duplicati.GUI.TrayIcon
/// </summary>
private void UpdateIcon()
{
if (nativeMenuItem == null)
return;
nativeMenuItem.Icon = this.Icon switch
{
MenuIcons.Status => AvaloniaRunner.LoadBitmap("context-menu-open.png"),
@@ -301,6 +307,14 @@ namespace Duplicati.GUI.TrayIcon
// Not currently supported by Avalonia,
// used to set the menu bold on Windows to indicate the default entry
}
public void SetHidden(bool hidden)
{
this.Hidden = hidden;
if (nativeMenuItem != null)
parent.RunOnUIThread(() => nativeMenuItem.IsVisible = !this.Hidden);
}
#endregion
public NativeMenuItem GetNativeItem()
@@ -324,9 +338,9 @@ namespace Duplicati.GUI.TrayIcon
public class AvaloniaApp : Application
{
private Avalonia.Controls.TrayIcon trayIcon;
private Avalonia.Controls.TrayIcon? trayIcon;
private List<AvaloniaMenuItem> menuItems;
private List<AvaloniaMenuItem>? menuItems;
public override void Initialize()
{
@@ -343,6 +357,9 @@ namespace Duplicati.GUI.TrayIcon
case TrayIcons.IdleError:
this.trayIcon.Icon = AvaloniaRunner.LoadIcon("normal-error.png");
break;
case TrayIcons.IdleWarning:
this.trayIcon.Icon = AvaloniaRunner.LoadIcon("normal-warning.png");
break;
case TrayIcons.Paused:
case TrayIcons.PausedError:
this.trayIcon.Icon = AvaloniaRunner.LoadIcon("normal-pause.png");
@@ -355,13 +372,17 @@ namespace Duplicati.GUI.TrayIcon
default:
this.trayIcon.Icon = AvaloniaRunner.LoadIcon("normal.png");
break;
case TrayIcons.Disconnected:
this.trayIcon.Icon = AvaloniaRunner.LoadIcon("normal-disconnected.png");
break;
}
}
public void Configure()
{
this.Name = Duplicati.Library.AutoUpdater.AutoUpdateSettings.AppName;
this.trayIcon.ToolTipText = this.Name;
if (this.trayIcon != null)
this.trayIcon.ToolTipText = this.Name;
}
public void SetMenu(IEnumerable<AvaloniaMenuItem> menuItems)
@@ -377,7 +398,18 @@ namespace Duplicati.GUI.TrayIcon
menu.Add(item.GetNativeItem());
}
trayIcon.Menu = menu;
trayIcon.Clicked += (_, _) => this.menuItems.FirstOrDefault(x => x.IsDefault)?.Callback();
trayIcon.Clicked -= HandleTrayIconClick;
trayIcon.Clicked += HandleTrayIconClick;
}
}
private readonly ClickDebouncer _clickDebouncer = new ClickDebouncer();
private void HandleTrayIconClick(object? sender, EventArgs e)
{
if (_clickDebouncer.ShouldProcessClick())
{
this.menuItems?.FirstOrDefault(x => x.IsDefault)?.Callback();
}
}
@@ -413,10 +445,10 @@ namespace Duplicati.GUI.TrayIcon
public bool IsEnabled(LogEventLevel level, string area)
=> level >= _minLevel;
public void Log(LogEventLevel level, string area, object source, string messageTemplate)
=> Log(level, area, source, messageTemplate, null);
public void Log(LogEventLevel level, string area, object? source, string messageTemplate)
=> Log(level, area, source, messageTemplate, []);
public void Log(LogEventLevel level, string area, object source, string messageTemplate, params object[] propertyValues)
public void Log(LogEventLevel level, string area, object? source, string messageTemplate, params object?[] propertyValues)
=> Console.WriteLine($"Avalonia [{level}]: {source} {messageTemplate} {string.Join(" ", propertyValues)}");
}
Duplicati/GUI/Duplicati.GUI.TrayIcon/ClickDebouncer.cs
+27
View File
@@ -0,0 +1,27 @@
using System;
namespace Duplicati.GUI.TrayIcon
{
internal class ClickDebouncer
{
private DateTime _lastClickTime = DateTime.MinValue;
private readonly TimeSpan _debounceInterval;
public ClickDebouncer(TimeSpan? debounceInterval = null)
{
_debounceInterval = debounceInterval ?? TimeSpan.FromSeconds(1);
}
public bool ShouldProcessClick()
{
var now = DateTime.UtcNow;
if (now - _lastClickTime < _debounceInterval)
{
return false;
}
_lastClickTime = now;
return true;
}
}
}
Duplicati/GUI/Duplicati.GUI.TrayIcon/HttpServerConnection.cs
+100 -55
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
@@ -88,6 +88,7 @@ namespace Duplicati.GUI.TrayIcon
private readonly string m_baseUri;
private string m_password;
private string m_accesstoken;
private bool m_isTryingWithPassword;
public delegate void StatusUpdateDelegate(IServerStatus status);
public event StatusUpdateDelegate OnStatusUpdated;
@@ -161,10 +162,6 @@ namespace Duplicati.GUI.TrayIcon
}
};
// TODO: Not nice to do in constructor
// Get a connection
UpdateStatus(false);
m_waitLock = new AutoResetEvent(false);
m_requestThread = new Thread(ThreadRunner);
m_pollThread = new Thread(LongPollRunner);
@@ -176,11 +173,14 @@ namespace Duplicati.GUI.TrayIcon
m_pollThread.Start();
}
private void UpdateStatus(bool longpoll)
public Task UpdateStatus()
=> UpdateStatusAsync(false);
private async Task UpdateStatusAsync(bool longpoll)
{
var query = longpoll ? $"?longpoll=true&lastEventId={m_lastEventId}&duration={LONGPOLL_TIMEOUT}" : "";
m_status = PerformRequest<ServerStatusImpl>("GET", $"/serverstate{query}", null, longpoll ? Library.Utility.Timeparser.ParseTimeSpan(LONGPOLL_TIMEOUT) : null);
m_status = await PerformRequestAsync<ServerStatusImpl>("GET", $"/serverstate{query}", null, longpoll ? Library.Utility.Timeparser.ParseTimeSpan(LONGPOLL_TIMEOUT) : null);
m_lastEventId = m_status.LastEventID;
if (OnStatusUpdated != null)
@@ -189,19 +189,19 @@ namespace Duplicati.GUI.TrayIcon
if (m_lastNotificationId != m_status.LastNotificationUpdateID)
{
m_lastNotificationId = m_status.LastNotificationUpdateID;
UpdateNotifications();
await UpdateNotificationsAsync().ConfigureAwait(false);
}
if (m_lastDataUpdateId != m_status.LastDataUpdateID)
{
m_lastDataUpdateId = m_status.LastDataUpdateID;
UpdateApplicationSettings();
await UpdateApplicationSettingsAsync().ConfigureAwait(false);
}
}
private void UpdateNotifications()
private async Task UpdateNotificationsAsync()
{
var notifications = PerformRequest<NotificationImpl[]>("GET", "/notifications", null, null);
var notifications = await PerformRequestAsync<NotificationImpl[]>("GET", "/notifications", null, null).ConfigureAwait(false);
if (notifications != null)
{
foreach (var n in notifications.Where(x => x.Timestamp > m_firstNotificationTime))
@@ -213,9 +213,9 @@ namespace Duplicati.GUI.TrayIcon
}
}
private void UpdateApplicationSettings()
private async Task UpdateApplicationSettingsAsync()
{
var settings = PerformRequest<Dictionary<string, string>>("GET", "/serversettings", null, null);
var settings = await PerformRequestAsync<Dictionary<string, string>>("GET", "/serversettings", null, null).ConfigureAwait(false);
if (settings != null && settings.TryGetValue("disable-tray-icon-login", out var str))
m_disableTrayIconLogin = Library.Utility.Utility.ParseBool(str, false);
}
@@ -235,12 +235,15 @@ namespace Duplicati.GUI.TrayIcon
if (waitTime.TotalSeconds > 0)
Thread.Sleep(waitTime);
started = DateTime.Now;
UpdateStatus(true);
UpdateStatusAsync(true).Await();
errorCount = 0;
}
catch (Exception ex)
{
errorCount++;
// More than 1 error, and we are disconnected
if (errorCount >= 2)
Task.Run(() => { OnStatusUpdated?.Invoke(new ServerStatusImpl(LiveControlState.Paused, SuggestedStatusIcon.Disconnected, m_lastEventId, m_lastDataUpdateId, m_lastNotificationId)); }).Await();
System.Diagnostics.Trace.WriteLine("Request error: " + ex.Message);
Library.Logging.Log.WriteWarningMessage(LOGTAG, "TrayIconRequestError", ex, "Failed to get response");
}
@@ -269,7 +272,7 @@ namespace Duplicati.GUI.TrayIcon
if (req != null)
{
any = true;
PerformRequest<string>(req.Method, req.Endpoint, req.Body, req.Timeout);
PerformRequestAsync<string>(req.Method, req.Endpoint, req.Body, req.Timeout).Await();
}
} while (req != null);
@@ -300,18 +303,18 @@ namespace Duplicati.GUI.TrayIcon
private sealed record SigninResponse(string AccessToken);
private T PerformRequest<T>(string method, string urlfragment, string body, TimeSpan? timeout)
private async Task<T> PerformRequestAsync<T>(string method, string urlfragment, string body, TimeSpan? timeout)
{
if (string.IsNullOrWhiteSpace(m_accesstoken) && !urlfragment.StartsWith("/auth/"))
ObtainAccessToken();
await ObtainAccessTokenAsync().ConfigureAwait(false);
var hasTriedPassword = false;
var hasTriedPassword = m_isTryingWithPassword;
while (true)
{
try
{
return PerformRequestInternal<T>(method, urlfragment, body, timeout);
return await PerformRequestInternalAsync<T>(method, urlfragment, body, timeout).ConfigureAwait(false);
}
catch (Exception ex)
when (ex is HttpRequestException hex && (hex.StatusCode == HttpStatusCode.Unauthorized || hex.StatusCode == HttpStatusCode.Forbidden)
@@ -321,20 +324,49 @@ namespace Duplicati.GUI.TrayIcon
if (hasTriedPassword)
throw;
// TODO: This error handling is error prone and can end up in infinite recursion
// Should rewrite the entire class and use websockets instead
// Only try once, and clear the token for the next try
hasTriedPassword = true;
m_accesstoken = null;
ObtainAccessToken();
try
{
m_isTryingWithPassword = true;
await ObtainAccessTokenAsync().ConfigureAwait(false);
}
finally
{
m_isTryingWithPassword = false;
}
}
}
}
private void ObtainAccessToken()
private async Task ObtainAccessTokenAsync()
{
var token = ObtainSignInToken();
// If we host the server, issue the access token from the service
if (FIXMEGlobal.IsServerStarted && m_passwordSource == Program.PasswordSource.HostedServer)
{
var provider = FIXMEGlobal.Provider.GetRequiredService<IJWTTokenProvider>();
m_accesstoken = provider.CreateAccessToken("trayicon", provider.TemporaryFamilyId);
return;
}
// If we know the password, issue a token from the API
if (!string.IsNullOrWhiteSpace(m_password))
{
m_accesstoken = (await PerformRequestInternalAsync<SigninResponse>("POST", "/auth/login", JsonSerializer.Serialize(new { Password = m_password }), null).ConfigureAwait(false)).AccessToken;
return;
}
// Otherwise, try to get a signin token
var token = await ObtainSignInTokenAsync().ConfigureAwait(false);
if (string.IsNullOrWhiteSpace(token))
return;
m_accesstoken = PerformRequestInternal<SigninResponse>("POST", "/auth/signin", JsonSerializer.Serialize(new { SigninToken = token }), null).AccessToken;
// Use the signin token to get an access token
m_accesstoken = (await PerformRequestInternalAsync<SigninResponse>("POST", "/auth/signin", JsonSerializer.Serialize(new { SigninToken = token }), null).ConfigureAwait(false)).AccessToken;
}
private async Task<T> PerformRequestInternalAsync<T>(string method, string endpoint, string body, TimeSpan? timeout)
@@ -358,13 +390,10 @@ namespace Duplicati.GUI.TrayIcon
if (typeof(T) == typeof(string))
return (T)(object)await response.Content.ReadAsStringAsync().ConfigureAwait(false);
using (var stream = await response.Content.ReadAsStreamAsync())
using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))
return await JsonSerializer.DeserializeAsync<T>(stream, serializerOptions).ConfigureAwait(false);
}
private T PerformRequestInternal<T>(string method, string endpoint, string body, TimeSpan? timeout)
=> PerformRequestInternalAsync<T>(method, endpoint, body, timeout).Await();
private void ExecuteAndNotify(string method, string urifragment, string body)
{
lock (m_lock)
@@ -391,21 +420,29 @@ namespace Duplicati.GUI.TrayIcon
}
private sealed record SigninTokenResponse(string Token);
private string IssueSigninToken(string password)
=> PerformRequest<SigninTokenResponse>("POST", "/auth/issuesignintoken", JsonSerializer.Serialize(new { Password = password }), null).Token;
private async Task<string> IssueSigninTokenAsync(string password)
=> (await PerformRequestInternalAsync<SigninTokenResponse>("POST", "/auth/issuesignintoken", JsonSerializer.Serialize(new { Password = password }), null).ConfigureAwait(false)).Token;
private string ObtainSignInToken()
private async Task<string> ObtainSignInTokenAsync()
{
string signinjwt = null;
// If we host the server, issue the token from the service
if (FIXMEGlobal.IsServerStarted && m_passwordSource == Program.PasswordSource.HostedServer)
{
if (FIXMEGlobal.DataConnection.ApplicationSettings.DisableSigninTokens)
return null;
signinjwt = FIXMEGlobal.Provider.GetRequiredService<IJWTTokenProvider>().CreateSigninToken("trayicon");
}
// If we have database access, grab the issuer key from the db and issue a token
if (string.IsNullOrWhiteSpace(signinjwt) && m_passwordSource == Program.PasswordSource.Database)
{
Program.databaseConnection.ApplicationSettings.ReloadSettings();
if (Program.databaseConnection.ApplicationSettings.DisableSigninTokens)
return null;
var cfg = Program.databaseConnection.ApplicationSettings.JWTConfig;
if (!string.IsNullOrWhiteSpace(cfg))
signinjwt = new JWTTokenProvider(JsonSerializer.Deserialize<JWTConfig>(cfg)).CreateSigninToken("trayicon");
@@ -413,20 +450,28 @@ namespace Duplicati.GUI.TrayIcon
// If we know the password, issue a token from the API
if (string.IsNullOrWhiteSpace(signinjwt) && !string.IsNullOrWhiteSpace(m_password))
signinjwt = IssueSigninToken(m_password);
signinjwt = await IssueSigninTokenAsync(m_password);
return signinjwt;
}
public string StatusWindowURL
public async Task<string> GetStatusWindowURLAsync()
{
get
string signinjwt = null;
if (!m_disableTrayIconLogin)
{
var signinjwt = m_disableTrayIconLogin ? null : ObtainSignInToken();
return string.IsNullOrWhiteSpace(signinjwt)
? m_baseUri + STATUS_WINDOW
: m_baseUri + SIGNIN_WINDOW + $"?token={signinjwt}";
try
{
signinjwt = await ObtainSignInTokenAsync().ConfigureAwait(false);
}
catch
{
}
}
return string.IsNullOrWhiteSpace(signinjwt)
? m_baseUri + STATUS_WINDOW
: m_baseUri + SIGNIN_WINDOW + $"?token={signinjwt}";
}
}
}
Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs
+48 -41
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
@@ -24,9 +24,14 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading;
using Duplicati.Library.AutoUpdater;
using Duplicati.Library.Interface;
using Duplicati.Library.RestAPI;
using Duplicati.Library.Utility;
using Duplicati.Server;
using Uri = System.Uri;
namespace Duplicati.GUI.TrayIcon
{
@@ -61,14 +66,12 @@ namespace Duplicati.GUI.TrayIcon
private const string DETACHED_PROCESS = "detached-process";
private const string BROWSER_COMMAND_OPTION = "browser-command";
private const string DEFAULT_HOSTURL = "http://localhost:8200";
private static string DEFAULT_HOSTURL => $"http://{Library.Utility.Utility.IpVersionCompatibleLoopback}:8200";
private static string _browser_command = null;
private static bool disableTrayIconLogin = false;
private static bool openui = false;
private static Uri serverURL = new Uri(DEFAULT_HOSTURL);
private static Uri serverURL = new(DEFAULT_HOSTURL);
public static string BrowserCommand { get { return _browser_command; } }
public static Server.Database.Connection databaseConnection = null;
@@ -85,15 +88,7 @@ namespace Duplicati.GUI.TrayIcon
if (OperatingSystem.IsWindows() && !Library.Utility.Utility.ParseBoolOption(options, DETACHED_PROCESS))
Library.Utility.Win32.AttachConsole(Library.Utility.Win32.ATTACH_PARENT_PROCESS);
foreach (string s in args)
if (
s.Equals("help", StringComparison.OrdinalIgnoreCase) ||
s.Equals("/help", StringComparison.OrdinalIgnoreCase) ||
s.Equals("usage", StringComparison.OrdinalIgnoreCase) ||
s.Equals("/usage", StringComparison.OrdinalIgnoreCase))
options["help"] = "";
if (options.ContainsKey("help"))
if (HelpOptionExtensions.IsArgumentAnyHelpString(args))
{
Console.WriteLine("Supported commandline arguments:");
Console.WriteLine();
@@ -154,7 +149,7 @@ namespace Duplicati.GUI.TrayIcon
}
else if (Library.Utility.Utility.ParseBoolOption(options, READCONFIGFROMDB_OPTION))
{
if (File.Exists(Path.Combine(Server.Program.GetDataFolderPath(options), Server.Program.SERVER_DATABASE_FILENAME)))
if (File.Exists(Path.Combine(DataFolderManager.DATAFOLDER, DataFolderManager.SERVER_DATABASE_FILENAME)))
{
passwordSource = PasswordSource.Database;
databaseConnection = Server.Program.GetDatabaseConnection(options, true);
@@ -211,31 +206,43 @@ No password provided, unable to connect to server, exiting");
do
{
if (reSpawn > 0)
Thread.Sleep(1000);
try
{
ServicePointManager.SecurityProtocol = SecurityProtocolType.SystemDefault;
using (Connection = new HttpServerConnection(serverURL, password, passwordSource, disableTrayIconLogin, acceptedHostCertificate, options))
{
// Make sure we have the latest status
// This will throw an exception if the server is not running,
// or the password is incorrect
Connection.UpdateStatus().Await();
using (var tk = RunTrayIcon())
{
if (hosted != null && Server.Program.ApplicationInstance != null)
Server.Program.ApplicationInstance.SecondInstanceDetected +=
new Server.SingleInstance.SecondInstanceDelegate(
x => { tk.ShowUrlInWindow(serverURL.ToString()); });
x => tk.ShowStatusWindow());
// TODO: If we change to hosted browser this should be a callback
if (openui)
{
try
Connection.GetStatusWindowURLAsync().ContinueWith(t =>
{
tk.ShowUrlInWindow(Connection.StatusWindowURL);
if (t.IsFaulted)
{
Console.WriteLine("Failed to get status window URL: " + t.Exception.Message);
tk.NotifyUser("Failed to get status window URL", t.Exception.Message, NotificationType.Error);
return;
}
tk.ShowUrlInWindow(t.Result);
Server.Program.IsFirstRun = false;
Server.Program.ServerPortChanged = false;
}
catch
{
}
});
}
// If the server shuts down, shut down the tray-icon as well
@@ -262,7 +269,7 @@ No password provided, unable to connect to server, exiting");
}
}
}
catch (WebException ex)
catch (HttpRequestException ex)
{
System.Diagnostics.Trace.WriteLine("Request error: " + ex);
Console.WriteLine("Request error: " + ex);
Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs
+70 -31
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
@@ -43,7 +43,8 @@ namespace Duplicati.GUI.TrayIcon
IdleWarning,
IdleError,
PausedError,
RunningError
RunningError,
Disconnected
}
public enum WindowIcons
@@ -64,11 +65,16 @@ namespace Duplicati.GUI.TrayIcon
void SetDefault(bool isDefault);
void SetIcon(MenuIcons icon);
void SetText(string text);
void SetEnabled(bool enabled);
void SetHidden(bool hidden);
}
public abstract class TrayIconBase : IDisposable
{
protected IMenuItem m_reconnectMenu;
protected IMenuItem m_openMenu;
protected IMenuItem m_pauseMenu;
protected IMenuItem m_quitMenu;
protected bool m_stateIsPaused;
protected Action m_onSingleClick;
protected Action m_onDoubleClick;
@@ -105,7 +111,7 @@ namespace Duplicati.GUI.TrayIcon
protected abstract void SetMenu(IEnumerable<IMenuItem> items);
protected abstract void NotifyUser(string title, string message, NotificationType type);
public abstract void NotifyUser(string title, string message, NotificationType type);
protected virtual void RegisterStatusUpdateCallback()
{
@@ -149,23 +155,49 @@ namespace Duplicati.GUI.TrayIcon
protected IEnumerable<IMenuItem> BuildMenu()
{
var tmp = CreateMenuItem("Open", MenuIcons.Status, OnStatusClicked, null);
tmp.SetDefault(true);
m_reconnectMenu = CreateMenuItem("Reconnect", MenuIcons.Resume, Reconnect, null);
m_reconnectMenu.SetHidden(true);
m_openMenu = CreateMenuItem("Open", MenuIcons.Status, OnStatusClicked, null);
m_openMenu.SetDefault(true);
return new IMenuItem[] {
tmp,
m_reconnectMenu,
m_openMenu,
m_pauseMenu = CreateMenuItem("Pause", MenuIcons.Pause, OnPauseClicked, null ),
CreateMenuItem("Quit", MenuIcons.Quit, OnQuitClicked, null),
m_quitMenu = CreateMenuItem("Quit", MenuIcons.Quit, OnQuitClicked, null),
};
}
protected void ShowStatusWindow()
private void Reconnect()
{
var window = ShowUrlInWindow(Program.Connection.StatusWindowURL);
if (window != null)
Program.Connection.UpdateStatus().ContinueWith(t =>
{
window.SetIcon(WindowIcons.Regular);
window.SetTitle("Duplicati status");
}
if (t.IsFaulted)
{
Console.WriteLine("Failed to reconnect: " + t.Exception.Message);
NotifyUser("Failed to reconnect", t.Exception.Message, NotificationType.Error);
}
});
}
public void ShowStatusWindow()
{
Program.Connection.GetStatusWindowURLAsync().ContinueWith(t =>
{
if (t.IsFaulted)
{
Console.WriteLine("Failed to get status window URL: " + t.Exception.Message);
NotifyUser("Failed to get status window URL", t.Exception.Message, NotificationType.Error);
return;
}
var window = ShowUrlInWindow(t.Result);
if (window != null)
{
window.SetIcon(WindowIcons.Regular);
window.SetTitle("Duplicati status");
}
});
}
protected void OnStatusClicked()
@@ -207,6 +239,9 @@ namespace Duplicati.GUI.TrayIcon
case SuggestedStatusIcon.Paused:
this.SetIcon(TrayIcons.Paused);
break;
case SuggestedStatusIcon.Disconnected:
this.SetIcon(TrayIcons.Disconnected);
break;
case SuggestedStatusIcon.Ready:
default:
this.SetIcon(TrayIcons.Idle);
@@ -226,6 +261,10 @@ namespace Duplicati.GUI.TrayIcon
m_pauseMenu.SetText("Resume");
m_stateIsPaused = true;
}
m_openMenu.SetHidden(status.SuggestedStatusIcon == SuggestedStatusIcon.Disconnected);
m_pauseMenu.SetHidden(status.SuggestedStatusIcon == SuggestedStatusIcon.Disconnected);
m_reconnectMenu.SetHidden(status.SuggestedStatusIcon != SuggestedStatusIcon.Disconnected);
});
}
Duplicati/Library/AutoUpdater/AutoUpdateSettings.cs
+204 -125
View File
@@ -1,24 +1,26 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Collections.Generic;
using System.Linq;
@@ -27,28 +29,78 @@ using System.Text.RegularExpressions;
namespace Duplicati.Library.AutoUpdater
{
/// <summary>
/// Contains settings for the auto-updater
/// </summary>
public static class AutoUpdateSettings
{
private static readonly Dictionary<string, string> _cache = new Dictionary<string, string>();
/// <summary>
/// The appname file resource name
/// </summary>
private const string APP_NAME = "AutoUpdateAppName.txt";
/// <summary>
/// The update URL file resource name
/// </summary>
private const string UPDATE_URL = "AutoUpdateURL.txt";
/// <summary>
/// The update key file resource name
/// </summary>
private const string UPDATE_KEY = "AutoUpdateSignKeys.txt";
/// <summary>
/// The update channel file resource name
/// </summary>
private const string UPDATE_CHANNEL = "AutoUpdateBuildChannel.txt";
/// <summary>
/// The update install file template resource name
/// </summary>
private const string UPDATE_INSTALL_FILE = "AutoUpdateInstallIDTemplate.txt";
/// <summary>
/// The update machine file template resource name
/// </summary>
private const string UPDATE_MACHINE_FILE = "AutoUpdateMachineIDTemplate.txt";
/// <summary>
/// The OEM file name
/// </summary>
private const string OEM_APP_NAME = "oem-app-name.txt";
/// <summary>
/// The OEM update URL file name
/// </summary>
private const string OEM_UPDATE_URL = "oem-update-url.txt";
/// <summary>
/// The OEM update key file name
/// </summary>
private const string OEM_UPDATE_KEY = "oem-update-key.txt";
/// <summary>
/// The OEM update install template file name
/// </summary>
private const string OEM_UPDATE_INSTALL_FILE = "oem-update-installid.txt";
/// <summary>
/// The update URL environment variable name template
/// </summary>
public const string UPDATEURL_ENVNAME_TEMPLATE = "AUTOUPDATER_{0}_URLS";
/// <summary>
/// The update channel environment variable name template
/// </summary>
public const string UPDATECHANNEL_ENVNAME_TEMPLATE = "AUTOUPDATER_{0}_CHANNEL";
/// <summary>
/// The package prefix group name in the <see cref="MATCH_AUTOUPDATE_URL"/> regex
/// </summary>
internal const string MATCH_UPDATE_URL_PREFIX_GROUP = "prefix";
/// <summary>
/// The channel group name in the <see cref="MATCH_AUTOUPDATE_URL"/> regex
/// </summary>
internal const string MATCH_UPDATE_URL_CHANNEL_GROUP = "channel";
/// <summary>
/// The filename group name in the <see cref="MATCH_AUTOUPDATE_URL"/> regex
/// </summary>
internal const string MATCH_UPDATE_URL_FILENAME_GROUP = "filename";
/// <summary>
/// The regex to match the auto-update URL and replace the channel
/// </summary>
internal static readonly Regex MATCH_AUTOUPDATE_URL =
new Regex(string.Format(
"(?<{0}>.+)(?<{1}>{3})(?<{2}>/([^/]+).manifest)",
@@ -57,27 +109,21 @@ namespace Duplicati.Library.AutoUpdater
MATCH_UPDATE_URL_FILENAME_GROUP,
string.Join("|", Enum.GetNames(typeof(ReleaseType)).Union(new[] { "preview" }))), RegexOptions.Compiled | RegexOptions.IgnoreCase);
static AutoUpdateSettings()
/// <summary>
/// Reads a resource text from the assembly, with an optional OEM filename override
/// </summary>
/// <param name="name">The embedded resource name</param>
/// <param name="oemname">The OEM filename override</param>
/// <returns>The text of the resource</returns>
private static string ReadResourceText(string name, string? oemname)
{
ReadResourceText(APP_NAME, OEM_APP_NAME);
ReadResourceText(UPDATE_URL, OEM_UPDATE_URL);
ReadResourceText(UPDATE_KEY, OEM_UPDATE_KEY);
ReadResourceText(UPDATE_INSTALL_FILE, OEM_UPDATE_INSTALL_FILE);
ReadResourceText(UPDATE_MACHINE_FILE, null);
ReadResourceText(UPDATE_CHANNEL, null);
}
private static string ReadResourceText(string name, string oemname)
{
// First try to read from _cache
if (_cache.TryGetValue(name, out string result))
return result;
string? result = null;
try
{
using (var rd = new System.IO.StreamReader(System.Reflection.Assembly.GetExecutingAssembly().GetManifestResourceStream(typeof(AutoUpdateSettings), name)))
result = rd.ReadToEnd();
using (var rs = System.Reflection.Assembly.GetExecutingAssembly().GetManifestResourceStream(typeof(AutoUpdateSettings), name))
if (rs != null)
using (var rd = new System.IO.StreamReader(rs))
result = rd.ReadToEnd();
}
catch (Exception ex)
{
@@ -99,121 +145,154 @@ namespace Duplicati.Library.AutoUpdater
else
result = result.Trim();
_cache[name] = result;
return result;
}
public static string[] URLs
{
get
{
if (UsesAlternateURLs)
return Environment.GetEnvironmentVariable(string.Format(UPDATEURL_ENVNAME_TEMPLATE, AppName)).Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
else
return ReadResourceText(UPDATE_URL, OEM_UPDATE_URL).Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
}
}
/// <summary>
/// The URLs to check for updates
/// </summary>
public static string[] URLs => _URLs.Value;
public static bool UsesAlternateURLs
/// <summary>
/// The alternate URLs to check for updates, lazy evaluated
/// </summary>
private static readonly Lazy<string?> _alternateURLs = new Lazy<string?>(() =>
{
get
{
return !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(string.Format(UPDATEURL_ENVNAME_TEMPLATE, AppName)));
}
}
return Environment.GetEnvironmentVariable(string.Format(UPDATEURL_ENVNAME_TEMPLATE, AppName));
});
public static ReleaseType DefaultUpdateChannel
/// <summary>
/// The URLs to check for updates, lazy evaluated
/// </summary>
private static readonly Lazy<string[]> _URLs = new Lazy<string[]>(() =>
{
get
{
var channelstring = Environment.GetEnvironmentVariable(string.Format(UPDATECHANNEL_ENVNAME_TEMPLATE, AppName));
if (!string.IsNullOrWhiteSpace(_alternateURLs.Value))
return _alternateURLs.Value.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
else
return ReadResourceText(UPDATE_URL, OEM_UPDATE_URL).Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
});
if (UsesAlternateURLs && string.IsNullOrWhiteSpace(channelstring))
/// <summary>
/// Indicates if alternate URLs are in use
/// </summary>
public static bool UsesAlternateURLs => !string.IsNullOrWhiteSpace(_alternateURLs.Value);
/// <summary>
/// The default update channel
/// </summary>
public static ReleaseType DefaultUpdateChannel => _defaultUpdateChannel.Value;
/// <summary>
/// The default update channel, lazy evaluated
/// </summary>
private static readonly Lazy<ReleaseType> _defaultUpdateChannel = new Lazy<ReleaseType>(() =>
{
var channelstring = Environment.GetEnvironmentVariable(string.Format(UPDATECHANNEL_ENVNAME_TEMPLATE, AppName));
if (UsesAlternateURLs && string.IsNullOrWhiteSpace(channelstring))
{
foreach (var url in URLs)
{
foreach (var url in URLs)
var match = AutoUpdateSettings.MATCH_AUTOUPDATE_URL.Match(url);
if (match.Success)
{
var match = AutoUpdateSettings.MATCH_AUTOUPDATE_URL.Match(url);
if (match.Success)
{
channelstring = match.Groups[AutoUpdateSettings.MATCH_UPDATE_URL_CHANNEL_GROUP].Value;
break;
}
channelstring = match.Groups[AutoUpdateSettings.MATCH_UPDATE_URL_CHANNEL_GROUP].Value;
break;
}
}
if (string.IsNullOrWhiteSpace(channelstring))
channelstring = BuildUpdateChannel;
// Update from older builds
if (string.Equals(channelstring, "preview", StringComparison.OrdinalIgnoreCase))
channelstring = ReleaseType.Experimental.ToString();
if (string.Equals(channelstring, "rene", StringComparison.OrdinalIgnoreCase))
channelstring = ReleaseType.Canary.ToString();
ReleaseType rt;
if (!Enum.TryParse<ReleaseType>(channelstring, true, out rt))
rt = ReleaseType.Stable;
return rt;
}
}
public static string AppName
{
get { return ReadResourceText(APP_NAME, OEM_APP_NAME); }
}
if (string.IsNullOrWhiteSpace(channelstring))
channelstring = BuildUpdateChannel;
public static string BuildUpdateChannel
{
get { return ReadResourceText(UPDATE_CHANNEL, null); }
}
// Update from older builds
if (string.Equals(channelstring, "preview", StringComparison.OrdinalIgnoreCase))
channelstring = ReleaseType.Experimental.ToString();
if (string.Equals(channelstring, "rene", StringComparison.OrdinalIgnoreCase))
channelstring = ReleaseType.Canary.ToString();
public static string UpdateInstallFileText
{
get { return string.Format(ReadResourceText(UPDATE_INSTALL_FILE, OEM_UPDATE_INSTALL_FILE), Guid.NewGuid().ToString("N")); }
}
ReleaseType rt;
if (!Enum.TryParse<ReleaseType>(channelstring, true, out rt))
rt = ReleaseType.Stable;
return rt;
});
/// <summary>
/// The application name
/// </summary>
public static string AppName => _appName.Value;
/// <summary>
/// The application name, lazy evaluated
/// </summary>
private static readonly Lazy<string> _appName = new Lazy<string>(() => ReadResourceText(APP_NAME, OEM_APP_NAME));
/// <summary>
/// The build update channel
/// </summary>
public static string BuildUpdateChannel => _buildUpdateChannel.Value;
/// <summary>
/// The build update channel, lazy evaluated
/// </summary>
private static readonly Lazy<string> _buildUpdateChannel = new Lazy<string>(() => ReadResourceText(UPDATE_CHANNEL, null));
/// <summary>
/// The text for the update install file
/// </summary>
public static string UpdateInstallFileText => string.Format(ReadResourceText(UPDATE_INSTALL_FILE, OEM_UPDATE_INSTALL_FILE), Guid.NewGuid().ToString("N"));
/// <summary>
/// Updates the machine file text with the machine ID
/// </summary>
/// <param name="machineid">The machine ID to use</param>
/// <returns>The updated text</returns>
public static string UpdateMachineFileText(string machineid)
=> string.Format(ReadResourceText(UPDATE_MACHINE_FILE, "{0}"), string.IsNullOrWhiteSpace(machineid) ? Guid.NewGuid().ToString("N") : machineid);
public static RSACryptoServiceProvider[] SignKeys
/// <summary>
/// The keys to use for signing
/// </summary>
public static RSACryptoServiceProvider[] SignKeys => _signKeys.Value;
/// <summary>
/// The keys to use for signing, lazy evaluated
/// </summary>
private static readonly Lazy<RSACryptoServiceProvider[]> _signKeys = new Lazy<RSACryptoServiceProvider[]>(() =>
{
get
var keys = new List<RSACryptoServiceProvider>();
try
{
var keys = new List<RSACryptoServiceProvider>();
var src = ReadResourceText(UPDATE_KEY, OEM_UPDATE_KEY);
try
// Allow multiple keys, one per line
// For fallback, read the whole string as a key, in case there are old ones with line breaks
var keystrings = src.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries)
.Select(x => x.Trim())
.Prepend(src.Trim())
.Where(x => !string.IsNullOrWhiteSpace(x))
.Distinct();
foreach (var str in keystrings)
{
var src = ReadResourceText(UPDATE_KEY, OEM_UPDATE_KEY);
// Allow multiple keys, one per line
// For fallback, read the whole string as a key, in case there are old ones with line breaks
var keystrings = src.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries)
.Select(x => x.Trim())
.Prepend(src.Trim())
.Where(x => !string.IsNullOrWhiteSpace(x))
.Distinct();
foreach (var str in keystrings)
try
{
try
{
var key = new RSACryptoServiceProvider();
key.FromXmlString(str);
keys.Add(key);
}
catch
{ }
var key = new RSACryptoServiceProvider();
key.FromXmlString(str);
keys.Add(key);
}
catch
{ }
}
catch
{
}
return keys.ToArray();
}
}
catch
{
}
return keys.ToArray();
});
}
}
Duplicati/Library/AutoUpdater/DataFolderLocator.cs
+148
View File
@@ -0,0 +1,148 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Linq;
using System.Collections.Generic;
using Duplicati.Library.Common.IO;
#nullable enable
namespace Duplicati.Library.AutoUpdater;
/// <summary>
/// Helper class to locate the default storage folder for the application
/// </summary>
public static class DataFolderLocator
{
/// <summary>
/// Finds a default storage folder, using the operating system specific locations.
/// The targetfilename is used to detect locations that are used in previous versions.
/// If the targetfilename is found in an old location, but not the current, the old location is used.
/// Note that the folder is not created, only the path is returned.
/// If the data folder is overriden, the overriden folder is used, and no search is performed.
/// </summary>
/// <param name="targetfilename">The filename to look for</param>
/// <param name="autoCreate">Whether to create the folder if it does not exist</param>
/// <returns>The default storage folder</returns>
public static string GetDefaultStorageFolder(string targetfilename, bool autoCreate)
{
var folder = DataFolderManager.OVERRIDEN_DATAFOLDER
? DataFolderManager.DATAFOLDER
: GetDefaultStorageFolderInternal(targetfilename, AutoUpdateSettings.AppName);
if (SystemIO.IO_OS.DirectoryExists(folder))
{
if (!SystemIO.IO_OS.FileExists(System.IO.Path.Combine(folder, Util.InsecurePermissionsMarkerFile)))
SystemIO.IO_OS.DirectorySetPermissionUserRWOnly(folder);
}
else if (autoCreate)
{
// Create the folder
SystemIO.IO_OS.DirectoryCreate(folder);
// Make sure the folder is only accessible by the current user
SystemIO.IO_OS.DirectorySetPermissionUserRWOnly(folder);
}
return folder;
}
/// <summary>
/// Finds a default storage folder, using the operating system specific locations.
/// The targetfilename is used to detect locations that are used in previous versions.
/// If the targetfilename is found in an old location, but not the current, the old location is used.
/// Note that the folder is not created, only the path is returned.
/// </summary>
/// <param name="targetfilename">The filename to look for</param>
/// <param name="appName">The name of the application</param>
/// <returns>The default storage folder</returns>
internal static string GetDefaultStorageFolderInternal(string targetfilename, string appName)
{
//Normal mode uses the systems "(Local) Application Data" folder
// %LOCALAPPDATA% on Windows, ~/.config on Linux
var folder = System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), appName);
if (OperatingSystem.IsWindows())
{
// Special handling for Windows:
// - Older versions use %APPDATA%
// - New versions use %LOCALAPPDATA%
// - And prevent using C:\Windows\
var newlocation = System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), appName);
var folderOrder = new List<string>() {
folder,
newlocation
};
// If %LOCALAPPDATA% is inside the Windows folder, prefer a LocalService folder instead
if (Common.IO.Util.IsPathUnderWindowsFolder(newlocation))
folderOrder.Add(System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), appName));
// Prefer the most recent location
var matches = folderOrder.AsEnumerable()
.Reverse()
.Where(x => System.IO.File.Exists(System.IO.Path.Combine(x, targetfilename)))
.ToList();
// Use the most recent location found with content
// If none are found, use the most recent location
folder = matches.FirstOrDefault() ?? folderOrder.Last();
}
if (OperatingSystem.IsMacOS())
{
// Special handling for MacOS:
// - Older versions use ~/.config/
// - but new versions use ~/Library/Application\ Support/
var configfolder = System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".config", appName);
var prevfile = System.IO.Path.Combine(configfolder, targetfilename);
var curfile = System.IO.Path.Combine(folder, targetfilename);
// If the old file exists, and not the new file, we use the old
// Otherwise we use the new location
if (!System.IO.File.Exists(curfile) && System.IO.File.Exists(prevfile))
folder = configfolder;
}
if (OperatingSystem.IsLinux() && (folder == $"/{appName}" || folder == appName))
{
// Special handling for Linux with no home folder:
// - Older versions use /
// - but new versions use /var/lib/
var libfolder = System.IO.Path.Combine("var", "lib", appName);
var curfile = System.IO.Path.Combine(libfolder, targetfilename);
var prevfile = System.IO.Path.Combine(folder, targetfilename);
// If the old file exists, and not the new file, we use the old
// Otherwise we use the new location
if (System.IO.File.Exists(curfile) || !System.IO.File.Exists(prevfile))
folder = libfolder;
}
return folder;
}
}
Duplicati/Library/AutoUpdater/DataFolderManager.cs
+253
View File
@@ -0,0 +1,253 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.IO;
using System.Linq;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Utility;
namespace Duplicati.Library.AutoUpdater;
/// <summary>
/// Manages the data folder for the application
/// </summary>
public static class DataFolderManager
{
/// <summary>
/// The folder where the machine id is placed
/// </summary>
public static readonly string DATAFOLDER;
/// <summary>
/// The installation ID filename stored in <see cref="DATAFOLDER"/>
/// </summary>
private const string INSTALL_FILE = "installation.txt";
/// <summary>
/// The machine ID filename stored in <see cref="DATAFOLDER"/>
/// </summary>
private const string MACHINE_FILE = "machineid.txt";
/// <summary>
/// The option name for portable mode
/// </summary>
public const string PORTABLE_MODE_OPTION = "portable-mode";
/// <summary>
/// The option anme for the server data folder
/// </summary>
public const string SERVER_DATAFOLDER_OPTION = "server-datafolder";
/// <summary>
/// The app name to use for variables
/// </summary>
private static readonly string APPNAME = AutoUpdateSettings.AppName;
/// <summary>
/// The name of the environment variable that allows overriding the path to the data folder used by Duplicati
/// </summary>
public static readonly string DATAFOLDER_ENV_NAME = $"{APPNAME}_HOME".ToUpperInvariant();
/// <summary>
/// Name of the database file
/// </summary>
public static readonly string SERVER_DATABASE_FILENAME = $"{APPNAME}-server.sqlite";
/// <summary>
/// Flag to indicate if the application is running in portable mode
/// </summary>
public static readonly bool PORTABLE_MODE;
/// <summary>
/// Flag to indicate if the data folder was overriden
/// </summary>
public static readonly bool OVERRIDEN_DATAFOLDER;
/// <summary>
/// Replication of the argument parsing from the main Duplicati codebase
/// </summary>
/// <param name="option">The option to extract</param>
/// <returns><c>null</c> if the option is not found, otherwise the value of the option</returns>
private static string? ExtractOptionSlim(string option)
{
var opt = $"--{option}";
var args = Environment.GetCommandLineArgs().Skip(1).ToArray();
var match = args.Select((token, index) => new { token, index })
.LastOrDefault(x =>
string.Equals(x.token, opt, StringComparison.OrdinalIgnoreCase)
||
x.token.StartsWith(opt + "=", StringComparison.OrdinalIgnoreCase)
);
// Not found, try the environment variable
if (string.IsNullOrWhiteSpace(match?.token))
return Environment.GetEnvironmentVariable($"{AutoUpdateSettings.AppName}__{option.Replace('-', '_')}".ToUpperInvariant());
// Found in the form --option=value
if (match.token.StartsWith(opt + "=", StringComparison.OrdinalIgnoreCase))
return match.token.Substring(opt.Length + 1).Trim('"');
// Found in the form --option value
if (match.index + 1 < args.Length)
{
var value = args[match.index + 1];
if (!value.StartsWith("--"))
return value;
}
// Found, but no value, just the option
return "";
}
/// <summary>
/// Replication of the boolean parsing from the main Duplicati codebase
/// </summary>
/// <param name="value">The value to parse</param>
/// <returns><c>true</c> if the value is a truthy value, otherwise <c>false</c></returns>
private static bool ParseBoolSlim(string? value)
{
// In debug builds, we default to portable mode
if (value == null)
#if DEBUG
return true;
#else
return false;
#endif
if (
value.Equals("false", StringComparison.OrdinalIgnoreCase)
|| value.Equals("0", StringComparison.OrdinalIgnoreCase)
|| value.Equals("no", StringComparison.OrdinalIgnoreCase)
|| value.Equals("off", StringComparison.OrdinalIgnoreCase)
)
return false;
return true;
}
static DataFolderManager()
{
// Trigger portable mode, if the flag is set
PORTABLE_MODE = ParseBoolSlim(ExtractOptionSlim(PORTABLE_MODE_OPTION));
// The environment variable is a legacy setting
var envOverride = Environment.GetEnvironmentVariable(DATAFOLDER_ENV_NAME);
// These are mainly supported by the Server
var datafolderArg = ExtractOptionSlim(SERVER_DATAFOLDER_OPTION);
// Prefer the command line argument over the environment variable
if (!string.IsNullOrWhiteSpace(datafolderArg))
{
OVERRIDEN_DATAFOLDER = true;
DATAFOLDER = Util.AppendDirSeparator(Environment.ExpandEnvironmentVariables(datafolderArg).Trim('"'));
}
// Portable mode is prefered over the environment variable
else if (PORTABLE_MODE)
{
OVERRIDEN_DATAFOLDER = true;
DATAFOLDER = Util.AppendDirSeparator(Path.Combine(UpdaterManager.INSTALLATIONDIR, "data"));
}
// Use the legacy environment variable, if set
else if (!string.IsNullOrWhiteSpace(envOverride))
{
OVERRIDEN_DATAFOLDER = true;
DATAFOLDER = Util.AppendDirSeparator(Environment.ExpandEnvironmentVariables(envOverride).Trim('"'));
}
// Use the default location
else
{
OVERRIDEN_DATAFOLDER = false;
DATAFOLDER = Util.AppendDirSeparator(DataFolderLocator.GetDefaultStorageFolderInternal(SERVER_DATABASE_FILENAME, APPNAME));
}
if (Directory.Exists(DATAFOLDER))
{
if (!File.Exists(Path.Combine(DATAFOLDER, Util.InsecurePermissionsMarkerFile)))
SystemIO.IO_OS.DirectorySetPermissionUserRWOnly(DATAFOLDER);
}
else
{
Directory.CreateDirectory(DATAFOLDER);
SystemIO.IO_OS.DirectorySetPermissionUserRWOnly(DATAFOLDER);
}
if (!File.Exists(Path.Combine(DATAFOLDER, INSTALL_FILE)))
{
// In case there was already a machine id file from 2.0.8.1 or older, copy it to the new location
if (File.Exists(Path.Combine(DATAFOLDER, "updates", INSTALL_FILE)))
File.Copy(Path.Combine(DATAFOLDER, "updates", INSTALL_FILE), Path.Combine(DATAFOLDER, INSTALL_FILE), true);
else
File.WriteAllText(Path.Combine(DATAFOLDER, INSTALL_FILE), AutoUpdateSettings.UpdateInstallFileText);
}
if (!File.Exists(Path.Combine(DATAFOLDER, MACHINE_FILE)))
File.WriteAllText(Path.Combine(DATAFOLDER, MACHINE_FILE), AutoUpdateSettings.UpdateMachineFileText(InstallID));
}
/// <summary>
/// The unique machine installation ID
/// </summary>
public static string InstallID => _installID.Value;
/// <summary>
/// The unique machine ID, lazy evaluated
/// </summary>
private static readonly Lazy<string> _installID = new(() =>
{
try { return File.ReadAllLines(Path.Combine(DATAFOLDER!, INSTALL_FILE)).FirstOrDefault(x => !string.IsNullOrWhiteSpace(x))?.Trim() ?? ""; }
catch { }
return "";
});
/// <summary>
/// The unique machine ID
/// </summary>
public static string MachineID => _machineID.Value;
/// <summary>
/// The unique machine ID, lazy evaluated
/// </summary>
private static readonly Lazy<string> _machineID = new(() =>
{
string? machinedId = null;
try { machinedId = File.ReadAllLines(Path.Combine(DATAFOLDER!, MACHINE_FILE)).FirstOrDefault(x => !string.IsNullOrWhiteSpace(x))?.Trim() ?? ""; }
catch { }
return string.IsNullOrWhiteSpace(machinedId)
? InstallID
: machinedId;
});
/// <summary>
/// The machine name, lazy evaluated
/// </summary>
private static readonly Lazy<string> _machineName = new(MachineNameReader.GetMachineName);
/// <summary>
/// The machine name
/// </summary>
public static readonly string MachineName = _machineName.Value;
}
Duplicati/Library/AutoUpdater/PackageHelper.cs
+10
View File
@@ -89,6 +89,14 @@ namespace Duplicati.Library.AutoUpdater
/// The secret tool
/// </summary>
SecretTool,
/// <summary>
/// The sync tool
/// </summary>
SyncTool,
/// <summary>
/// The source tool
/// </summary>
SourceTool
}
@@ -115,6 +123,8 @@ namespace Duplicati.Library.AutoUpdater
NamedExecutable.Service => OperatingSystem.IsWindows() ? "Duplicati.Service.exe" : "duplicati-service",
NamedExecutable.Agent => OperatingSystem.IsWindows() ? "Duplicati.Agent.exe" : "duplicati-agent",
NamedExecutable.SecretTool => OperatingSystem.IsWindows() ? "Duplicati.CommandLine.SecretTool.exe" : "duplicati-secret-tool",
NamedExecutable.SyncTool => OperatingSystem.IsWindows() ? "Duplicati.CommandLine.SyncTool.exe" : "duplicati-sync-tool",
NamedExecutable.SourceTool => OperatingSystem.IsWindows() ? "Duplicati.CommandLine.Sourcetool.exe" : "duplicati-source-tool",
_ => throw new ArgumentException($"Named executable not known: {exe}", nameof(exe))
};
Duplicati/Library/AutoUpdater/PreloadSettingsLoader.cs
+3 -47
View File
@@ -74,7 +74,7 @@ public static class PreloadSettingsLoader
// The path for preload settings with the install directory
Path.Combine(
Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) ?? "",
"preload.json"
FILE_NAME
),
// The path for preload settings specified with an environment variable
@@ -109,11 +109,7 @@ public static class PreloadSettingsLoader
// User-context path for preload settings,
// the same default path as where other data is stored
Path.Combine(
Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
FOLDER_NAME,
FILE_NAME
)
Path.Combine(DataFolderManager.DATAFOLDER, FILE_NAME),
}
.Concat(PortablePreloadPaths)
.Where(x => !string.IsNullOrEmpty(x))
@@ -136,7 +132,7 @@ public static class PreloadSettingsLoader
/// <param name="dbsettings">The database settings</param>
public static void ConfigurePreloadSettings(ref string[] arguments, PackageHelper.NamedExecutable executable, out Dictionary<string, string?> dbsettings)
{
var (env, args, db) = GetExecutableMergedSettings(executable, IsPortableModeRequested(arguments));
var (env, args, db) = GetExecutableMergedSettings(executable, DataFolderManager.PORTABLE_MODE);
dbsettings = db;
ApplyEnvironmentVariables(env);
@@ -151,46 +147,6 @@ public static class PreloadSettingsLoader
private static string GetArgumentName(string arg)
=> arg.Split('=', 2)[0];
/// <summary>
/// Helper function to check if portable mode is requested.
/// This is a rudimentary function that mimics the commandline parser,
/// which is not loaded during preload
/// </summary>
/// <param name="args">The arguments to check</param>
/// <returns><c>true</c> if portable mode is requested, <c>false</c> otherwise</returns>
private static bool IsPortableModeRequested(string[] args)
{
var candiate = args.Select((x, i) => (x, i))
.Where(x => x.x != null)
.Where(x => x.x.Equals("--portable-mode", StringComparison.OrdinalIgnoreCase) || x.x.StartsWith("--portable-mode=", StringComparison.OrdinalIgnoreCase))
.LastOrDefault();
if (candiate.x == null)
return false;
var value = (string?)null;
if (candiate.x.Contains("="))
value = candiate.x.Split('=', 2)[1];
else
{
value = args.ElementAtOrDefault(candiate.i + 1);
if (value == null || value.StartsWith("--"))
value = "true";
}
if (value == null)
return false;
value = value.TrimStart('"').TrimEnd('"').ToLowerInvariant();
return value switch
{
"1" => true,
"on" => true,
"true" => true,
"yes" => true,
_ => false,
};
}
/// <summary>
/// Gets the merged settings for the given executable
Duplicati/Library/AutoUpdater/Program.cs
+2 -1
View File
@@ -53,6 +53,7 @@ namespace Duplicati.Library.AutoUpdater
return 0;
case "check":
{
Console.WriteLine($"Checking for updates on channel {AutoUpdateSettings.DefaultUpdateChannel} ...");
var update = UpdaterManager.CheckForUpdate();
if (update == null)
Console.WriteLine("No updates found");
@@ -116,7 +117,7 @@ namespace Duplicati.Library.AutoUpdater
Console.WriteLine("{0} - Choose different channel than the default {1}, valid settings: {2}", string.Format(AutoUpdateSettings.UPDATECHANNEL_ENVNAME_TEMPLATE, AutoUpdateSettings.AppName), AutoUpdater.AutoUpdateSettings.DefaultUpdateChannel, string.Join(",", Enum.GetNames(typeof(ReleaseType)).Where(x => x != ReleaseType.Unknown.ToString())));
Console.WriteLine();
Console.WriteLine("Updates are downloaded from: {0}", string.Join(";", AutoUpdateSettings.URLs));
Console.WriteLine("Machine settings are installed in: {0}", UpdaterManager.UPDATEDIR);
Console.WriteLine("Settings and configuration files are placed in: {0}", DataFolderManager.DATAFOLDER);
Console.WriteLine("This version is \"{0}\" ({1}) and is installed in: {2}", UpdaterManager.SelfVersion.Displayname, UpdaterManager.SelfVersion.Version, UpdaterManager.INSTALLATIONDIR);
Console.WriteLine();
}
Duplicati/Library/AutoUpdater/UpdaterManager.cs
+85 -186
View File
@@ -1,4 +1,4 @@
// Copyright (C) 2025, The Duplicati Team
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
@@ -19,6 +19,8 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Linq;
using System.Collections.Generic;
@@ -28,32 +30,32 @@ using System.Diagnostics;
using System.Text.Json;
using System.Net.Http;
using System.Threading;
using Duplicati.Library.Common.IO;
namespace Duplicati.Library.AutoUpdater
{
/// <summary>
/// Handles operations related to updating the application
/// </summary>
public static class UpdaterManager
{
/// <summary>
/// The RSA key used to sign the manifest
/// </summary>
private static readonly System.Security.Cryptography.RSA[] SIGN_KEYS = AutoUpdateSettings.SignKeys;
private static System.Security.Cryptography.RSA[] SIGN_KEYS => AutoUpdateSettings.SignKeys;
/// <summary>
/// Urls to check for updated packages
/// </summary>
private static readonly string[] MANIFEST_URLS = AutoUpdateSettings.URLs;
private static string[] MANIFEST_URLS => AutoUpdateSettings.URLs;
/// <summary>
/// The app name to show
/// </summary>
private static readonly string APPNAME = AutoUpdateSettings.AppName;
private static string APPNAME => AutoUpdateSettings.AppName;
/// <summary>
/// The version that the updater supports
/// </summary>
public const int SUPPORTED_PACKAGE_UPDATER_VERSION = 2;
/// <summary>
/// The folder where the machine id is placed
/// </summary>
public static readonly string UPDATEDIR;
/// <summary>
/// The directory where the program is running from
/// </summary>
public static readonly string INSTALLATIONDIR;
@@ -62,6 +64,10 @@ namespace Duplicati.Library.AutoUpdater
/// </summary>
public static readonly bool DISABLE_UPDATE_CHECK = Debugger.IsAttached || Utility.Utility.ParseBool(Environment.GetEnvironmentVariable(string.Format(SKIPUPDATE_ENVNAME_TEMPLATE, APPNAME)), false);
/// <summary>
/// The operating system display name
/// </summary>
public static readonly string OperatingSystemName = OperatingSystem.IsWindows() ? "Windows" : OperatingSystem.IsLinux() ? "Linux" : OperatingSystem.IsMacOS() ? "MacOS" : "Unknown";
/// <summary>
/// The update information for the running version
/// </summary>
public static readonly UpdateInfo SelfVersion;
@@ -69,17 +75,13 @@ namespace Duplicati.Library.AutoUpdater
/// <summary>
/// Event trigger for errors on update
/// </summary>
public static event Action<Exception> OnError;
public static event Action<Exception>? OnError;
/// <summary>
/// Common formatting string for date-time values
/// </summary>
private const string DATETIME_FORMAT = "yyyymmddhhMMss";
/// <summary>
/// The template for the environment variable name that allows an overriden root folder
/// </summary>
private const string UPDATEINSTALLDIR_ENVNAME_TEMPLATE = "AUTOUPDATER_{0}_UPDATE_ROOT";
/// <summary>
/// The template for the environment variable that toggles disabling updates
/// </summary>
public const string SKIPUPDATE_ENVNAME_TEMPLATE = "AUTOUPDATER_{0}_SKIP_UPDATE";
@@ -91,20 +93,11 @@ namespace Duplicati.Library.AutoUpdater
/// The name of the file that contains the package type id, located in the <see cref="INSTALLATIONDIR"/> folder
/// </summary>
private const string PACKAGE_TYPE_FILE = "package_type_id.txt";
/// <summary>
/// The installation ID filename stored in <see cref="UPDATEDIR"/>
/// </summary>
private const string INSTALL_FILE = "installation.txt";
/// <summary>
/// The machine ID filename stored in <see cref="UPDATEDIR"/>
/// </summary>
private const string MACHINE_FILE = "machineid.txt";
/// <summary>
/// Gets the last version found from an update
/// </summary>
public static UpdateInfo LastUpdateCheckVersion { get; private set; }
public static UpdateInfo? LastUpdateCheckVersion { get; private set; }
/// <summary>
/// The default timeout in seconds for download operations
@@ -122,65 +115,10 @@ namespace Duplicati.Library.AutoUpdater
static UpdaterManager()
{
// Set the installation path
INSTALLATIONDIR = Path.GetDirectoryName(Duplicati.Library.Utility.Utility.getEntryAssembly().Location);
// Check for override
if (string.IsNullOrWhiteSpace(System.Environment.GetEnvironmentVariable(string.Format(UPDATEINSTALLDIR_ENVNAME_TEMPLATE, APPNAME))))
{
// OS specific folders for probing
var candidates = new List<string>();
if (OperatingSystem.IsWindows())
{
candidates.Add(System.IO.Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), APPNAME));
candidates.Add(System.IO.Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), APPNAME));
}
else
{
candidates.Add(System.IO.Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), APPNAME));
}
if (OperatingSystem.IsMacOS())
candidates.Add(System.IO.Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Personal), "Library", "Application Support", APPNAME));
// Find the first writeable directory in the list
UPDATEDIR = candidates.FirstOrDefault(p => !string.IsNullOrWhiteSpace(p) && System.IO.Directory.Exists(p) && TestDirectoryIsWriteable(p));
// Try to create a writeable folder, if none is found
if (string.IsNullOrWhiteSpace(UPDATEDIR))
UPDATEDIR = candidates.Where(p => !string.IsNullOrWhiteSpace(p) && !System.IO.Directory.Exists(p))
.Select(p =>
{
try { System.IO.Directory.CreateDirectory(p); }
catch { }
return p;
})
.Where(p => System.IO.Directory.Exists(p) && TestDirectoryIsWriteable(p))
.FirstOrDefault();
}
else
{
// Use override, no checks
UPDATEDIR = Environment.ExpandEnvironmentVariables(System.Environment.GetEnvironmentVariable(string.Format(UPDATEINSTALLDIR_ENVNAME_TEMPLATE, APPNAME)));
}
if (!string.IsNullOrWhiteSpace(UPDATEDIR))
{
if (!System.IO.File.Exists(System.IO.Path.Combine(UPDATEDIR, INSTALL_FILE)))
{
// In case there was already a machine id file, copy it to the new location
if (System.IO.File.Exists(System.IO.Path.Combine(UPDATEDIR, "updates", INSTALL_FILE)))
System.IO.File.Copy(System.IO.Path.Combine(UPDATEDIR, "updates", INSTALL_FILE), System.IO.Path.Combine(UPDATEDIR, INSTALL_FILE), true);
else
System.IO.File.WriteAllText(System.IO.Path.Combine(UPDATEDIR, INSTALL_FILE), AutoUpdateSettings.UpdateInstallFileText);
}
if (!System.IO.File.Exists(System.IO.Path.Combine(UPDATEDIR, MACHINE_FILE)))
System.IO.File.WriteAllText(System.IO.Path.Combine(UPDATEDIR, MACHINE_FILE), AutoUpdateSettings.UpdateMachineFileText(InstallID));
}
INSTALLATIONDIR = Util.AppendDirSeparator(Path.GetDirectoryName(Utility.Utility.getEntryAssembly().Location));
// Attempt to read the installed manifest file
UpdateInfo selfVersion = null;
UpdateInfo? selfVersion = null;
try
{
selfVersion = ReadInstalledManifest(INSTALLATIONDIR);
@@ -190,116 +128,62 @@ namespace Duplicati.Library.AutoUpdater
}
// In case the installed manifest is broken, try to set some sane values
if (selfVersion == null)
{
SelfVersion = new UpdateInfo(
MinimumCompatibleVersion: 1,
PackageUpdaterVersion: 1,
IncompatibleUpdateUrl: string.Empty,
GenericUpdatePageUrl: "https://duplicati.com/download",
UpdateSeverity: null,
ChangeInfo: null,
Packages: null,
Displayname: string.IsNullOrWhiteSpace(Duplicati.License.VersionNumbers.TAG) ? "Current" : Duplicati.License.VersionNumbers.TAG,
Version: System.Reflection.Assembly.GetExecutingAssembly().GetName().Version.ToString(),
ReleaseTime: new DateTime(0),
ReleaseType:
SelfVersion = selfVersion ?? new UpdateInfo(
MinimumCompatibleVersion: 1,
PackageUpdaterVersion: 1,
IncompatibleUpdateUrl: string.Empty,
GenericUpdatePageUrl: "https://duplicati.com/download",
UpdateSeverity: null,
ChangeInfo: null,
Packages: null,
Displayname: string.IsNullOrWhiteSpace(License.VersionNumbers.TAG) ? "Current" : License.VersionNumbers.TAG,
Version: System.Reflection.Assembly.GetExecutingAssembly().GetName().Version?.ToString(),
ReleaseTime: new DateTime(0),
ReleaseType:
#if DEBUG
"Debug"
"Debug"
#else
string.IsNullOrWhiteSpace(AutoUpdateSettings.BuildUpdateChannel) ? "Nightly" : AutoUpdateSettings.BuildUpdateChannel
#endif
);
}
);
}
public static Version TryParseVersion(string str)
public static Version TryParseVersion(string? str)
{
Version v;
if (Version.TryParse(str, out v))
if (Version.TryParse(str, out var v))
return v;
else
return new Version(0, 0);
}
private static bool TestDirectoryIsWriteable(string path)
{
var p2 = System.IO.Path.Combine(path, "test-" + DateTime.UtcNow.ToString(DATETIME_FORMAT, System.Globalization.CultureInfo.InvariantCulture));
var probe = System.IO.Directory.Exists(path) ? p2 : path;
if (!System.IO.Directory.Exists(probe))
{
try
{
System.IO.Directory.CreateDirectory(probe);
if (probe != path)
System.IO.Directory.Delete(probe);
return true;
}
catch
{
}
}
return false;
}
/// <summary>
/// The unique machine installation ID
/// </summary>
public static string InstallID
{
get
{
try { return System.IO.File.ReadAllLines(System.IO.Path.Combine(UPDATEDIR, INSTALL_FILE)).FirstOrDefault(x => !string.IsNullOrWhiteSpace(x))?.Trim() ?? ""; }
catch { }
return "";
}
}
/// <summary>
/// The unique machine ID
/// </summary>
public static string MachineID
{
get
{
string machinedId = null;
try { machinedId = System.IO.File.ReadAllLines(System.IO.Path.Combine(UPDATEDIR, MACHINE_FILE)).FirstOrDefault(x => !string.IsNullOrWhiteSpace(x))?.Trim() ?? ""; }
catch { }
return string.IsNullOrWhiteSpace(machinedId)
? InstallID
: machinedId;
}
}
/// <summary>
/// The machine name
/// </summary>
public static readonly string MachineName = MachineNameReader.GetMachineName();
/// <summary>
/// The package type ID
/// </summary>
public static string PackageTypeId
public static string PackageTypeId => _packageTypeId.Value;
/// <summary>
/// The package type ID, lazy evaluated
/// </summary>
private static readonly Lazy<string> _packageTypeId = new(() =>
{
get
{
try { return System.IO.File.ReadAllLines(System.IO.Path.Combine(INSTALLATIONDIR, PACKAGE_TYPE_FILE)).FirstOrDefault(x => !string.IsNullOrWhiteSpace(x))?.Trim() ?? ""; }
catch { }
try { return File.ReadAllLines(Path.Combine(INSTALLATIONDIR!, PACKAGE_TYPE_FILE)).FirstOrDefault(x => !string.IsNullOrWhiteSpace(x))?.Trim() ?? ""; }
catch { }
#if DEBUG
return "debug";
return "debug";
#else
return "";
return "";
#endif
}
}
});
public static UpdateInfo CheckForUpdate(ReleaseType channel = ReleaseType.Unknown)
/// <summary>
/// Checks for updates and returns the update information if available
/// </summary>
/// <param name="channel">The release channel to check for updates</param>
/// <returns>The update information if available, or null if no updates are available</returns>
public static UpdateInfo? CheckForUpdate(ReleaseType channel = ReleaseType.Unknown)
{
if (channel == ReleaseType.Unknown)
channel = AutoUpdateSettings.DefaultUpdateChannel;
@@ -330,20 +214,21 @@ namespace Duplicati.Library.AutoUpdater
if (SIGN_KEYS.Length == 0)
throw new Exception("No signing keys are available, cannot check update");
using (var tmpfile = new Library.Utility.TempFile())
using (var tmpfile = new TempFile())
{
using var request = new HttpRequestMessage(HttpMethod.Get, url);
request.Headers.Add(System.Net.HttpRequestHeader.UserAgent.ToString(), string.Format("{0} v{1}{2}", APPNAME, SelfVersion.Version, string.IsNullOrWhiteSpace(InstallID) ? "" : " -" + InstallID));
request.Headers.Add("X-Install-ID", InstallID);
request.Headers.Add(System.Net.HttpRequestHeader.UserAgent.ToString(), string.Format("{0} v{1}{2}", APPNAME, SelfVersion.Version, string.IsNullOrWhiteSpace(DataFolderManager.InstallID) ? "" : " -" + DataFolderManager.InstallID));
request.Headers.Add("X-Install-ID", DataFolderManager.InstallID);
request.Headers.Add("X-Package-Type-ID", PackageTypeId);
using var timeoutToken = new CancellationTokenSource();
timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS));
HttpClientHelper.DefaultClient.DownloadFile(request, tmpfile, null, timeoutToken.Token).ConfigureAwait(false).GetAwaiter().GetResult();
using (var client = HttpClientHelper.CreateClient())
client.DownloadFile(request, tmpfile, null, timeoutToken.Token).Await();
using (var fs = System.IO.File.OpenRead(tmpfile))
using (var fs = File.OpenRead(tmpfile))
{
var verifyOps = SIGN_KEYS.Select(k => new JSONSignature.VerifyOperation(
Algorithm: JSONSignature.RSA_SHA256,
@@ -354,6 +239,8 @@ namespace Duplicati.Library.AutoUpdater
throw new Exception("No valid signature found in manifest file");
var update = JsonSerializer.Deserialize<UpdateInfo>(fs);
if (update == null)
return null;
if (TryParseVersion(update.Version) <= TryParseVersion(SelfVersion.Version))
return null;
@@ -394,10 +281,15 @@ namespace Duplicati.Library.AutoUpdater
return null;
}
private static UpdateInfo ReadInstalledManifest(string folder)
/// <summary>
/// Reads the installed manifest file
/// </summary>
/// <param name="folder">The folder to read the manifest from</param>
/// <returns>The manifest if found, or null if not found</returns>
private static UpdateInfo? ReadInstalledManifest(string folder)
{
var manifest = System.IO.Path.Combine(folder, UPDATE_MANIFEST_FILENAME);
if (System.IO.File.Exists(manifest))
var manifest = Path.Combine(folder, UPDATE_MANIFEST_FILENAME);
if (File.Exists(manifest))
{
try
{
@@ -406,7 +298,7 @@ namespace Duplicati.Library.AutoUpdater
PublicKey: k.ToXmlString(false)
));
using (var fs = System.IO.File.OpenRead(manifest))
using (var fs = File.OpenRead(manifest))
{
if (!JSONSignature.VerifyAtLeastOne(fs, verifyOps))
throw new Exception("Installed manifest signature is invalid");
@@ -424,11 +316,16 @@ namespace Duplicati.Library.AutoUpdater
return null;
}
public static bool DownloadUpdate(UpdateInfo version, PackageEntry package, string targetPath, Action<double> progress = null)
/// <summary>
/// Downloads the update package
/// </summary>
/// <param name="version">The version to download</param>
/// <param name="package">The package to download</param>
/// <param name="targetPath">The path to save the downloaded package to</param>
/// <param name="progress">The progress callback</param>
/// <returns>True if the download was successful, otherwise false</returns>
public static bool DownloadUpdate(UpdateInfo version, PackageEntry package, string targetPath, Action<double>? progress = null)
{
if (UPDATEDIR == null)
return false;
var updates = package.RemoteUrls.ToList();
// If alternate update URLs are specified,
@@ -455,21 +352,22 @@ namespace Duplicati.Library.AutoUpdater
{
try
{
using (var tempfile = System.IO.File.Open(tempfilename, FileMode.Create, FileAccess.ReadWrite, FileShare.None))
using (var tempfile = File.Open(tempfilename, FileMode.Create, FileAccess.ReadWrite, FileShare.None))
{
Action<long> cb = null;
Action<long>? cb = null;
if (progress != null)
cb = (s) => { progress(Math.Min(1.0, Math.Max(0.0, (double)s / package.Length))); };
using var request = new HttpRequestMessage(HttpMethod.Get, url);
request.Headers.Add(System.Net.HttpRequestHeader.UserAgent.ToString(), string.Format("{0} v{1}", APPNAME, SelfVersion.Version));
request.Headers.Add("X-Install-ID", InstallID);
request.Headers.Add("X-Install-ID", DataFolderManager.InstallID);
using var timeoutToken = new CancellationTokenSource();
timeoutToken.CancelAfter(TimeSpan.FromSeconds(DOWNLOAD_OPERATION_TIMEOUT_SECONDS));
HttpClientHelper.DefaultClient.DownloadFile(request, tempfile, cb, timeoutToken.Token).ConfigureAwait(false).GetAwaiter().GetResult();
using (var client = HttpClientHelper.CreateClient())
client.DownloadFile(request, tempfile, cb, timeoutToken.Token).Await();
var sha256 = System.Security.Cryptography.SHA256.Create();
var md5 = System.Security.Cryptography.MD5.Create();
@@ -512,10 +410,11 @@ namespace Duplicati.Library.AutoUpdater
/// <param name="version">The version of the manifest</param>
/// <param name="incompatibleUpdateUrl">The URL to use for incompatible updates</param>
/// <param name="genericUpdatePageUrl">The URL to use for generic updates</param>
public static void CreateSignedManifest(IEnumerable<System.Security.Cryptography.RSA> keys, string sourcedata, string outputfolder, string version = null, string incompatibleUpdateUrl = null, string genericUpdatePageUrl = null, string releaseType = null, IEnumerable<PackageEntry> packages = null)
public static void CreateSignedManifest(IEnumerable<System.Security.Cryptography.RSA> keys, string sourcedata, string outputfolder, string? version = null, string? incompatibleUpdateUrl = null, string? genericUpdatePageUrl = null, string? releaseType = null, IEnumerable<PackageEntry>? packages = null)
{
// Read the existing manifest
var remoteManifest = JsonSerializer.Deserialize<UpdateInfo>(string.IsNullOrWhiteSpace(sourcedata) ? "{}" : sourcedata);
var remoteManifest = JsonSerializer.Deserialize<UpdateInfo>(string.IsNullOrWhiteSpace(sourcedata) ? "{}" : sourcedata)
?? throw new Exception("Failed to deserialize the manifest from source data");
if (remoteManifest.ReleaseTime.Ticks == 0)
remoteManifest = remoteManifest with { ReleaseTime = DateTime.UtcNow };
Duplicati/Library/Backend/AliyunOSS/AliyunOSSBackend.cs
+36 -38
View File
@@ -22,11 +22,11 @@ using Aliyun.OSS;
using Aliyun.OSS.Common;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
@@ -99,7 +99,8 @@ namespace Duplicati.Library.Backend.AliyunOSS
});
}
public IEnumerable<IFileEntry> List()
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
var bucketName = _ossOptions.BucketName;
@@ -107,51 +108,48 @@ namespace Duplicati.Library.Backend.AliyunOSS
var client = GetClient(false);
var list = new List<OssObjectSummary>();
try
var nextMarker = string.Empty;
bool isTruncated;
do
{
var nextMarker = string.Empty;
var isTruncated = false;
do
var listObjectsRequest = new ListObjectsRequest(bucketName)
{
var listObjectsRequest = new ListObjectsRequest(bucketName)
{
MaxKeys = 1000,
Marker = nextMarker,
Prefix = prefix,
};
var result = client.ListObjects(listObjectsRequest);
MaxKeys = 1000,
Marker = nextMarker,
Prefix = prefix,
};
ObjectListing result;
try
{
result = await Task.Run(() => client.ListObjects(listObjectsRequest), cancelToken).ConfigureAwait(ConfigureAwaitOptions.ForceYielding);
if (result.HttpStatusCode != HttpStatusCode.OK)
{
throw new Exception(result.HttpStatusCode.ToString());
}
foreach (var summary in result.ObjectSummaries)
{
list.Add(summary);
}
}
catch (OssException ex)
{
Logging.Log.WriteErrorMessage(LOGTAG, "List", ex, "List object failed. {0}, {1}", ex.Message, ex.ErrorCode);
throw;
}
catch (Exception ex)
{
Logging.Log.WriteErrorMessage(LOGTAG, "List", ex, "List object failed. {0}", ex.Message);
throw;
}
nextMarker = result.NextMarker;
isTruncated = result.IsTruncated;
} while (isTruncated);
}
catch (OssException ex)
{
Logging.Log.WriteErrorMessage(LOGTAG, "List", ex, "List object failed. {0}, {1}", ex.Message, ex.ErrorCode);
throw;
}
catch (Exception ex)
{
Logging.Log.WriteErrorMessage(LOGTAG, "List", ex, "List object failed. {0}", ex.Message);
throw;
}
foreach (var summary in result.ObjectSummaries)
{
var fileName = Path.GetFileName(summary.Key);
var time = summary.LastModified; // DateTimeOffset.Parse(summary.LastModified).ToLocalTime().DateTime;
yield return new FileEntry(fileName, summary.Size, time, time);
}
foreach (var item in list)
{
var fileName = Path.GetFileName(item.Key);
var time = item.LastModified; // DateTimeOffset.Parse(item.LastModified).ToLocalTime().DateTime;
yield return new FileEntry(fileName, item.Size, time, time);
}
nextMarker = result.NextMarker;
isTruncated = result.IsTruncated;
} while (isTruncated);
}
public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken)
Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs
+27 -47
View File
@@ -19,12 +19,9 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Azure;
using Azure.Storage.Blobs.Models;
using Duplicati.Library.Interface;
using Microsoft.Azure.Storage.Shared.Protocol;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Backend.AzureBlob
{
@@ -77,33 +74,22 @@ namespace Duplicati.Library.Backend.AzureBlob
{
throw new UserInformationException(Strings.AzureBlobBackend.NoAccessKeyOrSasToken, "AzureNoAccessKeyOrSasToken");
}
_azureBlob = new AzureBlobWrapper(storageAccountName, accessKey, sasToken, containerName);
}
public string DisplayName
{
get { return Strings.AzureBlobBackend.DisplayName; }
}
public string DisplayName => Strings.AzureBlobBackend.DisplayName;
public string ProtocolKey
{
get { return "azure"; }
}
public string ProtocolKey => "azure";
public IEnumerable<IFileEntry> List()
{
return _azureBlob.ListContainerEntries();
}
public IAsyncEnumerable<IFileEntry> ListAsync(CancellationToken cancelToken)
=> _azureBlob.ListContainerEntriesAsync(cancelToken);
public async Task PutAsync(string remotename, string localname, CancellationToken cancelToken)
{
using (var fs = File.Open(localname,
FileMode.Open, FileAccess.Read, FileShare.Read))
{
await PutAsync(remotename, fs, cancelToken);
}
await using var fs = File.Open(localname,
FileMode.Open, FileAccess.Read, FileShare.Read);
await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false);
}
public Task PutAsync(string remotename, Stream input, CancellationToken cancelToken)
@@ -113,12 +99,10 @@ namespace Duplicati.Library.Backend.AzureBlob
public async Task GetAsync(string remotename, string localname, CancellationToken cancellationToken)
{
using (var fs = File.Open(localname,
await using var fs = File.Open(localname,
FileMode.Create, FileAccess.Write,
FileShare.None))
{
await GetAsync(remotename, fs, cancellationToken).ConfigureAwait(false);
}
FileShare.None);
await GetAsync(remotename, fs, cancellationToken).ConfigureAwait(false);
}
public Task GetAsync(string remotename, Stream output, CancellationToken cancellationToken)
@@ -161,42 +145,38 @@ namespace Duplicati.Library.Backend.AzureBlob
}
}
public string Description
{
get
{
return Strings.AzureBlobBackend.Description_v2;
}
}
public string Description => Strings.AzureBlobBackend.DescriptionV2;
public Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(_azureBlob.DnsNames);
public Task TestAsync(CancellationToken cancellationToken)
{
return WrapWithExceptionHandler(Task.Run(() => this.TestList()));
}
=> this.TestListAsync(cancellationToken);
public Task CreateFolderAsync(CancellationToken cancellationToken)
{
return WrapWithExceptionHandler(_azureBlob.AddContainerAsync(cancellationToken));
}
/// <summary>
/// Wraps the task with exception handling
/// </summary>
private async Task WrapWithExceptionHandler(Task task)
{
try
{
await task.ConfigureAwait(false);
}
catch (Microsoft.WindowsAzure.Storage.StorageException e)
when (e.RequestInformation.HttpStatusCode == 404
|| e.RequestInformation.HttpStatusMessage == StorageErrorCodeStrings.ResourceNotFound)
catch (RequestFailedException e)
when (e.Status == 404
|| e.ErrorCode == BlobErrorCode.BlobNotFound
|| e.ErrorCode == BlobErrorCode.ResourceNotFound)
{
throw new FileMissingException(e.Message, e);
}
catch (Microsoft.WindowsAzure.Storage.StorageException e)
when (e.RequestInformation.HttpStatusMessage == StorageErrorCodeStrings.ContainerNotFound
|| e.RequestInformation.HttpStatusMessage == StorageErrorCodeStrings.ContainerBeingDeleted
|| e.RequestInformation.HttpStatusMessage == StorageErrorCodeStrings.ContainerDisabled)
catch (RequestFailedException e)
when (e.ErrorCode == BlobErrorCode.ContainerNotFound
|| e.ErrorCode == BlobErrorCode.ContainerBeingDeleted
|| e.ErrorCode == BlobErrorCode.ContainerDisabled)
{
throw new FolderMissingException(e.Message, e);
}
Duplicati/Library/Backend/AzureBlob/AzureBlobWrapper.cs
+92 -105
View File
@@ -19,16 +19,12 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using System.Runtime.CompilerServices;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using Uri = System.Uri;
namespace Duplicati.Library.Backend.AzureBlob
{
@@ -37,142 +33,133 @@ namespace Duplicati.Library.Backend.AzureBlob
/// </summary>
public class AzureBlobWrapper
{
private readonly string _containerName;
private readonly CloudBlobContainer _container;
private readonly OperationContext _operationContext;
// Note: May need metadata; need to test with Azure blobs
private const BlobListingDetails ListDetails = BlobListingDetails.None;
private readonly BlobContainerClient _container;
/// <summary>
/// Gets an array of DNS names associated with the blob container.
/// </summary>
/// <returns>An array of DNS hostnames for the primary and secondary URIs of the container.</returns>
public string[] DnsNames
{
get
{
var lst = new List<string>();
if (_container != null)
{
if (_container.Uri != null)
lst.Add(_container.Uri.Host);
if (_container.StorageUri != null)
{
if (_container.StorageUri.PrimaryUri != null)
lst.Add(_container.StorageUri.PrimaryUri.Host);
if (_container.StorageUri.SecondaryUri != null)
lst.Add(_container.StorageUri.SecondaryUri.Host);
}
}
if (_container != null && _container.Uri != null) lst.Add(_container.Uri.Host);
return lst.ToArray();
}
}
/// <summary>
/// Initializes a new instance of the AzureBlobWrapper class.
/// </summary>
/// <param name="accountName">The Azure storage account name.</param>
/// <param name="accessKey">The access key for the storage account.</param>
/// <param name="sasToken">The Shared Access Signature (SAS) token for authentication.</param>
/// <param name="containerName">The name of the blob container.</param>
public AzureBlobWrapper(string accountName, string accessKey, string sasToken, string containerName)
{
_operationContext = new()
{
CustomUserAgent = string.Format(
"APN/1.0 Duplicati/{0} AzureBlob/2.0 {1}",
System.Reflection.Assembly.GetExecutingAssembly().GetName().Version,
Microsoft.WindowsAzure.Storage.Shared.Protocol.Constants.HeaderConstants.UserAgent
)
};
string connectionString;
BlobServiceClient blobServiceClient;
if (sasToken != null)
{
connectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};SharedAccessSignature={1}",
accountName, sasToken);
var sasUri = new Uri($"https://{accountName}.blob.core.windows.net/?{sasToken}");
blobServiceClient = new BlobServiceClient(sasUri);
}
else
{
connectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}",
accountName, accessKey);
var connectionString = $"DefaultEndpointsProtocol=https;AccountName={accountName};AccountKey={accessKey};EndpointSuffix=core.windows.net";
blobServiceClient = new BlobServiceClient(connectionString);
}
var storageAccount = CloudStorageAccount.Parse(connectionString);
var blobClient = storageAccount.CreateCloudBlobClient();
_containerName = containerName;
_container = blobClient.GetContainerReference(containerName);
_container = blobServiceClient.GetBlobContainerClient(containerName);
}
/// <summary>
/// Creates a new blob container asynchronously and sets its access permissions to private.
/// </summary>
/// <param name="cancellationToken">A token to monitor for cancellation requests.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
public async Task AddContainerAsync(CancellationToken cancellationToken)
{
await _container.CreateAsync(default, default, _operationContext, cancellationToken).ConfigureAwait(false);
await _container.SetPermissionsAsync(new BlobContainerPermissions { PublicAccess = BlobContainerPublicAccessType.Off }, default, default, _operationContext, cancellationToken).ConfigureAwait(false);
// Even though PublicAccessType.None is by default, we set it explicitly to highlight it.
await _container.CreateAsync(PublicAccessType.None, cancellationToken: cancellationToken).ConfigureAwait(false);
}
public virtual Task GetFileStreamAsync(string keyName, Stream target, CancellationToken cancellationToken)
/// <summary>
/// Downloads a blob to a stream asynchronously.
/// </summary>
/// <param name="keyName">The name of the blob to download.</param>
/// <param name="target">The stream to download the blob to.</param>
/// <param name="cancellationToken">A token to monitor for cancellation requests.</param>
/// <returns>A task that represents the asynchronous download operation.</returns>
public async Task GetFileStreamAsync(string keyName, Stream target, CancellationToken cancellationToken)
{
return _container.GetBlockBlobReference(keyName).DownloadToStreamAsync(target, default, default, _operationContext, cancellationToken);
var blobClient = _container.GetBlobClient(keyName);
await blobClient.DownloadToAsync(target, cancellationToken).ConfigureAwait(false);
}
public virtual Task AddFileStream(string keyName, Stream source, CancellationToken cancelToken)
/// <summary>
/// Uploads a stream to a blob asynchronously.
/// </summary>
/// <param name="keyName">The name to give the uploaded blob.</param>
/// <param name="source">The stream containing the data to upload.</param>
/// <param name="cancelToken">A token to monitor for cancellation requests.</param>
/// <returns>A task that represents the asynchronous upload operation.</returns>
public async Task AddFileStream(string keyName, Stream source, CancellationToken cancelToken)
{
return _container.GetBlockBlobReference(keyName).UploadFromStreamAsync(source, source.Length, default, default, _operationContext, cancelToken);
var blobClient = _container.GetBlobClient(keyName);
await blobClient.UploadAsync(source, true, cancelToken).ConfigureAwait(false);
}
public Task DeleteObjectAsync(string keyName, CancellationToken cancelToken)
/// <summary>
/// Deletes a blob if it exists asynchronously.
/// </summary>
/// <param name="keyName">The name of the blob to delete.</param>
/// <param name="cancelToken">A token to monitor for cancellation requests.</param>
public async Task DeleteObjectAsync(string keyName, CancellationToken cancelToken)
{
return _container.GetBlockBlobReference(keyName).DeleteIfExistsAsync(default, default, default, _operationContext, cancelToken);
var blobClient = _container.GetBlobClient(keyName);
await blobClient.DeleteIfExistsAsync(cancellationToken: cancelToken).ConfigureAwait(false);
}
private async Task<List<IListBlobItem>> ListContainerEntriesAsync()
/// <summary>
/// List container files.
/// </summary>
/// <param name="cancelToken">A token to monitor for cancellation requests.</param>
/// <returns></returns>
/// <exception cref="FolderMissingException">Thrown when the container is not found</exception>
public virtual async IAsyncEnumerable<IFileEntry> ListContainerEntriesAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
var segment = await _container.ListBlobsSegmentedAsync(null, false, ListDetails, null, null, null, _operationContext);
var list = new List<IListBlobItem>();
list.AddRange(segment.Results);
while (segment.ContinuationToken != null)
await using var blobEnumerator = _container.GetBlobsAsync().GetAsyncEnumerator(cancelToken);
while (true)
{
segment = await _container.ListBlobsSegmentedAsync(null, false, ListDetails, null, segment.ContinuationToken, null, _operationContext);
list.AddRange(segment.Results);
}
return list;
}
public virtual List<IFileEntry> ListContainerEntries()
{
var listBlobItems = ListContainerEntriesAsync().GetAwaiter().GetResult();
try
{
return listBlobItems.Select(x =>
bool hasNext;
try
{
var absolutePath = x.StorageUri.PrimaryUri.AbsolutePath;
var containerSegment = string.Concat("/", _containerName, "/");
var blobName = absolutePath.Substring(absolutePath.IndexOf(
containerSegment, System.StringComparison.Ordinal) + containerSegment.Length);
try
{
if (x is CloudBlockBlob cb)
{
var lastModified = new System.DateTime();
if (cb.Properties.LastModified != null)
lastModified = new System.DateTime(cb.Properties.LastModified.Value.Ticks, System.DateTimeKind.Utc);
return new FileEntry(Uri.UrlDecode(blobName.Replace("+", "%2B")), cb.Properties.Length, lastModified, lastModified);
}
}
catch
{
// If the metadata fails to parse, return the basic entry
}
return new FileEntry(Uri.UrlDecode(blobName.Replace("+", "%2B")));
})
.Cast<IFileEntry>()
.ToList();
}
catch (StorageException ex)
{
if (ex.RequestInformation.HttpStatusCode == 404)
hasNext = await blobEnumerator.MoveNextAsync().ConfigureAwait(false);
}
catch (Azure.RequestFailedException ex) when (ex.Status == 404)
{
throw new FolderMissingException(ex);
}
throw;
if (!hasNext) break;
cancelToken.ThrowIfCancellationRequested();
if (blobEnumerator.Current is { } blob)
{
var blobName = Uri.UnescapeDataString(blob.Name.Replace("+", "%2B"));
var lastModified = blob.Properties.LastModified?.UtcDateTime ?? DateTime.UtcNow;
yield return new FileEntry(
blobName,
blob.Properties.ContentLength ?? 0,
lastModified,
lastModified
);
}
}
}
}
Duplicati/Library/Backend/AzureBlob/Duplicati.Library.Backend.AzureBlob.csproj
+3 -10
View File
@@ -4,18 +4,11 @@
<TargetFramework>net8.0</TargetFramework>
<OutputType>Library</OutputType>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Azure.KeyVault.Core" Version="3.0.4" />
<PackageReference Include="Microsoft.Azure.KeyVault.WebKey" Version="3.0.4" />
<PackageReference Include="Microsoft.Azure.Storage.Blob" Version="11.0.1" />
<PackageReference Include="Microsoft.Azure.Storage.Common" Version="11.0.1" />
<PackageReference Include="Microsoft.Azure.Storage.File" Version="11.0.1" />
<PackageReference Include="Microsoft.Azure.Storage.Queue" Version="11.0.1" />
<PackageReference Include="Microsoft.Rest.ClientRuntime" Version="2.3.24" />
<PackageReference Include="Microsoft.Rest.ClientRuntime.Azure" Version="3.3.19" />
<PackageReference Include="WindowsAzure.Storage" Version="9.3.3" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="Azure.Storage.Blobs" Version="12.23.0" />
<PackageReference Include="Azure.Storage.Common" Version="12.22.0" />
</ItemGroup>
<ItemGroup>
Duplicati/Library/Backend/AzureBlob/Strings.cs
+21 -20
View File
@@ -20,23 +20,24 @@
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.AzureBlob.Strings {
internal static class AzureBlobBackend {
public static string Description_v2 { get { return LC.L(@"This backend can read and write data to Azure blob storage. Allowed format is ""azure://bucketname""."); } }
public static string DisplayName { get { return LC.L(@"Azure blob"); } }
public static string ContainerNameDescriptionLong { get { return LC.L(@"All files will be written to the container specified."); } }
public static string ContainerNameDescriptionShort { get { return LC.L(@"The name of the storage container"); } }
public static string NoStorageAccountName { get { return LC.L(@"No Azure storage account name given"); } }
public static string StorageAccountNameDescriptionLong { get { return LC.L(@"The Azure storage account name which can be obtained by clicking the ""Manage Access Keys"" button on the storage account dashboard."); } }
public static string StorageAccountNameDescriptionShort { get { return LC.L(@"The storage account name"); } }
public static string AccessKeyDescriptionLong { get { return LC.L(@"The Azure access key which can be obtained by clicking the ""Manage Access Keys"" button on the storage account dashboard."); } }
public static string AccessKeyDescriptionShort { get { return LC.L(@"The access key"); } }
public static string SasTokenDescriptionLong { get { return LC.L(@"The Azure shared access signature (SAS) token which can be obtained by selecting the ""Shared access signature"" blade on the storage account dashboard, or inside a container blade."); } }
public static string SasTokenDescriptionShort { get { return LC.L(@"The SAS token"); } }
public static string NoAccessKeyOrSasToken { get { return LC.L(@"No Azure access key or SAS token given"); } }
public static string AuthPasswordDescriptionLong { get { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); } }
public static string AuthPasswordDescriptionShort { get { return LC.L(@"Supply the password used to connect to the server"); } }
public static string AuthUsernameDescriptionLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } }
public static string AuthUsernameDescriptionShort { get { return LC.L(@"Supply the username used to connect to the server"); } }
}
}
namespace Duplicati.Library.Backend.AzureBlob.Strings;
internal static class AzureBlobBackend {
public static string DescriptionV2 => LC.L(@"This backend can read and write data to Azure blob storage. Allowed format is ""azure://bucketname"".");
public static string DisplayName => LC.L(@"Azure blob");
public static string ContainerNameDescriptionLong => LC.L(@"All files will be written to the container specified.");
public static string ContainerNameDescriptionShort => LC.L(@"The name of the storage container");
public static string NoStorageAccountName => LC.L(@"No Azure storage account name given");
public static string StorageAccountNameDescriptionLong => LC.L(@"The Azure storage account name which can be obtained by clicking the ""Manage Access Keys"" button on the storage account dashboard.");
public static string StorageAccountNameDescriptionShort => LC.L(@"The storage account name");
public static string AccessKeyDescriptionLong => LC.L(@"The Azure access key which can be obtained by clicking the ""Manage Access Keys"" button on the storage account dashboard.");
public static string AccessKeyDescriptionShort => LC.L(@"The access key");
public static string SasTokenDescriptionLong => LC.L(@"The Azure shared access signature (SAS) token which can be obtained by selecting the ""Shared access signature"" blade on the storage account dashboard, or inside a container blade.");
public static string SasTokenDescriptionShort => LC.L(@"The SAS token");
public static string NoAccessKeyOrSasToken => LC.L(@"No Azure access key or SAS token given");
public static string AuthPasswordDescriptionLong => LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD"".");
public static string AuthPasswordDescriptionShort => LC.L(@"Supply the password used to connect to the server");
public static string AuthUsernameDescriptionLong => LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME"".");
public static string AuthUsernameDescriptionShort => LC.L(@"Supply the username used to connect to the server");
}
Duplicati/Library/Backend/Backblaze/B2.cs
+100 -75
View File
@@ -1,30 +1,33 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using System.Net;
using System.Net.Http.Headers;
using System.Reflection;
using Duplicati.Library.Backend.Backblaze.Model;
using FileEntry = Duplicati.Library.Common.IO.FileEntry;
using System.Runtime.CompilerServices;
namespace Duplicati.Library.Backend.Backblaze;
@@ -78,6 +81,11 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
/// </summary>
private const int B2_RECOMMENDED_CHUNK_SIZE = 100 * 1024 * 1024;
/// <summary>
/// Default retry-after time in seconds for B2 API requests
/// </summary>
private const int B2_RETRY_AFTER_SECONDS = 5;
/// <summary>
/// The name of the B2 bucket being accessed
/// </summary>
@@ -264,7 +272,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
_uploadUrl = await _b2AuthHelper.PostAndGetJsonDataAsync<UploadUrlResponse>(
$"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_get_upload_url",
new UploadUrlRequest { BucketID = GetBucketAsync(cancellationToken).Await().BucketID },
new UploadUrlRequest { BucketID = (await GetBucketAsync(cancellationToken)).BucketID },
combinedCancellationToken.Token
).ConfigureAwait(false);
@@ -275,14 +283,15 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
/// Retrieves the fileID searching by filename.
/// </summary>
/// <param name="filename">Filename</param>
/// <param name="cancelToken">Cancellation Token</param>
/// <returns></returns>
/// <exception cref="FileMissingException">Exception thrown when file is not found</exception>
private string GetFileId(string filename)
private async Task<string> GetFileId(string filename, CancellationToken cancelToken)
{
if (_filecache != null && _filecache.TryGetValue(filename, out var value))
return value.OrderByDescending(x => x.UploadTimestamp).First().FileID;
List();
await RebuildFileCache(cancelToken).ConfigureAwait(false);
if (_filecache != null && _filecache.TryGetValue(filename, out var value1))
return value1.OrderByDescending(x => x.UploadTimestamp).First().FileID;
@@ -350,7 +359,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
}
if (_filecache == null)
await ListAsync(cancelToken).ConfigureAwait(false);
await RebuildFileCache(cancelToken).ConfigureAwait(false);
var uploadUrlData = await GetUploadUrlDataAsync(cancelToken).ConfigureAwait(false);
try
@@ -419,11 +428,11 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
{
if (_filecache == null || !_filecache.ContainsKey(remotename))
await ListAsync(cancellationToken).ConfigureAwait(false);
await RebuildFileCache(cancellationToken).ConfigureAwait(false);
using var request = _filecache != null && _filecache.ContainsKey(remotename)
? _b2AuthHelper.CreateRequest(
$"{DownloadUrl}/b2api/v1/b2_download_file_by_id?fileId={Utility.Uri.UrlEncode(GetFileId(remotename))}")
$"{DownloadUrl}/b2api/v1/b2_download_file_by_id?fileId={Utility.Uri.UrlEncode(await GetFileId(remotename, cancellationToken))}")
: _b2AuthHelper.CreateRequest(
$"{DownloadUrl}/{_urlencodedPrefix}{Utility.Uri.UrlPathEncode(remotename)}");
@@ -455,70 +464,89 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
/// Implementation of interface method for listing remote folder contents
/// </summary>
/// <returns>List of IFileEntry with directory listing result</returns>
private async Task<IEnumerable<IFileEntry>> ListAsync(CancellationToken cancellationToken)
private async Task RebuildFileCache(CancellationToken cancellationToken)
{
_filecache = null;
var cache = new Dictionary<string, List<FileEntity>>();
string nextFileId = null;
string nextFileName = null;
string listVersionUrl = $"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_file_versions";
var listRetryHelper = RetryAfterHelper.CreateOrGetRetryAfterHelper(listVersionUrl);
do
{
using var timeoutToken = new CancellationTokenSource();
timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS));
using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken);
var resp = await _b2AuthHelper.PostAndGetJsonDataAsync<ListFilesResponse>(
$"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_file_versions",
new ListFilesRequest
{
BucketID = GetBucketAsync(cancellationToken).Await().BucketID,
MaxFileCount = _pageSize,
Prefix = _prefix,
StartFileID = nextFileId,
StartFileName = nextFileName
},
combinedCancellationToken.Token
).ConfigureAwait(false);
nextFileId = resp.NextFileID;
nextFileName = resp.NextFileName;
if (resp.Files == null || resp.Files.Length == 0)
break;
foreach (var file in resp.Files)
try
{
if (!file.FileName.StartsWith(_prefix, StringComparison.Ordinal))
continue;
using var timeoutToken = new CancellationTokenSource();
timeoutToken.CancelAfter(TimeSpan.FromSeconds(SHORT_OPERATION_TIMEOUT_SECONDS));
using var combinedCancellationToken =
CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken);
var name = file.FileName[_prefix.Length..];
if (name.Contains('/'))
continue;
var resp = await _b2AuthHelper.PostAndGetJsonDataAsync<ListFilesResponse>(
$"{_b2AuthHelper.ApiUrl}/b2api/v1/b2_list_file_versions",
new ListFilesRequest
{
BucketID = (await GetBucketAsync(cancellationToken)).BucketID,
MaxFileCount = _pageSize,
Prefix = _prefix,
StartFileID = nextFileId,
StartFileName = nextFileName
},
combinedCancellationToken.Token
).ConfigureAwait(false);
cache.TryGetValue(name, out var lst);
if (lst == null)
cache[name] = lst = new List<FileEntity>(1);
lst.Add(file);
nextFileId = resp.NextFileID;
nextFileName = resp.NextFileName;
if (resp.Files == null || resp.Files.Length == 0)
break;
foreach (var file in resp.Files)
{
if (!file.FileName.StartsWith(_prefix, StringComparison.Ordinal))
continue;
var name = file.FileName[_prefix.Length..];
if (name.Contains('/'))
continue;
cache.TryGetValue(name, out var lst);
if (lst == null)
cache[name] = lst = new System.Collections.Generic.List<FileEntity>(1);
lst.Add(file);
}
}
catch (TooManyRequestException tex)
{
// Backblaze's B2 API rate limit reached. Presently they don't add Retry-After headers, we will default to a delay, but respect it if present.
if (tex.RetryAfter == null || (tex.RetryAfter.Date == null && tex.RetryAfter.Delta == null))
listRetryHelper.SetRetryAfter(
new RetryConditionHeaderValue(TimeSpan.FromSeconds(B2_RETRY_AFTER_SECONDS)));
else
listRetryHelper.SetRetryAfter(tex.RetryAfter);
await listRetryHelper.WaitForRetryAfterAsync(cancellationToken);
}
} while (nextFileId != null);
_filecache = cache;
return
(from x in _filecache
let newest = x.Value.OrderByDescending(y => y.UploadTimestamp).First()
let ts = Utility.Utility.EPOCH.AddMilliseconds(newest.UploadTimestamp)
select (IFileEntry)new FileEntry(x.Key, newest.Size, ts, ts)
).ToList();
}
/// <summary>
/// Wrapper method of legacy non async call to list files in the remote folder
/// Implementation of interface method for listing remote folder contents
/// </summary>
/// <returns></returns>
public IEnumerable<IFileEntry> List() => ListAsync(CancellationToken.None).Await();
/// <returns>List of IFileEntry with directory listing result</returns>
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancellationToken)
{
await RebuildFileCache(cancellationToken).ConfigureAwait(false);
foreach (var x in _filecache)
{
var newest = x.Value.OrderByDescending(y => y.UploadTimestamp).First();
var ts = Utility.Utility.EPOCH.AddMilliseconds(newest.UploadTimestamp);
yield return new FileEntry(x.Key, newest.Size, ts, ts);
}
}
//<inheritdoc/>
public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken)
@@ -558,7 +586,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(timeoutToken.Token, cancellationToken);
if (_filecache == null || !_filecache.ContainsKey(remotename))
await ListAsync(cancellationToken).ConfigureAwait(false);
await RebuildFileCache(cancellationToken).ConfigureAwait(false);
if (!_filecache.TryGetValue(remotename, out var value))
throw new FileMissingException();
@@ -588,10 +616,7 @@ public class B2 : IStreamingBackend, ITimeoutExemptBackend
/// </summary>
/// <param name="cancelToken">Cancellation Token</param>
public Task TestAsync(CancellationToken cancelToken)
{
this.TestList();
return Task.CompletedTask;
}
=> this.TestListAsync(cancelToken);
/// <summary>
/// Create remote folder
Duplicati/Library/Backend/Backblaze/B2AuthHelper.cs
+22 -19
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.Net;
@@ -202,6 +202,9 @@ public class B2AuthHelper(string userid, string password, HttpClient httpClient)
if (ex is not HttpRequestException || responseContext == null)
return;
if (ex is HttpRequestException && responseContext != null && responseContext.StatusCode == HttpStatusCode.TooManyRequests)
throw new TooManyRequestException(responseContext.Headers.RetryAfter);
using var stream = responseContext.Content.ReadAsStream();
using var reader = new StreamReader(stream);
var rawData = reader.ReadToEnd();
Duplicati/Library/Backend/Backblaze/Duplicati.Library.Backend.Backblaze.csproj
+1
View File
@@ -8,6 +8,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>
<ItemGroup>
Duplicati/Library/Backend/Box/BoxBackend.cs
+30 -34
View File
@@ -1,32 +1,32 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
@@ -133,7 +133,7 @@ namespace Duplicati.Library.Backend.Box
foreach (var p in m_path.Split(new string[] { "/" }, StringSplitOptions.RemoveEmptyEntries))
{
var el = (MiniFolder)PagedFileListResponse(parentid, true).FirstOrDefault(x => x.Name == p);
var el = (MiniFolder)await PagedFileListResponse(parentid, true, cancelToken).FirstOrDefaultAsync(x => x.Name == p).ConfigureAwait(false);
if (el == null)
{
if (!create)
@@ -159,7 +159,7 @@ namespace Duplicati.Library.Backend.Box
// Make sure we enumerate this, otherwise the m_filecache is empty.
var currentFolder = await GetCurrentFolderWithCacheAsync(cancelToken).ConfigureAwait(false);
PagedFileListResponse(currentFolder, false).LastOrDefault();
await PagedFileListResponse(currentFolder, false, cancelToken).LastOrDefaultAsync().ConfigureAwait(false);
if (m_filecache.ContainsKey(name))
return m_filecache[name];
@@ -167,7 +167,7 @@ namespace Duplicati.Library.Backend.Box
throw new FileMissingException();
}
private IEnumerable<FileEntity> PagedFileListResponse(string parentid, bool onlyfolders)
private async IAsyncEnumerable<FileEntity> PagedFileListResponse(string parentid, bool onlyfolders, [EnumeratorCancellation] CancellationToken cancelToken)
{
var offset = 0;
var done = false;
@@ -177,7 +177,7 @@ namespace Duplicati.Library.Backend.Box
do
{
var resp = m_oauth.GetJSONData<ShortListResponse>(string.Format("{0}/folders/{1}/items?limit={2}&offset={3}&fields=name,size,modified_at", BOX_API_URL, parentid, PAGE_SIZE, offset));
var resp = await m_oauth.GetJSONDataAsync<ShortListResponse>($"{BOX_API_URL}/folders/{parentid}/items?limit={PAGE_SIZE}&offset={offset}&fields=name,size,modified_at", cancelToken).ConfigureAwait(false);
if (resp.Entries == null || resp.Entries.Length == 0)
break;
@@ -221,7 +221,7 @@ namespace Duplicati.Library.Backend.Box
};
if (m_filecache.Count == 0)
PagedFileListResponse(currentFolder, false);
await PagedFileListResponse(currentFolder, false, cancelToken).LastOrDefaultAsync().ConfigureAwait(false);
var existing = m_filecache.ContainsKey(remotename);
@@ -262,12 +262,11 @@ namespace Duplicati.Library.Backend.Box
#region IBackend implementation
public System.Collections.Generic.IEnumerable<IFileEntry> List()
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
var currentFolder = GetCurrentFolderWithCacheAsync(CancellationToken.None).Await();
return
from n in PagedFileListResponse(currentFolder, false)
select (IFileEntry)new FileEntry(n.Name, n.Size, n.ModifiedAt, n.ModifiedAt) { IsFolder = n.Type == "folder" };
var currentFolder = await GetCurrentFolderWithCacheAsync(cancelToken).ConfigureAwait(false);
await foreach (var n in PagedFileListResponse(currentFolder, false, cancelToken).ConfigureAwait(false))
yield return new FileEntry(n.Name, n.Size, n.ModifiedAt, n.ModifiedAt) { IsFolder = n.Type == "folder" };
}
public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken)
@@ -304,10 +303,7 @@ namespace Duplicati.Library.Backend.Box
}
public Task TestAsync(CancellationToken cancelToken)
{
this.TestList();
return Task.CompletedTask;
}
=> this.TestListAsync(cancelToken);
public Task CreateFolderAsync(CancellationToken cancellationToken)
{
Duplicati/Library/Backend/Box/Duplicati.Library.Backend.Box.csproj
+1
View File
@@ -8,6 +8,7 @@
<ItemGroup>
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>
<ItemGroup>
Duplicati/Library/Backend/CIFS/CIFS.cs
+41 -25
View File
@@ -28,15 +28,18 @@ using System.Threading;
using System.Threading.Tasks;
using Duplicati.Library.Backend.CIFS;
using Duplicati.Library.Backend.CIFS.Model;
using Duplicati.Library.Utility;
using SMBLibrary;
using Duplicati.Library.SourceProvider;
using System.Runtime.CompilerServices;
using Duplicati.Library.Common.IO;
using System.Runtime.CompilerServices;
namespace Duplicati.Library.Backend;
/// <summary>
/// Native CIFS/SMB Backend implementation
/// </summary>
public class CIFSBackend : IStreamingBackend
public class CIFSBackend : IStreamingBackend, IFolderEnabledBackend
{
/// <summary>
/// Implementation of interface property for the backend key
@@ -72,12 +75,12 @@ public class CIFSBackend : IStreamingBackend
/// Shared connection between all methods to avoid re-authentication
/// </summary>
private SMBShareConnection _sharedConnection;
/// <summary>
/// Read buffer size for SMB operations (will be capped automatically by SMB negotiated values)
/// </summary>
private const string READ_BUFFER_SIZE_OPTION = "read-buffer-size";
/// <summary>
/// Write buffer size for SMB operations (will be capped automatically by SMB negotiated values)
/// </summary>
@@ -139,33 +142,33 @@ public class CIFSBackend : IStreamingBackend
var uri = new Utility.Uri(url);
uri.RequireHost();
_DnsName = uri.Host;
var input = uri.Path.TrimEnd('/');
var slashIndex = input.IndexOf('/'); // Find first slash to separate server and share if present.
options.TryGetValue(AUTH_USERNAME_OPTION, out string authUsername);
options.TryGetValue(AUTH_PASSWORD_OPTION, out string authPassword);
options.TryGetValue(AUTH_DOMAIN_OPTION, out string authDomain);
options.TryGetValue(TRANSPORT_OPTION, out string transport);
int? readBufferSize = null, writeBufferSize = null;
options.TryGetValue(READ_BUFFER_SIZE_OPTION, out string readBufferSizeConfig);
if (!string.IsNullOrWhiteSpace(readBufferSizeConfig)) readBufferSize = Int32.TryParse(readBufferSizeConfig, out int value) ? value : null;
options.TryGetValue(WRITE_BUFFER_SIZE_OPTION, out string writeBufferSizeConfig);
if (!string.IsNullOrWhiteSpace(writeBufferSizeConfig)) writeBufferSize = Int32.TryParse(readBufferSizeConfig, out int value) ? value : null;
// Normalize to 10KB minimum buffers size
readBufferSize = readBufferSize < 1024 * 10 ? null : readBufferSize;
writeBufferSize = writeBufferSize < 1024 * 10 ? null : writeBufferSize;
SMBTransportType transportType = _transportMap.TryGetValue(
string.IsNullOrEmpty(transport) ? DEFAULT_TRANSPORT : transport.ToLower(),
string.IsNullOrEmpty(transport) ? DEFAULT_TRANSPORT : transport.ToLower(),
out SMBTransportType type)
? type
: throw new UserInformationException($"Transport must be one of: {string.Join(", ", _transportMap.Keys)}","CIFSConfig");
: throw new UserInformationException($"Transport must be one of: {string.Join(", ", _transportMap.Keys)}", "CIFSConfig");
_connectionParameters = new SMBConnectionParameters(
uri.Host,
transportType,
@@ -196,17 +199,12 @@ public class CIFSBackend : IStreamingBackend
/// Implementation of interface method for listing remote folder contents
/// </summary>
/// <returns>List of IFileEntry with directory listing result</returns>
private async Task<IEnumerable<IFileEntry>> ListAsync(CancellationToken cancellationToken)
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancellationToken)
{
return await GetConnection().ListAsync(cancellationToken).ConfigureAwait(false);
foreach (var v in await GetConnection().ListAsync(_connectionParameters.Path, cancellationToken).ConfigureAwait(false))
yield return v;
}
/// <summary>
/// Wrapper method of legacy non async call to list files in the remote folder
/// </summary>
/// <returns></returns>
public IEnumerable<IFileEntry> List() => ListAsync(CancellationToken.None).Await();
/// <summary>
/// Upload files to remote location
/// </summary>
@@ -282,7 +280,7 @@ public class CIFSBackend : IStreamingBackend
/// </summary>
/// <param name="cancellationToken">CancellationToken, in this call not used.</param>
/// <returns></returns>
public Task<string[]> GetDNSNamesAsync(CancellationToken cancellationToken) =>
public Task<string[]> GetDNSNamesAsync(CancellationToken cancellationToken) =>
Task.FromResult(new[] { _DnsName ?? string.Empty });
/// <summary>
@@ -293,7 +291,7 @@ public class CIFSBackend : IStreamingBackend
public async Task TestAsync(CancellationToken cancellationToken)
{
// This will throw an exception if the folder is missing
await GetConnection().ListAsync(cancellationToken).ConfigureAwait(false);
await GetConnection().ListAsync(_connectionParameters.Path, cancellationToken).ConfigureAwait(false);
}
/// <summary>
@@ -306,10 +304,10 @@ public class CIFSBackend : IStreamingBackend
{
var pathParts = _connectionParameters.Path?
.Split(PATH_SEPARATORS, StringSplitOptions.RemoveEmptyEntries);
if (pathParts == null || pathParts.Length == 0)
return;
await GetConnection().CreateFolderAsync(_connectionParameters.Path, cancellationToken)
.ConfigureAwait(false);
}
@@ -320,6 +318,8 @@ public class CIFSBackend : IStreamingBackend
/// <returns>An SMB connection that can be used for file operations</returns>
private SMBShareConnection GetConnection() => _sharedConnection ??= new SMBShareConnection(_connectionParameters);
/// <summary>
/// Implementation of Dispose pattern enforced by interface
/// </summary>
@@ -335,4 +335,20 @@ public class CIFSBackend : IStreamingBackend
System.Diagnostics.Debug.WriteLine($"Error disposing CIFS connection: {ex.Message}");
}
}
/// <inheritdoc/>
public async IAsyncEnumerable<IFileEntry> ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken)
{
var sourcePath = _connectionParameters.Path;
if (!string.IsNullOrWhiteSpace(sourcePath))
sourcePath = Util.AppendDirSeparator(sourcePath, "/");
foreach (var v in await GetConnection().ListAsync(sourcePath + BackendSourceFileEntry.NormalizePathTo(path, '/'), cancellationToken).ConfigureAwait(false))
if (v.Name != "." && v.Name != "..")
yield return v;
}
/// <inheritdoc/>
public Task<IFileEntry> GetEntryAsync(string path, CancellationToken cancellationToken)
=> Task.FromResult<IFileEntry>(null);
}
Duplicati/Library/Backend/CIFS/Duplicati.Library.Backend.CIFS.csproj
+1
View File
@@ -10,6 +10,7 @@
<ProjectReference Include="..\..\Interface\Duplicati.Library.Interface.csproj" />
<ProjectReference Include="..\..\Localization\Duplicati.Library.Localization.csproj" />
<ProjectReference Include="..\..\Utility\Duplicati.Library.Utility.csproj" />
<ProjectReference Include="..\..\SourceProvider\Builtin\Duplicati.Library.SourceProvider.Builtin.csproj" />
</ItemGroup>
<ItemGroup>
Duplicati/Library/Backend/CIFS/SMBShareConnection.cs
+8 -7
View File
@@ -79,7 +79,7 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable
if (!_smb2Client.Connect(connectionParameters.ServerName, connectionParameters.TransportType))
throw new UserInformationException($"{LC.L("Failed to connect to server")} {connectionParameters.ServerName}", "ConnectionError");
var status = _smb2Client.Login(connectionParameters.AuthDomain, connectionParameters.AuthUser, connectionParameters.AuthPassword);
var status = _smb2Client.Login(connectionParameters.AuthDomain ?? "", connectionParameters.AuthUser ?? "", connectionParameters.AuthPassword ?? "");
if (status != NTStatus.STATUS_SUCCESS)
throw new UserInformationException($"{LC.L("Failed to authenticate to server")} {connectionParameters.ServerName} with status {status}", "ConnectionError");
@@ -194,9 +194,10 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable
/// <summary>
/// Lists the folder contents of the share and path specified in the connection parameters.
/// </summary>
/// <param name="path">Path to list</param>
/// <param name="cancellationToken">Cancellation Token</param>
/// <exception cref="UserInformationException">Exception to be displayed to user</exception>
public async Task<List<IFileEntry>> ListAsync(CancellationToken cancellationToken)
public async Task<List<IFileEntry>> ListAsync(string path, CancellationToken cancellationToken)
{
await _semaphore.WaitAsync(cancellationToken);
@@ -206,11 +207,10 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable
{
try
{
var status = _smbFileStore.CreateFile(
out directoryHandle,
out fileStatus,
NormalizeSlashes(_connectionParameters.Path),
NormalizeSlashes(path),
AccessMask.GENERIC_READ,
FileAttributes.Directory,
ShareAccess.Read | ShareAccess.Write,
@@ -222,7 +222,7 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable
if (status == NTStatus.STATUS_OBJECT_PATH_NOT_FOUND || status == NTStatus.STATUS_OBJECT_NAME_NOT_FOUND)
throw new FolderMissingException();
else
throw new UserInformationException($"{LC.L("Failed to open directory")} {NormalizeSlashes(_connectionParameters.Path)} with status {status.ToString()}", "DirectoryOpenError");
throw new UserInformationException($"{LC.L("Failed to open directory")} {NormalizeSlashes(path)} with status {status}", "DirectoryOpenError");
List<QueryDirectoryFileInformation> fileList;
status = _smbFileStore.QueryDirectory(
@@ -232,7 +232,7 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable
FileInformationClass.FileDirectoryInformation);
if (status != NTStatus.STATUS_NO_MORE_FILES)
throw new UserInformationException($"{LC.L("Failed to query directory contents")} with status {status.ToString()}", "DirectoryQueryError");
throw new UserInformationException($"{LC.L("Failed to query directory contents")} with status {status}", "DirectoryQueryError");
return
[
@@ -244,7 +244,8 @@ public class SMBShareConnection : IDisposable, IAsyncDisposable
info.LastAccessTime,
info.LastWriteTime)
{
IsFolder = info.FileAttributes == FileAttributes.Directory
IsFolder = info.FileAttributes == FileAttributes.Directory,
Created = info.CreationTime
})
.ToList()
];
Duplicati/Library/Backend/CloudFiles/CloudFiles.cs
+28 -31
View File
@@ -1,31 +1,31 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Runtime.CompilerServices;
using System.Security.Cryptography;
using System.Threading;
using System.Threading.Tasks;
@@ -133,10 +133,10 @@ namespace Duplicati.Library.Backend
get { return "cloudfiles"; }
}
public IEnumerable<IFileEntry> List()
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
string extraUrl = "?format=xml&limit=" + ITEM_LIST_LIMIT.ToString();
string markerUrl = "";
var extraUrl = "?format=xml&limit=" + ITEM_LIST_LIMIT.ToString();
var markerUrl = "";
bool repeat;
@@ -169,14 +169,14 @@ namespace Duplicati.Library.Backend
//The response should be 404 from the server, but it is not :(
if (lst.Count == 0 && markerUrl == "") //Only on first iteration
{
try { CreateFolderAsync(CancellationToken.None).Await(); }
try { await CreateFolderAsync(cancelToken).ConfigureAwait(false); }
catch { } //Ignore
}
string lastItemName = "";
var lastItemName = "";
foreach (System.Xml.XmlNode n in lst)
{
string name = n["name"].InnerText;
var name = n["name"].InnerText;
long size;
DateTime mod;
@@ -264,12 +264,9 @@ namespace Duplicati.Library.Backend
#region IBackend_v2 Members
public Task TestAsync(CancellationToken cancelToken)
{
public Task TestAsync(CancellationToken cancelToken) =>
//The "Folder not found" is not detectable :(
this.TestList();
return Task.CompletedTask;
}
this.TestListAsync(cancelToken);
public Task CreateFolderAsync(CancellationToken cancelToken)
{
Duplicati/Library/Backend/Dropbox/Dropbox.cs
+20 -7
View File
@@ -23,6 +23,7 @@ using Duplicati.Library.Interface;
using System;
using System.Collections.Generic;
using System.IO;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
@@ -112,16 +113,31 @@ namespace Duplicati.Library.Backend
}
}
public IEnumerable<IFileEntry> List()
private async Task<T> HandleListExceptions<T>(Func<Task<T>> func)
{
var lfr = HandleListExceptions(() => dbx.ListFiles(m_path));
try
{
return await func().ConfigureAwait(false);
}
catch (DropboxException de)
{
if (de.errorJSON["error"][".tag"].ToString() == "path" && de.errorJSON["error"]["path"][".tag"].ToString() == "not_found")
throw new FolderMissingException();
throw;
}
}
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
var lfr = await HandleListExceptions(() => dbx.ListFiles(m_path, cancelToken)).ConfigureAwait(false);
foreach (var md in lfr.entries)
yield return ParseEntry(md);
while (lfr.has_more)
{
lfr = HandleListExceptions(() => dbx.ListFilesContinue(lfr.cursor));
lfr = await HandleListExceptions(() => dbx.ListFilesContinue(lfr.cursor, cancelToken)).ConfigureAwait(false);
foreach (var md in lfr.entries)
yield return ParseEntry(md);
}
@@ -168,10 +184,7 @@ namespace Duplicati.Library.Backend
public Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(WebApi.Dropbox.Hosts());
public Task TestAsync(CancellationToken cancelToken)
{
this.TestList();
return Task.CompletedTask;
}
=> this.TestListAsync(cancelToken);
public async Task CreateFolderAsync(CancellationToken cancelToken)
{
Duplicati/Library/Backend/Dropbox/DropboxHelper.cs
+4 -4
View File
@@ -43,7 +43,7 @@ namespace Duplicati.Library.Backend
base.AccessTokenOnly = !accessToken.Contains(":");
}
public ListFolderResult ListFiles(string path)
public async Task<ListFolderResult> ListFiles(string path, CancellationToken cancelToken)
{
var pa = new PathArg
{
@@ -52,7 +52,7 @@ namespace Duplicati.Library.Backend
try
{
return PostAndGetJSONData<ListFolderResult>(WebApi.Dropbox.ListFilesUrl(), pa);
return await PostAndGetJSONDataAsync<ListFolderResult>(WebApi.Dropbox.ListFilesUrl(), cancelToken, pa).ConfigureAwait(false);
}
catch (Exception ex)
{
@@ -61,13 +61,13 @@ namespace Duplicati.Library.Backend
}
}
public ListFolderResult ListFilesContinue(string cursor)
public async Task<ListFolderResult> ListFilesContinue(string cursor, CancellationToken cancelToken)
{
var lfca = new ListFolderContinueArg() { cursor = cursor };
try
{
return PostAndGetJSONData<ListFolderResult>(WebApi.Dropbox.ListFilesContinueUrl(), lfca);
return await PostAndGetJSONDataAsync<ListFolderResult>(WebApi.Dropbox.ListFilesContinueUrl(), cancelToken, lfca).ConfigureAwait(false);
}
catch (Exception ex)
{
Duplicati/Library/Backend/FTP/Duplicati.Library.Backend.FTP.csproj
+1
View File
@@ -19,6 +19,7 @@
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>
</Project>
Duplicati/Library/Backend/FTP/FTPBackend.cs
+54 -55
View File
@@ -28,6 +28,7 @@ using System.IO;
using System.Linq;
using System.Net;
using System.Net.Security;
using System.Runtime.CompilerServices;
using System.Security.Authentication;
using System.Text;
using System.Threading;
@@ -324,7 +325,7 @@ namespace Duplicati.Library.Backend
var dataConnectionType = CoreUtility.ParseEnumOption(options, CONFIG_KEY_FTP_DATA_CONNECTION_TYPE, DEFAULT_DATA_CONNECTION_TYPE);
var encryptionMode = CoreUtility.ParseEnumOption(options, CONFIG_KEY_FTP_ENCRYPTION_MODE, DEFAULT_ENCRYPTION_MODE);
var sslProtocols = CoreUtility.ParseEnumOption(options, CONFIG_KEY_FTP_SSL_PROTOCOLS, DEFAULT_SSL_PROTOCOLS);
var sslProtocols = CoreUtility.ParseFlagsOption(options, CONFIG_KEY_FTP_SSL_PROTOCOLS, DEFAULT_SSL_PROTOCOLS);
// Process options of the legacy FTP backend
if (ProtocolKey == "ftp")
@@ -366,63 +367,14 @@ namespace Duplicati.Library.Backend
}
/// <inheritdoc />
public IEnumerable<IFileEntry> List()
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
FtpListItem[] items;
try
{
var list = new List<IFileEntry>();
var client = CreateClient(CancellationToken.None).Await();
var remotePath = PreparePathForClient(null);
foreach (var item in client.GetListing(remotePath, FtpListOption.Modify | FtpListOption.Size).Await())
{
switch (item.Type)
{
case FtpObjectType.Directory:
if (item.Name == "." || item.Name == "..")
continue;
list.Add(new FileEntry(item.Name, -1, new DateTime(), item.Modified)
{
IsFolder = true,
});
break;
case FtpObjectType.File:
list.Add(new FileEntry(item.Name, item.Size, new DateTime(), item.Modified));
break;
case FtpObjectType.Link:
{
if (item.Name == "." || item.Name == "..")
continue;
if (item.LinkObject != null)
{
switch (item.LinkObject.Type)
{
case FtpObjectType.Directory:
if (item.Name == "." || item.Name == "..")
continue;
list.Add(new FileEntry(item.Name, -1, new DateTime(), item.Modified)
{
IsFolder = true,
});
break;
case FtpObjectType.File:
list.Add(new FileEntry(item.Name, item.Size, new DateTime(), item.Modified));
break;
}
}
break;
}
}
}
return list;
items = await client.GetListing(remotePath, FtpListOption.Modify | FtpListOption.Size).ConfigureAwait(false);
}
catch (Exception e)
{
@@ -431,6 +383,54 @@ namespace Duplicati.Library.Backend
throw;
}
foreach (var item in items)
{
switch (item.Type)
{
case FtpObjectType.Directory:
if (item.Name == "." || item.Name == "..")
continue;
yield return new FileEntry(item.Name, -1, new DateTime(), item.Modified)
{
IsFolder = true,
};
break;
case FtpObjectType.File:
yield return new FileEntry(item.Name, item.Size, new DateTime(), item.Modified);
break;
case FtpObjectType.Link:
{
if (item.Name == "." || item.Name == "..")
continue;
if (item.LinkObject != null)
{
switch (item.LinkObject.Type)
{
case FtpObjectType.Directory:
if (item.Name == "." || item.Name == "..")
continue;
yield return new FileEntry(item.Name, -1, new DateTime(), item.Modified)
{
IsFolder = true,
};
break;
case FtpObjectType.File:
yield return new FileEntry(item.Name, item.Size, new DateTime(), item.Modified);
break;
}
}
break;
}
}
}
}
/// <inheritdoc />
@@ -552,8 +552,7 @@ namespace Duplicati.Library.Backend
// Remove the file if it exists
try
{
var list = List();
if (list.Any(entry => entry.Name == TEST_FILE_NAME))
if (await ListAsync(cancellationToken).AnyAsync(entry => entry.Name == TEST_FILE_NAME).ConfigureAwait(false))
await DeleteAsync(TEST_FILE_NAME, cancellationToken).ConfigureAwait(false);
}
catch (Exception e)
Duplicati/Library/Backend/File/Duplicati.Library.Backend.File.csproj
+2
View File
@@ -11,6 +11,7 @@
<ProjectReference Include="..\..\Interface\Duplicati.Library.Interface.csproj" />
<ProjectReference Include="..\..\Localization\Duplicati.Library.Localization.csproj" />
<ProjectReference Include="..\..\Utility\Duplicati.Library.Utility.csproj" />
<ProjectReference Include="..\..\Snapshots\Duplicati.Library.Snapshots.csproj" />
</ItemGroup>
<ItemGroup>
@@ -18,6 +19,7 @@
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>
</Project>
Duplicati/Library/Backend/File/FileBackend.cs
+27 -27
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Common.IO;
@@ -24,6 +24,7 @@ using Duplicati.Library.Interface;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Runtime.Versioning;
using System.Threading;
@@ -172,14 +173,15 @@ namespace Duplicati.Library.Backend
get { return "file"; }
}
public IEnumerable<IFileEntry> List()
/// <inheritdoc />
public IAsyncEnumerable<IFileEntry> ListAsync(CancellationToken cancelToken)
{
PreAuthenticate();
if (!systemIO.DirectoryExists(m_path))
throw new FolderMissingException(Strings.FileBackend.FolderMissingError(m_path));
return systemIO.EnumerateFileEntries(m_path);
return systemIO.EnumerateFileEntries(m_path).ToAsyncEnumerable();
}
#if DEBUG_RETRY
@@ -281,10 +283,7 @@ namespace Duplicati.Library.Backend
}
public Task TestAsync(CancellationToken cancelToken)
{
this.TestList();
return Task.CompletedTask;
}
=> this.TestListAsync(cancelToken);
public Task CreateFolderAsync(CancellationToken cancelToken)
{
@@ -452,14 +451,15 @@ namespace Duplicati.Library.Backend
long? sourceStreamLength = sourceStream == null ? null : Utility.Utility.GetStreamLength(sourceStream, out isStreamPostion);
if (sourceStreamLength.HasValue && targetFileInfo.Length != sourceStreamLength.Value)
throw new FileMissingException($"Target file size ({targetFileInfo.Length:n0}) is different from the source length ({sourceStreamLength.Value:n0}){(isStreamPostion ? " - ending stream position)" : "")}. Target: {targetFilePath}");
throw new FileMissingException($"Target file size ({targetFileInfo.Length:n0}) is different from the source length ({sourceStreamLength.Value:n0}){(isStreamPostion ? " - ending stream position" : "")}. Target: {targetFilePath}");
if (expectedLength.HasValue && targetFileInfo.Length != expectedLength.Value)
throw new FileMissingException($"Target file size ({targetFileInfo.Length:n0}) is different from the expected length ({expectedLength.Value:n0}). Target: {targetFilePath}");
}
catch
{
try { System.IO.File.Delete(targetFilePath); } catch { }
try { System.IO.File.Delete(targetFilePath); }
catch { }
throw;
}
}
Duplicati/Library/Backend/GoogleServices/Duplicati.Library.Backend.GoogleServices.csproj
+1
View File
@@ -7,6 +7,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
</ItemGroup>
<ItemGroup>
Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs
+26 -27
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Backend.GoogleServices;
@@ -27,6 +27,7 @@ using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Net;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
@@ -100,11 +101,11 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
public string storageClass { get; set; }
}
private T HandleListExceptions<T>(Func<T> func)
private async Task<T> HandleListExceptions<T>(Func<Task<T>> func)
{
try
{
return func();
return await func().ConfigureAwait(false);
}
catch (WebException wex)
{
@@ -116,12 +117,13 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
}
#region IBackend implementation
public IEnumerable<IFileEntry> List()
/// <inheritdoc />
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
var url = WebApi.GoogleCloudStorage.ListUrl(m_bucket, Utility.Uri.UrlEncode(m_prefix));
while (true)
{
var resp = HandleListExceptions(() => m_oauth.ReadJSONResponse<ListBucketResponse>(url));
var resp = await HandleListExceptions(() => m_oauth.ReadJSONResponseAsync<ListBucketResponse>(url, cancelToken)).ConfigureAwait(false);
if (resp.items != null)
foreach (var f in resp.items)
@@ -165,10 +167,7 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
}
public Task TestAsync(CancellationToken cancelToken)
{
this.TestList();
return Task.CompletedTask;
}
=> this.TestListAsync(cancelToken);
public async Task CreateFolderAsync(CancellationToken cancelToken)
{
Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs
+32 -33
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Backend.GoogleServices;
@@ -29,6 +29,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
@@ -78,8 +79,7 @@ namespace Duplicati.Library.Backend.GoogleDrive
foreach (var p in path.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries))
{
var res = ListFolder(curparent, true, p).ToArray();
var res = await ListFolder(curparent, true, p, cancelToken).ToArrayAsync(cancelToken).ConfigureAwait(false);
if (res.Length == 0)
{
if (!autocreate)
@@ -119,7 +119,7 @@ namespace Duplicati.Library.Backend.GoogleDrive
return entries;
var currentFolderId = await GetCurrentFolderIdAsync(cancelToken).ConfigureAwait(false);
entries = ListFolder(currentFolderId, false, remotename).ToArray();
entries = await ListFolder(currentFolderId, false, remotename, cancelToken).ToArrayAsync(cancelToken).ConfigureAwait(false);
if (entries == null || entries.Length == 0)
{
@@ -145,7 +145,7 @@ namespace Duplicati.Library.Backend.GoogleDrive
{
// Figure out if we update or create the file
if (m_filecache.Count == 0)
foreach (var file in List()) { /* Enumerate the full listing */ }
await foreach (var file in ListAsync(cancelToken).ConfigureAwait(false)) { /* Enumerate the full listing */ }
GoogleDriveFolderItem[] files;
m_filecache.TryGetValue(remotename, out files);
@@ -188,7 +188,7 @@ namespace Duplicati.Library.Backend.GoogleDrive
{
// Prevent repeated download url lookups
if (m_filecache.Count == 0)
foreach (var file in List()) { /* Enumerate the full listing */ }
await foreach (var file in ListAsync(cancelToken).ConfigureAwait(false)) { /* Enumerate the full listing */ }
var fileId = (await GetFileEntriesAsync(remotename, true, cancelToken).ConfigureAwait(false)).OrderByDescending(x => x.createdDate).First().id;
@@ -203,7 +203,9 @@ namespace Duplicati.Library.Backend.GoogleDrive
#region IBackend implementation
public IEnumerable<IFileEntry> List()
/// <inheritdoc />
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
bool success = false;
try
@@ -211,8 +213,8 @@ namespace Duplicati.Library.Backend.GoogleDrive
m_filecache.Clear();
// For now, this class assumes that List() fully populates the file cache
var currentFolderId = GetCurrentFolderIdAsync(CancellationToken.None).Await();
foreach (var n in ListFolder(currentFolderId))
var currentFolderId = await GetCurrentFolderIdAsync(cancelToken).ConfigureAwait(false);
await foreach (var n in ListFolder(currentFolderId, null, null, cancelToken).ConfigureAwait(false))
{
FileEntry fe = null;
@@ -294,10 +296,7 @@ namespace Duplicati.Library.Backend.GoogleDrive
}
public Task TestAsync(CancellationToken cancelToken)
{
this.TestList();
return Task.CompletedTask;
}
=> this.TestListAsync(cancelToken);
public async Task CreateFolderAsync(CancellationToken cancelToken)
{
@@ -432,7 +431,7 @@ namespace Duplicati.Library.Backend.GoogleDrive
public string rootFolderId { get; set; }
}
private IEnumerable<GoogleDriveFolderItem> ListFolder(string parentfolder, bool? onlyFolders = null, string name = null)
private async IAsyncEnumerable<GoogleDriveFolderItem> ListFolder(string parentfolder, bool? onlyFolders, string name, [EnumeratorCancellation] CancellationToken cancelToken)
{
var fileQuery = new string[] {
string.IsNullOrEmpty(name) ? null : string.Format("title = '{0}'", EscapeTitleEntries(name)),
@@ -446,7 +445,7 @@ namespace Duplicati.Library.Backend.GoogleDrive
while (true)
{
var res = m_oauth.GetJSONData<GoogleDriveListResponse>(url);
var res = await m_oauth.GetJSONDataAsync<GoogleDriveListResponse>(url, cancelToken).ConfigureAwait(false);
foreach (var n in res.items)
yield return n;
Duplicati/Library/Backend/Idrivee2/Duplicati.Library.Backend.Idrivee2.csproj
+1
View File
@@ -8,6 +8,7 @@
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="../../SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj" />
<ProjectReference Include="..\..\Interface\Duplicati.Library.Interface.csproj" />
<ProjectReference Include="..\..\Localization\Duplicati.Library.Localization.csproj" />
<ProjectReference Include="..\..\Utility\Duplicati.Library.Utility.csproj" />
Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs
+20 -16
View File
@@ -21,17 +21,17 @@
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Backend
{
public class Idrivee2Backend : IBackend, IStreamingBackend
public class Idrivee2Backend : IBackend, IStreamingBackend, IFolderEnabledBackend
{
private static readonly string LOGTAG = Logging.Log.LogTagFromType<Idrivee2Backend>();
@@ -80,7 +80,6 @@ namespace Duplicati.Library.Backend
m_s3Client = new S3AwsClient(accessKeyId, accessKeySecret, null, host, null, true, false, options);
}
public string GetRegionEndpoint(string url)
@@ -125,17 +124,10 @@ namespace Duplicati.Library.Backend
public bool SupportsStreaming => true;
public IEnumerable<IFileEntry> List()
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
foreach (IFileEntry file in Connection.ListBucket(m_bucket, m_prefix))
{
((FileEntry)file).Name = file.Name.Substring(m_prefix.Length);
if (file.Name.StartsWith("/", StringComparison.Ordinal) && !m_prefix.StartsWith("/", StringComparison.Ordinal))
((FileEntry)file).Name = file.Name.Substring(1);
await foreach (IFileEntry file in Connection.ListBucketAsync(m_bucket, m_prefix, false, cancelToken).ConfigureAwait(false))
yield return file;
}
}
public async Task PutAsync(string remotename, string localname, CancellationToken cancelToken)
@@ -189,10 +181,7 @@ namespace Duplicati.Library.Backend
}
public Task TestAsync(CancellationToken cancelToken)
{
this.TestList();
return Task.CompletedTask;
}
=> this.TestListAsync(cancelToken);
public Task CreateFolderAsync(CancellationToken cancelToken)
{
@@ -230,5 +219,20 @@ namespace Duplicati.Library.Backend
//AWS SDK encodes the filenames correctly
return m_prefix + name;
}
/// <inheritdoc/>
public IAsyncEnumerable<IFileEntry> ListAsync(string path, CancellationToken cancellationToken)
{
var filterPath = GetFullKey(path);
if (!string.IsNullOrWhiteSpace(filterPath))
filterPath = Util.AppendDirSeparator(filterPath, "/");
return m_s3Client.ListBucketAsync(m_bucket, filterPath, true, cancellationToken);
}
/// <inheritdoc/>
public Task<IFileEntry> GetEntryAsync(string path, CancellationToken cancellationToken)
=> Task.FromResult<IFileEntry>(null);
}
}
Duplicati/Library/Backend/Jottacloud/Jottacloud.cs
+7 -5
View File
@@ -25,6 +25,7 @@ using Duplicati.Library.Localization.Short;
using System;
using System.Collections.Generic;
using System.Net;
using System.Runtime.CompilerServices;
using System.Security.Cryptography;
using System.Threading;
using System.Threading.Tasks;
@@ -182,8 +183,12 @@ namespace Duplicati.Library.Backend
get { return "jottacloud"; }
}
public IEnumerable<IFileEntry> List()
/// <inheritdoc />
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
// Remove warning until this backend is rewritten with HttpClient
await Task.CompletedTask;
var doc = new System.Xml.XmlDocument();
try
{
@@ -330,10 +335,7 @@ namespace Duplicati.Library.Backend
}
public Task TestAsync(CancellationToken cancelToken)
{
this.TestList();
return Task.CompletedTask;
}
=> this.TestListAsync(cancelToken);
public Task CreateFolderAsync(CancellationToken cancelToken)
{
Duplicati/Library/Backend/Mega/MegaBackend.cs
+29 -30
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using CG.Web.MegaApiClient;
@@ -28,7 +28,7 @@ using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using OtpNet;
using Duplicati.Library.Utility;
using System.Runtime.CompilerServices;
namespace Duplicati.Library.Backend.Mega
{
@@ -190,15 +190,17 @@ namespace Duplicati.Library.Backend.Mega
#region IBackend implementation
public IEnumerable<IFileEntry> List()
/// <inheritdoc />
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
if (m_filecache == null)
ResetFileCacheAsync(null, CancellationToken.None).Await();
await ResetFileCacheAsync(null, cancelToken).ConfigureAwait(false);
return
from n in m_filecache.Values
let item = n.OrderByDescending(x => x.ModificationDate).First()
select new FileEntry(item.Name, item.Size, item.ModificationDate ?? new DateTime(0), item.ModificationDate ?? new DateTime(0));
foreach (var n in m_filecache.Values)
{
var item = n.OrderByDescending(x => x.ModificationDate).First();
yield return new FileEntry(item.Name, item.Size, item.ModificationDate ?? new DateTime(0), item.ModificationDate ?? new DateTime(0));
}
}
public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken)
@@ -236,10 +238,7 @@ namespace Duplicati.Library.Backend.Mega
}
public Task TestAsync(CancellationToken cancelToken)
{
this.TestList();
return Task.CompletedTask;
}
=> this.TestListAsync(cancelToken);
public Task CreateFolderAsync(CancellationToken cancelToken)
{
Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs
+26 -23
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Backend.MicrosoftGraph;
using Duplicati.Library.Common.IO;
@@ -31,6 +31,7 @@ using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
@@ -318,9 +319,11 @@ namespace Duplicati.Library.Backend
}
}
public IEnumerable<IFileEntry> List()
/// <inheritdoc />
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
foreach (DriveItem item in this.Enumerate<DriveItem>(string.Format("{0}/root:{1}:/children", this.DrivePrefix, this.RootPath)))
await foreach (var item in this.Enumerate<DriveItem>(string.Format("{0}/root:{1}:/children", this.DrivePrefix, this.RootPath), cancelToken).ConfigureAwait(false))
{
// Exclude non-files and deleted items (not sure if they show up in this listing, but make sure anyway)
if (item.IsFile && !item.IsDeleted)
@@ -667,7 +670,7 @@ namespace Duplicati.Library.Backend
}
}
private IEnumerable<T> Enumerate<T>(string url)
private async IAsyncEnumerable<T> Enumerate<T>(string url, [EnumeratorCancellation] CancellationToken cancelToken)
{
string nextUrl = url;
while (!string.IsNullOrEmpty(nextUrl))
@@ -675,7 +678,7 @@ namespace Duplicati.Library.Backend
GraphCollection<T> results;
try
{
results = this.GetAsync<GraphCollection<T>>(nextUrl, CancellationToken.None).Await();
results = await this.GetAsync<GraphCollection<T>>(nextUrl, cancelToken).ConfigureAwait(false);
}
catch (DriveItemNotFoundException ex)
{
Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs
+29 -27
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
@@ -32,6 +32,7 @@ using System.Text;
using Duplicati.Library.Common.IO;
using System.Threading;
using System.Threading.Tasks;
using System.Runtime.CompilerServices;
namespace Duplicati.Library.Backend.OpenStack
{
@@ -75,7 +76,9 @@ namespace Duplicati.Library.Backend.OpenStack
new KeyValuePair<string, string>("Infomaniak Swiss Backup cluster 1", "https://swiss-backup.infomaniak.com/identity/v3"),
new KeyValuePair<string, string>("Infomaniak Swiss Backup cluster 2", "https://swiss-backup02.infomaniak.com/identity/v3"),
new KeyValuePair<string, string>("Infomaniak Swiss Backup cluster 3", "https://swiss-backup03.infomaniak.com/identity/v3"),
new KeyValuePair<string, string>("Infomaniak Swiss Backup cluster 4", "https://swiss-backup04.infomaniak.com/identity/v3"),
new KeyValuePair<string, string>("Infomaniak Public Cloud 1", "https://api.pub1.infomaniak.cloud/identity/v3"),
new KeyValuePair<string, string>("Infomaniak Public Cloud 2", "https://api.pub2.infomaniak.cloud/identity/v3"),
new KeyValuePair<string, string>("Catalyst Cloud - nz-hlz-1 (NZ)", "https://api.nz-hlz-1.catalystcloud.io:5000/v3"),
new KeyValuePair<string, string>("Catalyst Cloud - nz-por-1 (NZ)", "https://api.nz-por-1.catalystcloud.io:5000/v3"),
};
@@ -528,11 +531,11 @@ namespace Duplicati.Library.Backend.OpenStack
#endregion
#region IBackend implementation
private T HandleListExceptions<T>(Func<T> func)
private async Task<T> HandleListExceptions<T>(Func<Task<T>> func)
{
try
{
return func();
return await func().ConfigureAwait(false);
}
catch (WebException wex)
{
@@ -543,7 +546,8 @@ namespace Duplicati.Library.Backend.OpenStack
}
}
public IEnumerable<IFileEntry> List()
/// <inheritdoc />
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
var plainurl = JoinUrls(SimpleStorageEndPoint, m_container) + string.Format("?format=json&delimiter=/&limit={0}", PAGE_LIMIT);
if (!string.IsNullOrEmpty(m_prefix))
@@ -556,7 +560,7 @@ namespace Duplicati.Library.Backend.OpenStack
var req = m_helper.CreateRequest(url);
req.Accept = "application/json";
var items = HandleListExceptions(() => m_helper.ReadJSONResponse<OpenStackStorageItem[]>(req));
var items = await HandleListExceptions(() => m_helper.ReadJSONResponseAsync<OpenStackStorageItem[]>(req, cancelToken)).ConfigureAwait(false);
foreach (var n in items)
{
var name = n.name;
@@ -596,10 +600,8 @@ namespace Duplicati.Library.Backend.OpenStack
return m_helper.ReadJSONResponseAsync<object>(url, cancelToken, null, "DELETE");
}
public Task TestAsync(CancellationToken cancelToken)
{
this.TestList();
return Task.CompletedTask;
}
=> this.TestListAsync(cancelToken);
public async Task CreateFolderAsync(CancellationToken cancelToken)
{
var url = JoinUrls(SimpleStorageEndPoint, m_container);
Duplicati/Library/Backend/Rclone/Rclone.cs
+25 -27
View File
@@ -1,33 +1,33 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
@@ -192,14 +192,15 @@ namespace Duplicati.Library.Backend
}
public IEnumerable<IFileEntry> List()
/// <inheritdoc />
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
String str_result;
string str_result;
try
{
str_result = RcloneCommandExecuter(rclone_executable, $"lsjson {remote_repo}:{remote_path}", CancellationToken.None).Await();
str_result = await RcloneCommandExecuter(rclone_executable, $"lsjson {remote_repo}:{remote_path}", cancelToken).ConfigureAwait(false);
// this will give an error if the executable does not exist.
}
@@ -297,10 +298,7 @@ namespace Duplicati.Library.Backend
public Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { remote_repo });
public Task TestAsync(CancellationToken cancelToken)
{
this.TestList();
return Task.CompletedTask;
}
=> this.TestListAsync(cancelToken);
public Task CreateFolderAsync(CancellationToken cancelToken)
{
Duplicati/Library/Backend/S3/Duplicati.Library.Backend.S3.csproj
+1 -1
View File
@@ -12,10 +12,10 @@
<PackageReference Include="AWSSDK.IdentityManagement" Version="3.7.402.39" />
<PackageReference Include="AWSSDK.S3" Version="3.7.405.9" />
<PackageReference Include="Minio" Version="3.1.13" />
<PackageReference Include="System.Reactive.Linq" Version="4.3.2" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../../SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj" />
<ProjectReference Include="..\..\Interface\Duplicati.Library.Interface.csproj" />
<ProjectReference Include="..\..\Localization\Duplicati.Library.Localization.csproj" />
<ProjectReference Include="..\..\Utility\Duplicati.Library.Utility.csproj" />
Duplicati/Library/Backend/S3/IS3Client.cs
+11 -11
View File
@@ -22,24 +22,24 @@ using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
namespace Duplicati.Library.Backend
{
public interface IS3Client : IDisposable
{
IEnumerable<FileEntry> ListBucket(string bucketName, string prefix);
public interface IS3Client : IDisposable
{
IAsyncEnumerable<IFileEntry> ListBucketAsync(string bucketName, string prefix, bool recursive, CancellationToken cancellationToken);
Task AddBucketAsync(string bucketName, CancellationToken cancelToken);
Task AddBucketAsync(string bucketName, CancellationToken cancelToken);
Task DeleteObjectAsync(string bucketName, string keyName, CancellationToken cancelToken);
Task DeleteObjectAsync(string bucketName, string keyName, CancellationToken cancelToken);
Task RenameFileAsync(string bucketName, string source, string target, CancellationToken cancelToken);
Task RenameFileAsync(string bucketName, string source, string target, CancellationToken cancelToken);
Task GetFileStreamAsync(string bucketName, string keyName, System.IO.Stream target, CancellationToken cancelToken);
Task GetFileStreamAsync(string bucketName, string keyName, System.IO.Stream target, CancellationToken cancelToken);
string GetDnsHost();
string GetDnsHost();
Task AddFileStreamAsync(string bucketName, string keyName, System.IO.Stream source, CancellationToken cancelToken);
}
Task AddFileStreamAsync(string bucketName, string keyName, System.IO.Stream source, CancellationToken cancelToken);
}
}
Duplicati/Library/Backend/S3/S3AwsClient.cs
+81 -21
View File
@@ -28,6 +28,7 @@ using Duplicati.Library.Utility;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
@@ -69,6 +70,7 @@ namespace Duplicati.Library.Backend
/// The DNS host of the S3 server
/// </summary>
private readonly string m_dnsHost;
private readonly bool m_useV2ListApi;
/// <summary>
/// The archive classes that are considered archive classes
@@ -98,6 +100,7 @@ namespace Duplicati.Library.Backend
m_client = new AmazonS3Client(awsID, awsKey, cfg);
m_useV2ListApi = string.Equals(options.GetValueOrDefault("list-api-version", "v1"), "v2", StringComparison.OrdinalIgnoreCase);
m_locationConstraint = locationConstraint;
m_storageClass = storageClass;
m_dnsHost = string.IsNullOrWhiteSpace(cfg.ServiceURL) ? null : new System.Uri(cfg.ServiceURL).Host;
@@ -243,32 +246,67 @@ namespace Duplicati.Library.Backend
return m_client.DeleteObjectAsync(objectDeleteRequest, cancellationToken);
}
public virtual IEnumerable<FileEntry> ListBucket(string bucketName, string prefix)
/// <summary>
/// Lists the contents of a bucket, using a plugable API call
/// </summary>
/// <param name="bucketName">The bucket to list</param>
/// <param name="prefix">The prefix to list</param>
/// <param name="recursive">If true, the list is recursive</param>
/// <param name="cancellationToken">The cancellation token</param>
/// <returns>The list of files</returns>
public async IAsyncEnumerable<IFileEntry> ListBucketAsync(string bucketName, string prefix, bool recursive, [EnumeratorCancellation] CancellationToken cancellationToken)
{
bool isTruncated = true;
var isTruncated = true;
string filename = null;
var delimiter = recursive ? "" : "/";
if (!string.IsNullOrWhiteSpace(prefix))
prefix = Util.AppendDirSeparator(prefix, "/");
//TODO: Figure out if this is the case with AWSSDK too
//Unfortunately S3 sometimes reports duplicate values when requesting more than one page of results
//So, track the files that have already been returned and skip any duplicates.
HashSet<string> alreadyReturned = new HashSet<string>();
var alreadyReturned = new HashSet<string>();
//We truncate after ITEM_LIST_LIMIT elements, and then repeat
while (isTruncated)
{
var listRequest = new ListObjectsRequest { BucketName = bucketName };
if (!string.IsNullOrEmpty(filename))
listRequest.Marker = filename;
listRequest.MaxKeys = ITEM_LIST_LIMIT;
if (!string.IsNullOrEmpty(prefix))
listRequest.Prefix = prefix;
ListObjectsResponse listResponse;
ListObjectsV2Response listResponse;
try
{
listResponse = m_client.ListObjectsAsync(listRequest).Await();
if (m_useV2ListApi)
{
var listRequest = new ListObjectsV2Request
{
BucketName = bucketName,
Prefix = prefix,
ContinuationToken = filename,
MaxKeys = ITEM_LIST_LIMIT,
Delimiter = delimiter
};
listResponse = await m_client.ListObjectsV2Async(listRequest, cancellationToken).ConfigureAwait(false);
}
else
{
// Use the V1 API
var listRequest = new ListObjectsRequest
{
BucketName = bucketName,
Prefix = prefix,
Marker = filename,
MaxKeys = ITEM_LIST_LIMIT,
Delimiter = delimiter
};
var listResponsev1 = await m_client.ListObjectsAsync(listRequest, cancellationToken).ConfigureAwait(false);
// Map the V1 response to the V2 response
listResponse = new ListObjectsV2Response
{
CommonPrefixes = listResponsev1.CommonPrefixes,
IsTruncated = listResponsev1.IsTruncated,
NextContinuationToken = listResponsev1.NextMarker,
S3Objects = listResponsev1.S3Objects
};
}
}
catch (AmazonS3Exception e)
{
@@ -282,18 +320,40 @@ namespace Duplicati.Library.Backend
}
isTruncated = listResponse.IsTruncated;
filename = listResponse.NextMarker;
filename = listResponse.NextContinuationToken;
foreach (var obj in listResponse.CommonPrefixes)
{
if (obj == prefix || !obj.StartsWith(prefix))
continue;
// Because the prefixes are returned, and not the folder objects
// we do not get the folder modification date
yield return new FileEntry(
obj.Substring(prefix.Length),
-1,
new DateTime(0),
new DateTime(0)
)
{ IsFolder = true };
}
foreach (var obj in listResponse.S3Objects.Where(obj => alreadyReturned.Add(obj.Key)))
{
yield return new Common.IO.FileEntry(
obj.Key,
// Skip self-prefix, this discards the folder modification date :/
if (obj.Key == prefix || !obj.Key.StartsWith(prefix))
continue;
yield return new FileEntry(
obj.Key.Substring(prefix.Length),
obj.Size,
obj.LastModified,
obj.LastModified,
false,
m_archiveClasses.Contains(obj.StorageClass)
);
obj.LastModified
)
{
IsFolder = obj.Key.EndsWith("/"),
IsArchived = m_archiveClasses.Contains(obj.StorageClass)
};
}
}
}
Duplicati/Library/Backend/S3/S3Backend.cs
+28 -19
View File
@@ -25,6 +25,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
@@ -32,7 +33,7 @@ using System.Threading.Tasks;
namespace Duplicati.Library.Backend
{
public class S3 : IBackend, IStreamingBackend, IRenameEnabledBackend
public class S3 : IBackend, IStreamingBackend, IRenameEnabledBackend, IFolderEnabledBackend
{
private static readonly string LOGTAG = Logging.Log.LogTagFromType<S3>();
@@ -42,6 +43,8 @@ namespace Duplicati.Library.Backend
private const string SSL_OPTION = "use-ssl";
private const string S3_CLIENT_OPTION = "s3-client";
private const string S3_DISABLE_CHUNK_ENCODING_OPTION = "s3-disable-chunk-encoding";
private const string S3_LIST_API_VERSION_OPTION = "s3-list-api-version";
private const string S3_RECURSIVE_LIST = "s3-recursive-list";
public static readonly Dictionary<string, string> KNOWN_S3_PROVIDERS = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) {
{ "Amazon S3", "s3.amazonaws.com" },
@@ -65,7 +68,9 @@ namespace Duplicati.Library.Backend
{ "Infomaniak Swiss Backup cluster 1", "s3.swiss-backup.infomaniak.com" },
{ "Infomaniak Swiss Backup cluster 2", "s3.swiss-backup02.infomaniak.com" },
{ "Infomaniak Swiss Backup cluster 3", "s3.swiss-backup03.infomaniak.com" },
{ "Infomaniak Swiss Backup cluster 4", "s3.swiss-backup04.infomaniak.com" },
{ "Infomaniak Public Cloud 1", "s3.pub1.infomaniak.cloud" },
{ "Infomaniak Public Cloud 2", "s3.pub2.infomaniak.cloud" },
{ "さくらのクラウド (Sakura Cloud)", "s3.isk01.sakurastorage.jp" },
{ "Seagate Lyve - US-East-1", "https://s3.us-east-1.lyvecloud.seagate.com" },
{ "Seagate Lyve - US-West-1", "https://s3.us-west-1.lyvecloud.seagate.com" },
@@ -172,6 +177,7 @@ namespace Duplicati.Library.Backend
private readonly string m_bucket;
private readonly string m_prefix;
private readonly bool m_recurseLists;
private const string DEFAULT_S3_HOST = "s3.amazonaws.com";
private IS3Client s3Client;
@@ -224,6 +230,8 @@ namespace Duplicati.Library.Backend
if (m_prefix.Length != 0)
m_prefix = Util.AppendDirSeparator(m_prefix, "/");
m_recurseLists = Utility.Utility.ParseBoolOption(options, S3_RECURSIVE_LIST);
// Auto-disable DNS lookup for non-AWS configurations
if (!options.ContainsKey("s3-ext-forcepathstyle") && !hostname.EndsWith(".amazonaws.com", StringComparison.OrdinalIgnoreCase))
options["s3-ext-forcepathstyle"] = "true";
@@ -268,20 +276,8 @@ namespace Duplicati.Library.Backend
get { return true; }
}
public IEnumerable<IFileEntry> List()
{
foreach (var file in Connection.ListBucket(m_bucket, m_prefix))
{
file.Name = file.Name.Substring(m_prefix.Length);
//Fix for a bug in Duplicati 1.0 beta 3 and earlier, where filenames are incorrectly prefixed with a slash
if (file.Name.StartsWith("/", StringComparison.Ordinal) && !m_prefix.StartsWith("/", StringComparison.Ordinal))
file.Name = file.Name.Substring(1);
yield return file;
}
}
public IAsyncEnumerable<IFileEntry> ListAsync(CancellationToken cancelToken)
=> Connection.ListBucketAsync(m_bucket, m_prefix, m_recurseLists, cancelToken);
public async Task PutAsync(string remotename, string localname, CancellationToken cancelToken)
{
@@ -334,6 +330,8 @@ namespace Duplicati.Library.Backend
new CommandLineArgument(S3_CLIENT_OPTION, CommandLineArgument.ArgumentType.Enumeration, Strings.S3Backend.S3ClientDescriptionShort, Strings.S3Backend.S3ClientDescriptionLong, "aws", null, new string[] { "aws", "minio" }),
new CommandLineArgument(S3_DISABLE_CHUNK_ENCODING_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionDisableChunkEncodingShort, Strings.S3Backend.DescriptionDisableChunkEncodingLong, "false"),
new CommandLineArgument(S3AwsClient.S3_ARCHIVE_CLASSES_OPTION, CommandLineArgument.ArgumentType.Flags, Strings.S3Backend.S3ArchiveClassesDescriptionShort, Strings.S3Backend.S3ArchiveClassesDescriptionLong, string.Join(",", S3AwsClient.DEFAULT_ARCHIVE_CLASSES.Select(x => x.Value)), null, S3AwsClient.DEFAULT_ARCHIVE_CLASSES.Select(x => x.Value).ToArray()),
new CommandLineArgument(S3_LIST_API_VERSION_OPTION, CommandLineArgument.ArgumentType.Enumeration, Strings.S3Backend.DescriptionListApiVersionShort, Strings.S3Backend.DescriptionListApiVersionLong, "v1", null, ["v1", "v2"]),
new CommandLineArgument(S3_RECURSIVE_LIST, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionRecursiveListShort, Strings.S3Backend.DescriptionRecursiveListLong, "false"),
new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, Strings.S3Backend.AuthPasswordDescriptionShort, Strings.S3Backend.AuthPasswordDescriptionLong),
new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, Strings.S3Backend.AuthUsernameDescriptionShort, Strings.S3Backend.AuthUsernameDescriptionLong),
};
@@ -352,10 +350,7 @@ namespace Duplicati.Library.Backend
}
public Task TestAsync(CancellationToken cancelToken)
{
this.TestList();
return Task.CompletedTask;
}
=> this.TestListAsync(cancelToken);
public Task CreateFolderAsync(CancellationToken cancelToken)
{
@@ -396,5 +391,19 @@ namespace Duplicati.Library.Backend
//AWS SDK encodes the filenames correctly
return m_prefix + name;
}
/// <inheritdoc/>
public IAsyncEnumerable<IFileEntry> ListAsync(string path, CancellationToken cancellationToken)
{
var filterPath = GetFullKey(path);
if (!string.IsNullOrWhiteSpace(filterPath))
filterPath = Util.AppendDirSeparator(filterPath, "/");
return s3Client.ListBucketAsync(m_bucket, filterPath, m_recurseLists, cancellationToken);
}
/// <inheritdoc/>
public Task<IFileEntry> GetEntryAsync(string path, CancellationToken cancellationToken)
=> Task.FromResult<IFileEntry>(null);
}
}
Duplicati/Library/Backend/S3/S3MinioClient.cs
+35 -7
View File
@@ -22,7 +22,9 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Reactive.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Channels;
using System.Threading.Tasks;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
@@ -59,21 +61,47 @@ namespace Duplicati.Library.Backend
m_dnsHost = servername;
}
public IEnumerable<FileEntry> ListBucket(string bucketName, string prefix)
private static async IAsyncEnumerable<T> ToAsyncEnumerable<T>(
IObservable<T> observable,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
var channel = Channel.CreateUnbounded<T>(); // Buffered channel for async iteration
using var subscription = observable.Subscribe(
item => channel.Writer.TryWrite(item),
ex => channel.Writer.TryComplete(ex),
() => channel.Writer.TryComplete()
);
while (await channel.Reader.WaitToReadAsync(cancellationToken))
{
while (channel.Reader.TryRead(out var item))
{
yield return item;
}
}
}
public async IAsyncEnumerable<IFileEntry> ListBucketAsync(string bucketName, string prefix, bool recursive, [EnumeratorCancellation] CancellationToken cancellationToken)
{
ThrowExceptionIfBucketDoesNotExist(bucketName);
var observable = m_client.ListObjectsAsync(bucketName, prefix, true);
if (!string.IsNullOrWhiteSpace(prefix))
prefix = Util.AppendDirSeparator(prefix, "/");
foreach (var obj in observable.ToEnumerable())
var observable = m_client.ListObjectsAsync(bucketName, prefix, recursive, cancellationToken);
await foreach (var obj in ToAsyncEnumerable(observable, cancellationToken).ConfigureAwait(false))
{
if (obj.Key == prefix || !obj.Key.StartsWith(prefix))
continue;
yield return new FileEntry(
obj.Key,
obj.Key.Substring(prefix.Length),
(long)obj.Size,
Convert.ToDateTime(obj.LastModified),
Convert.ToDateTime(obj.LastModified),
obj.IsDir
);
Convert.ToDateTime(obj.LastModified)
)
{ IsFolder = obj.Key.EndsWith("/") };
}
}
Duplicati/Library/Backend/S3/Strings.cs
+34 -30
View File
@@ -21,36 +21,40 @@
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.Strings
{
internal static class S3Backend
{
public static string Description_v2 { get { return LC.L(@"This backend can read and write data to an S3 compatible server. Allowed format is ""s3://bucketname/prefix""."); } }
public static string DisplayName { get { return LC.L(@"S3 compatible"); } }
public static string AMZKeyDescriptionLong { get { return LC.L(@"AWS Secret Access Key can be obtained after logging into your AWS account. This can also be supplied through the option --{0}.", "auth-password"); } }
public static string AMZKeyDescriptionShort { get { return LC.L(@"AWS Secret Access Key"); } }
public static string AMZUserIDDescriptionLong { get { return LC.L(@"AWS Access Key ID can be obtained after logging into your AWS account. This can also be supplied through the option --{0}.", "auth-username"); } }
public static string AMZUserIDDescriptionShort { get { return LC.L(@"AWS Access Key ID"); } }
public static string AuthPasswordDescriptionLong { get { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); } }
public static string AuthPasswordDescriptionShort { get { return LC.L(@"Supply the password used to connect to the server"); } }
public static string AuthUsernameDescriptionLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } }
public static string AuthUsernameDescriptionShort { get { return LC.L(@"Supply the username used to connect to the server"); } }
public static string NoAMZKeyError { get { return LC.L(@"No S3 secret key given"); } }
public static string NoAMZUserIDError { get { return LC.L(@"No S3 userID given"); } }
public static string S3LocationDescriptionLong(string regions) { return LC.L(@"This option is only used when creating new buckets. Use this option to change what region the data is stored in. Amazon charges slightly more for non-US buckets. Known bucket locations:
internal static class S3Backend
{
public static string Description_v2 { get { return LC.L(@"This backend can read and write data to an S3 compatible server. Allowed format is ""s3://bucketname/prefix""."); } }
public static string DisplayName { get { return LC.L(@"S3 compatible"); } }
public static string AMZKeyDescriptionLong { get { return LC.L(@"AWS Secret Access Key can be obtained after logging into your AWS account. This can also be supplied through the option --{0}.", "auth-password"); } }
public static string AMZKeyDescriptionShort { get { return LC.L(@"AWS Secret Access Key"); } }
public static string AMZUserIDDescriptionLong { get { return LC.L(@"AWS Access Key ID can be obtained after logging into your AWS account. This can also be supplied through the option --{0}.", "auth-username"); } }
public static string AMZUserIDDescriptionShort { get { return LC.L(@"AWS Access Key ID"); } }
public static string AuthPasswordDescriptionLong { get { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); } }
public static string AuthPasswordDescriptionShort { get { return LC.L(@"Supply the password used to connect to the server"); } }
public static string AuthUsernameDescriptionLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } }
public static string AuthUsernameDescriptionShort { get { return LC.L(@"Supply the username used to connect to the server"); } }
public static string NoAMZKeyError { get { return LC.L(@"No S3 secret key given"); } }
public static string NoAMZUserIDError { get { return LC.L(@"No S3 userID given"); } }
public static string S3LocationDescriptionLong(string regions) { return LC.L(@"This option is only used when creating new buckets. Use this option to change what region the data is stored in. Amazon charges slightly more for non-US buckets. Known bucket locations:
{0}", regions); }
public static string S3LocationDescriptionShort { get { return LC.L(@"Specify S3 location constraints"); } }
public static string S3ServerNameDescriptionLong(string providers) { return LC.L(@"Companies other than Amazon are now supporting the S3 API, meaning that this backend can read and write data to those providers as well. Use this option to set the hostname. Currently known providers are:
public static string S3LocationDescriptionShort { get { return LC.L(@"Specify S3 location constraints"); } }
public static string S3ServerNameDescriptionLong(string providers) { return LC.L(@"Companies other than Amazon are now supporting the S3 API, meaning that this backend can read and write data to those providers as well. Use this option to set the hostname. Currently known providers are:
{0}", providers); }
public static string S3ServerNameDescriptionShort { get { return LC.L(@"Specify an alternate S3 server name"); } }
public static string S3ClientDescriptionLong { get { return LC.L(@"Set either to aws or minio. Then either the AWS SDK or Minio SDK will be used to communicate with S3 services."); } }
public static string S3ClientDescriptionShort { get { return LC.L(@"Specify the S3 client library to use"); } }
public static string DescriptionUseSSLLong { get { return LC.L(@"Use this option to communicate using Secure Socket Layer (SSL) over http (https). Note that bucket names containing a period has problems with SSL connections."); } }
public static string DescriptionUseSSLShort { get { return LC.L(@"Instruct Duplicati to use an SSL (https) connection"); } }
public static string DescriptionDisableChunkEncodingLong { get { return LC.L(@"This disables chunk encoding for the aws client, which is not supported by all S3 providers."); } }
public static string DescriptionDisableChunkEncodingShort { get { return LC.L(@"Disable chunk encoding (aws client only)"); } }
public static string S3StorageclassDescriptionLong { get { return LC.L(@"Use this option to specify a storage class. If this option is not used, the server will choose a default storage class."); } }
public static string S3StorageclassDescriptionShort { get { return LC.L(@"Specify storage class"); } }
public static string S3ArchiveClassesDescriptionShort { get { return LC.L(@"Specify archive storage class"); } }
public static string S3ArchiveClassesDescriptionLong { get { return LC.L(@"Use this option to specify what storage classes are considered archive storage classes. With this option it is possible to allow lifecycle policies to move data to cheaper storage classes and prevent Duplicati from accessing archived data. This option is only supported for the AWS client."); } }
public static string UnknownS3ClientError(string client) { return LC.L(@"Unknown S3 client: {0}", client); }
}
public static string S3ServerNameDescriptionShort { get { return LC.L(@"Specify an alternate S3 server name"); } }
public static string S3ClientDescriptionLong { get { return LC.L(@"Set either to aws or minio. Then either the AWS SDK or Minio SDK will be used to communicate with S3 services."); } }
public static string S3ClientDescriptionShort { get { return LC.L(@"Specify the S3 client library to use"); } }
public static string DescriptionUseSSLLong { get { return LC.L(@"Use this option to communicate using Secure Socket Layer (SSL) over http (https). Note that bucket names containing a period has problems with SSL connections."); } }
public static string DescriptionUseSSLShort { get { return LC.L(@"Instruct Duplicati to use an SSL (https) connection"); } }
public static string DescriptionDisableChunkEncodingLong { get { return LC.L(@"This disables chunk encoding for the aws client, which is not supported by all S3 providers."); } }
public static string DescriptionDisableChunkEncodingShort { get { return LC.L(@"Disable chunk encoding (aws client only)"); } }
public static string S3StorageclassDescriptionLong { get { return LC.L(@"Use this option to specify a storage class. If this option is not used, the server will choose a default storage class."); } }
public static string S3StorageclassDescriptionShort { get { return LC.L(@"Specify storage class"); } }
public static string S3ArchiveClassesDescriptionShort { get { return LC.L(@"Specify archive storage class"); } }
public static string S3ArchiveClassesDescriptionLong { get { return LC.L(@"Use this option to specify what storage classes are considered archive storage classes. With this option it is possible to allow lifecycle policies to move data to cheaper storage classes and prevent Duplicati from accessing archived data. This option is only supported for the AWS client."); } }
public static string DescriptionListApiVersionShort { get { return LC.L(@"Specify the S3 list API version to use"); } }
public static string DescriptionListApiVersionLong { get { return LC.L(@"Use this option to specify the S3 list API version to use. This can be used to work around issues with some S3 providers."); } }
public static string DescriptionRecursiveListShort { get { return LC.L(@"Use this option to list all files in the bucket"); } }
public static string DescriptionRecursiveListLong { get { return LC.L(@"To reduce the number of objects listed, the default is to only list the first level of objects. Use this option to list all objects in the bucket."); } }
public static string UnknownS3ClientError(string client) { return LC.L(@"Unknown S3 client: {0}", client); }
}
}
Duplicati/Library/Backend/SSHv2/Duplicati.Library.Backend.SSHv2.csproj
+1
View File
@@ -12,6 +12,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../../SourceProvider/Builtin/Duplicati.Library.SourceProvider.Builtin.csproj" />
<ProjectReference Include="..\..\Interface\Duplicati.Library.Interface.csproj" />
<ProjectReference Include="..\..\Localization\Duplicati.Library.Localization.csproj" />
<ProjectReference Include="..\..\Utility\Duplicati.Library.Utility.csproj" />
Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs
+113 -63
View File
@@ -1,43 +1,46 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.SourceProvider;
using Renci.SshNet;
using Renci.SshNet.Common;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Backend
{
public class SSHv2 : IStreamingBackend, IRenameEnabledBackend
public class SSHv2 : IStreamingBackend, IRenameEnabledBackend, IFolderEnabledBackend
{
private const string SSH_KEYFILE_OPTION = "ssh-keyfile";
private const string SSH_KEYFILE_INLINE = "ssh-key";
private const string SSH_FINGERPRINT_OPTION = "ssh-fingerprint";
private const string SSH_RELATIVE_PATH_OPTION = "ssh-relative-path";
private const string SSH_FINGERPRINT_ACCEPT_ANY_OPTION = "ssh-accept-any-fingerprints";
public const string KEYFILE_URI = "sshkey://";
private const string SSH_TIMEOUT_OPTION = "ssh-operation-timeout";
@@ -55,6 +58,8 @@ namespace Duplicati.Library.Backend
private readonly TimeSpan m_keepaliveinterval;
private readonly int m_port = 22;
private readonly bool m_useRelativePath;
private string m_initialDirectory;
private SftpClient m_con;
@@ -81,6 +86,7 @@ namespace Duplicati.Library.Backend
m_password = uri.Password;
m_fingerprintallowall = Utility.Utility.ParseBoolOption(options, SSH_FINGERPRINT_ACCEPT_ANY_OPTION);
m_useRelativePath = Utility.Utility.ParseBoolOption(options, SSH_RELATIVE_PATH_OPTION);
m_path = uri.Path;
@@ -89,8 +95,15 @@ namespace Duplicati.Library.Backend
m_path = Util.AppendDirSeparator(m_path, "/");
}
if (!m_path.StartsWith("/", StringComparison.Ordinal))
m_path = "/" + m_path;
if (m_useRelativePath)
{
m_path = m_path.TrimStart('/');
}
else
{
if (!m_path.StartsWith("/", StringComparison.Ordinal))
m_path = "/" + m_path;
}
m_server = uri.Host;
@@ -112,22 +125,19 @@ namespace Duplicati.Library.Backend
#region IBackend Members
public Task TestAsync(CancellationToken cancelToken)
{
this.TestList();
return Task.CompletedTask;
}
=> this.TestListAsync(cancelToken);
public Task CreateFolderAsync(CancellationToken cancelToken)
public async Task CreateFolderAsync(CancellationToken cancelToken)
{
CreateConnection();
await CreateConnection(cancelToken).ConfigureAwait(false);
// Since the SftpClient.CreateDirectory method does not create all the parent directories
// as needed, this has to be done manually.
string partialPath = String.Empty;
var partialPath = string.Empty;
foreach (string part in m_path.Split('/').Where(x => !String.IsNullOrEmpty(x)))
{
partialPath += $"/{part}";
if (this.m_con.Exists(partialPath))
if (await this.m_con.ExistsAsync(partialPath, cancelToken))
{
if (!this.m_con.GetAttributes(partialPath).IsDirectory)
{
@@ -136,11 +146,10 @@ namespace Duplicati.Library.Backend
}
else
{
this.m_con.CreateDirectory(partialPath);
await this.m_con.CreateDirectoryAsync(partialPath, cancelToken).ConfigureAwait(false);
}
}
return Task.CompletedTask;
}
public string DisplayName => Strings.SSHv2Backend.DisplayName;
@@ -165,8 +174,8 @@ namespace Duplicati.Library.Backend
{
try
{
CreateConnection();
ChangeDirectory(m_path);
await CreateConnection(cancelToken).ConfigureAwait(false);
await SetWorkingDirectory(cancelToken).ConfigureAwait(false);
await m_con.DeleteFileAsync(remotename, cancelToken).ConfigureAwait(false);
}
catch (SftpPathNotFoundException ex)
@@ -205,6 +214,9 @@ namespace Duplicati.Library.Backend
new CommandLineArgument(SSH_KEEPALIVE_OPTION, CommandLineArgument.ArgumentType.Timespan,
Strings.SSHv2Backend.DescriptionSshkeepaliveShort,
Strings.SSHv2Backend.DescriptionSshkeepaliveLong, "0"),
new CommandLineArgument(SSH_RELATIVE_PATH_OPTION, CommandLineArgument.ArgumentType.Boolean,
Strings.SSHv2Backend.DescriptionRelativePathShort,
Strings.SSHv2Backend.DescriptionRelativePathLong)
});
}
}
@@ -239,11 +251,11 @@ namespace Duplicati.Library.Backend
#region IStreamingBackend Implementation
public Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
{
CreateConnection();
ChangeDirectory(m_path);
return Task.Factory.FromAsync(
await CreateConnection(cancelToken).ConfigureAwait(false);
await SetWorkingDirectory(cancelToken).ConfigureAwait(false);
await Task.Factory.FromAsync(
(cb, state) => m_con.BeginUploadFile(stream, remotename, cb, state, _ => cancelToken.ThrowIfCancellationRequested()),
m_con.EndUploadFile,
null);
@@ -251,8 +263,8 @@ namespace Duplicati.Library.Backend
public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
{
CreateConnection();
ChangeDirectory(m_path);
await CreateConnection(cancelToken).ConfigureAwait(false);
await SetWorkingDirectory(cancelToken).ConfigureAwait(false);
try
{
@@ -271,25 +283,25 @@ namespace Duplicati.Library.Backend
#region IRenameEnabledBackend Implementation
public Task RenameAsync(string source, string target, CancellationToken cancelToken)
public async Task RenameAsync(string source, string target, CancellationToken cancelToken)
{
CreateConnection();
ChangeDirectory(m_path);
return m_con.RenameFileAsync(source, target, cancelToken);
await CreateConnection(cancelToken).ConfigureAwait(false);
await SetWorkingDirectory(cancelToken).ConfigureAwait(false);
await m_con.RenameFileAsync(source, target, cancelToken).ConfigureAwait(false);
}
#endregion
#region Implementation
private void CreateConnection()
private async Task CreateConnection(CancellationToken cancelToken)
{
if (m_con != null && m_con.IsConnected)
return;
if (m_con != null && !m_con.IsConnected)
{
this.TryConnect(m_con);
await this.TryConnect(m_con, cancelToken).ConfigureAwait(false);
return;
}
@@ -344,44 +356,50 @@ namespace Duplicati.Library.Backend
if (m_keepaliveinterval.Ticks != 0)
con.KeepAliveInterval = m_keepaliveinterval;
this.TryConnect(con);
await this.TryConnect(con, cancelToken).ConfigureAwait(false);
if (m_useRelativePath && (string.IsNullOrWhiteSpace(con.WorkingDirectory) || !con.WorkingDirectory.StartsWith("/")))
throw new UserInformationException("Server does not report absolute initial directory, please switch to absolute paths", "RelativePathNotSupported");
m_initialDirectory = con.WorkingDirectory;
m_con = con;
}
private void TryConnect(SftpClient client)
{
client.Connect();
}
private Task TryConnect(SftpClient client, CancellationToken cancelToken)
=> client.ConnectAsync(cancelToken);
private void ChangeDirectory(string path)
private async Task SetWorkingDirectory(CancellationToken cancelToken)
{
if (string.IsNullOrEmpty(path))
if (string.IsNullOrEmpty(m_path))
return;
var targetPath = m_useRelativePath
? Util.AppendDirSeparator(m_initialDirectory, "/") + m_path
: m_path;
var workingDir = Util.AppendDirSeparator(m_con.WorkingDirectory, "/");
if (workingDir == path)
if (workingDir == targetPath)
return;
try
{
m_con.ChangeDirectory(path);
await m_con.ChangeDirectoryAsync(targetPath, cancelToken).ConfigureAwait(false);
}
catch (Exception ex)
{
throw new Interface.FolderMissingException(
Strings.SSHv2Backend.FolderNotFoundManagedError(path, ex.Message), ex);
Strings.SSHv2Backend.FolderNotFoundManagedError(m_path, ex.Message), ex);
}
}
public IEnumerable<IFileEntry> List()
/// <inheritdoc />
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
string path = ".";
var path = ".";
CreateConnection();
ChangeDirectory(m_path);
await CreateConnection(cancelToken).ConfigureAwait(false);
await SetWorkingDirectory(cancelToken).ConfigureAwait(false);
foreach (var ls in m_con.ListDirectory(path))
await foreach (var ls in m_con.ListDirectoryAsync(path, cancelToken).ConfigureAwait(false))
{
if (ls.Name.ToString() == "." || ls.Name.ToString() == "..") continue;
yield return new FileEntry(ls.Name, ls.Length,
@@ -396,7 +414,7 @@ namespace Duplicati.Library.Backend
{
if (!filename.StartsWith(KEYFILE_URI, StringComparison.OrdinalIgnoreCase))
{
return String.IsNullOrEmpty(password)
return string.IsNullOrEmpty(password)
? new PrivateKeyFile(filename)
: new PrivateKeyFile(filename, password);
}
@@ -409,7 +427,7 @@ namespace Duplicati.Library.Backend
ms.Position = 0;
return String.IsNullOrEmpty(password) ? new PrivateKeyFile(ms) : new PrivateKeyFile(ms, password);
return string.IsNullOrEmpty(password) ? new PrivateKeyFile(ms) : new PrivateKeyFile(ms, password);
}
}
catch (Exception ex)
@@ -429,5 +447,37 @@ namespace Duplicati.Library.Backend
}
public Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { m_server });
/// <inheritdoc/>
public async IAsyncEnumerable<IFileEntry> ListAsync(string path, [EnumeratorCancellation] CancellationToken cancellationToken)
{
await CreateConnection(cancellationToken).ConfigureAwait(false);
await SetWorkingDirectory(cancellationToken).ConfigureAwait(false);
var prefixPath = m_useRelativePath ? "" : m_path;
if (!string.IsNullOrWhiteSpace(prefixPath))
prefixPath = Util.AppendDirSeparator(prefixPath, "/");
var filterPath = prefixPath + BackendSourceFileEntry.NormalizePathTo(path, '/');
if (!string.IsNullOrWhiteSpace(filterPath))
filterPath = Util.AppendDirSeparator(filterPath, "/");
if (string.IsNullOrWhiteSpace(filterPath))
filterPath = ".";
await foreach (var x in Client.ListDirectoryAsync(filterPath, cancellationToken).ConfigureAwait(false))
yield return new FileEntry(
x.Name,
x.Attributes.Size,
x.Attributes.LastAccessTimeUtc,
x.Attributes.LastWriteTimeUtc)
{
IsFolder = x.Attributes.IsDirectory
};
}
/// <inheritdoc/>
public Task<IFileEntry> GetEntryAsync(string path, CancellationToken cancellationToken)
=> Task.FromResult<IFileEntry>(null);
}
}
Duplicati/Library/Backend/SSHv2/Strings.cs
+21 -19
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.Strings
@@ -61,6 +61,8 @@ namespace Duplicati.Library.Backend.Strings
public static string DescriptionSshtimeoutShort { get { return LC.L(@"Set the operation timeout value"); } }
public static string DescriptionSshkeepaliveLong { get { return LC.L(@"Use this option to enable the keep-alive interval for the SSH connection. If the connection is idle, aggressive firewalls might close the connection. Using keep-alive will keep the connection open in this scenario. If this value is set to zero, the keep-alive is disabled."); } }
public static string DescriptionSshkeepaliveShort { get { return LC.L(@"Set a keepalive value"); } }
public static string DescriptionRelativePathShort { get { return LC.L(@"Treat source path as relative to the initial path"); } }
public static string DescriptionRelativePathLong { get { return LC.L(@"Use this option to treat the source path as relative to the initial path. This is useful when the full path of the system is not known."); } }
public static string FolderNotFoundManagedError(string foldername, string message) { return LC.L(@"Unable to set folder to {0}, error message: {1}", foldername, message); }
public static string FingerprintNotMatchManagedError(string fingerprint) { return LC.L(@"Validation of server fingerprint failed. Server returned fingerprint ""{0}"". Cause of this message is either not correct configuration or Man-in-the-middle attack!", fingerprint); }
public static string FingerprintNotSpecifiedManagedError(string fingerprint, string hostkeyoption, string allkeysoptions) { return LC.L(@"Please add --{1}=""{0}"" to trust this host. Optionally you can use --{2} (NOT SECURE) for testing!", fingerprint, hostkeyoption, allkeysoptions); }
Duplicati/Library/Backend/SharePoint/SharePointBackend.cs
+14 -6
View File
@@ -28,6 +28,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
@@ -473,8 +474,11 @@ namespace Duplicati.Library.Backend
await testContextForWebAsync(ctx, true, cancelToken).ConfigureAwait(false);
}
public IEnumerable<IFileEntry> List() { return doListAsync(false, CancellationToken.None).Await(); }
private async Task<IEnumerable<IFileEntry>> doListAsync(bool useNewContext, CancellationToken cancelToken)
/// <inheritdoc />
public IAsyncEnumerable<IFileEntry> ListAsync(CancellationToken cancelToken)
=> doListAsync(false, cancelToken);
private async IAsyncEnumerable<IFileEntry> doListAsync(bool useNewContext, [EnumeratorCancellation] CancellationToken cancelToken)
{
var ctx = await getSpClientContextAsync(useNewContext, cancelToken).ConfigureAwait(false);
SP.Folder remoteFolder = null;
@@ -502,15 +506,19 @@ namespace Duplicati.Library.Backend
if (retry)
{
// An exception was caught, and List() should be retried.
return await doListAsync(true, cancelToken).ConfigureAwait(false);
await foreach (var f in doListAsync(true, cancelToken).ConfigureAwait(false))
yield return f;
}
else
{
return remoteFolder.Folders.Where(ff => ff.Exists)
var list = remoteFolder.Folders.Where(ff => ff.Exists)
.Select(f => new FileEntry(f.Name, -1, f.TimeLastModified, f.TimeLastModified, true, false))
.Concat(remoteFolder.Files.Where(ff => ff.Exists)
.Select(f => new FileEntry(f.Name, f.Length, f.TimeLastModified, f.TimeLastModified, false, false)))
.ToArray();
.Select(f => new FileEntry(f.Name, f.Length, f.TimeLastModified, f.TimeLastModified, false, false)));
foreach (var f in list)
yield return f;
}
}
Duplicati/Library/Backend/Sia/Sia.cs
+23 -25
View File
@@ -26,6 +26,7 @@ using System;
using System.Collections.Generic;
using System.Globalization;
using System.Net;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
@@ -161,19 +162,22 @@ namespace Duplicati.Library.Backend.Sia
public SiaDownloadFile[] Files { get; set; }
}
private SiaFileList GetFiles()
private async Task<SiaFileList> GetFiles(CancellationToken cancelToken)
{
// Remove warning until this is rewritten to use HttpClient
await Task.CompletedTask;
var fl = new SiaFileList();
string endpoint = "/renter/files";
var endpoint = "/renter/files";
try
{
System.Net.HttpWebRequest req = CreateRequest(endpoint);
var req = CreateRequest(endpoint);
req.Method = System.Net.WebRequestMethods.Http.Get;
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
var areq = new Utility.AsyncHttpRequest(req);
using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse())
using (var resp = (System.Net.HttpWebResponse)areq.GetResponse())
{
int code = (int)resp.StatusCode;
if (code < 200 || code >= 300)
@@ -196,9 +200,9 @@ namespace Duplicati.Library.Backend.Sia
return fl;
}
private bool IsUploadComplete(string siafilename)
private async Task<bool> IsUploadComplete(string siafilename, CancellationToken cancelToken)
{
SiaFileList fl = GetFiles();
var fl = await GetFiles(cancelToken).ConfigureAwait(false);
if (fl.Files == null)
return false;
@@ -286,10 +290,7 @@ namespace Duplicati.Library.Backend.Sia
#region IBackend Members
public Task TestAsync(CancellationToken cancelToken)
{
this.TestList();
return Task.CompletedTask;
}
=> this.TestListAsync(cancelToken);
public Task CreateFolderAsync(CancellationToken cancelToken)
{
@@ -307,12 +308,13 @@ namespace Duplicati.Library.Backend.Sia
get { return "sia"; }
}
public IEnumerable<IFileEntry> List()
/// <inheritdoc />
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
SiaFileList fl;
try
{
fl = GetFiles();
fl = await GetFiles(cancelToken).ConfigureAwait(false);
}
catch (System.Net.WebException wex)
{
@@ -338,10 +340,10 @@ namespace Duplicati.Library.Backend.Sia
}
}
public Task PutAsync(string remotename, string filename, CancellationToken cancelToken)
public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken)
{
string endpoint = "";
string siafile = m_targetpath + "/" + remotename;
var endpoint = "";
var siafile = m_targetpath + "/" + remotename;
try
{
@@ -351,29 +353,25 @@ namespace Duplicati.Library.Backend.Sia
Utility.Uri.UrlEncode(filename).Replace("+", "%20")
);
HttpWebRequest req = CreateRequest(endpoint);
var req = CreateRequest(endpoint);
req.Method = WebRequestMethods.Http.Post;
AsyncHttpRequest areq = new AsyncHttpRequest(req);
var areq = new AsyncHttpRequest(req);
using (HttpWebResponse resp = (HttpWebResponse)areq.GetResponse())
using (var resp = (HttpWebResponse)areq.GetResponse())
{
int code = (int)resp.StatusCode;
if (code < 200 || code >= 300)
throw new WebException(resp.StatusDescription, null, WebExceptionStatus.ProtocolError, resp);
while (!IsUploadComplete(siafile))
{
Thread.Sleep(5000);
}
while (!await IsUploadComplete(siafile, cancelToken).ConfigureAwait(false))
await Task.Delay(5000, cancelToken).ConfigureAwait(false);
}
}
catch (WebException wex)
{
throw new Exception(getResponseBodyOnError(endpoint, wex));
}
return Task.FromResult(true);
}
public async Task GetAsync(string remotename, string localname, CancellationToken cancelToken)
Duplicati/Library/Backend/Storj/StorjBackend.cs
+5 -12
View File
@@ -19,10 +19,10 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using System;
using System.Collections.Generic;
using System.IO;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using uplink.NET.Interfaces;
@@ -238,29 +238,22 @@ namespace Duplicati.Library.Backend.Storj
await download.StartDownloadAsync().ConfigureAwait(false);
}
public IEnumerable<IFileEntry> List()
/// <inheritdoc />
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
return ListAsync().Await();
}
private async Task<IEnumerable<IFileEntry>> ListAsync()
{
List<StorjFile> files = new List<StorjFile>();
var bucket = await _bucketService.EnsureBucketAsync(_bucket).ConfigureAwait(false);
var prefix = GetBasePath();
var objects = await _objectService.ListObjectsAsync(bucket, new ListObjectsOptions { Recursive = true, System = true, Custom = true, Prefix = prefix }).ConfigureAwait(false);
foreach (var obj in objects.Items)
{
StorjFile file = new StorjFile(obj);
var file = new StorjFile(obj);
if (prefix != "")
{
file.Name = file.Name.Replace(prefix, "");
}
files.Add(file);
yield return file;
}
return files;
}
public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken)
Duplicati/Library/Backend/Storj/StorjFile.cs
+1
View File
@@ -33,6 +33,7 @@ namespace Duplicati.Library.Backend.Storj
public DateTime LastAccess { get; set; }
public DateTime LastModification { get; set; }
public DateTime Created { get; set; }
public string Name { get; set; }
Duplicati/Library/Backend/TahoeLAFS/TahoeBackend.cs
+28 -26
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Common.IO;
@@ -26,6 +26,7 @@ using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Net;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
@@ -124,10 +125,7 @@ namespace Duplicati.Library.Backend
#region IBackend Members
public Task TestAsync(CancellationToken cancelToken)
{
this.TestList();
return Task.CompletedTask;
}
=> this.TestListAsync(cancelToken);
public Task CreateFolderAsync(CancellationToken cancelToken)
{
@@ -150,8 +148,12 @@ namespace Duplicati.Library.Backend
get { return "tahoe"; }
}
public IEnumerable<IFileEntry> List()
/// <inheritdoc />
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
// Remove warning until this is rewritten to use HttpClient
await Task.CompletedTask;
TahoeEl data;
try
@@ -160,9 +162,9 @@ namespace Duplicati.Library.Backend
req.Method = System.Net.WebRequestMethods.Http.Get;
var areq = new Utility.AsyncHttpRequest(req);
using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse())
using (var resp = (System.Net.HttpWebResponse)areq.GetResponse())
{
int code = (int)resp.StatusCode;
var code = (int)resp.StatusCode;
if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp);
Duplicati/Library/Backend/TencentCOS/COSBackend.cs
+11 -12
View File
@@ -26,11 +26,11 @@ using COSXML.Model.Tag;
using COSXML.Utils;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
@@ -178,7 +178,8 @@ namespace Duplicati.Library.Backend.TencentCOS
return new CosXmlServer(config, cosCredentialProvider);
}
public IEnumerable<IFileEntry> List()
/// <inheritdoc />
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
bool isTruncated = true;
string filename = null;
@@ -186,26 +187,24 @@ namespace Duplicati.Library.Backend.TencentCOS
while (isTruncated)
{
cosXml = GetCosXml();
string bucket = _cosOptions.Bucket;
string prefix = _cosOptions.Path;
GetBucketRequest request = new GetBucketRequest(bucket);
var bucket = _cosOptions.Bucket;
var prefix = _cosOptions.Path;
var request = new GetBucketRequest(bucket);
request.SetSign(TimeUtils.GetCurrentTime(TimeUnit.SECONDS), 600);
if (!string.IsNullOrEmpty(filename))
{
request.SetMarker(filename);
}
if (!string.IsNullOrEmpty(prefix))
{
request.SetPrefix(prefix);
}
GetBucketResult result = cosXml.GetBucket(request);
var tcs = new TaskCompletionSource<GetBucketResult>();
cancelToken.Register(() => tcs.TrySetCanceled());
cosXml.GetBucket(request, (result) => tcs.SetResult(result as GetBucketResult), (clientEx, serverEx) => tcs.SetException((Exception)clientEx ?? serverEx));
var result = await tcs.Task.ConfigureAwait(false);
ListBucket info = result.listBucket;
var info = result.listBucket;
isTruncated = result.listBucket.isTruncated;
filename = result.listBucket.nextMarker;
Duplicati/Library/Backend/WEBDAV/WEBDAV.cs
+32 -39
View File
@@ -27,6 +27,7 @@ using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
@@ -75,7 +76,7 @@ namespace Duplicati.Library.Backend
/// It is used to detect a problem with IIS where a file is listed,
/// but IIS responds 404 because the file mapping is incorrect.
/// </summary>
private List<string> m_filenamelist = null;
private HashSet<string> m_filenamelist = null;
// According to the WEBDAV standard, the "allprop" request should return all properties, however this seems to fail on some servers (box.net).
// I've found this description: http://www.webdav.org/specs/rfc2518.html#METHOD_PROPFIND
@@ -173,11 +174,24 @@ namespace Duplicati.Library.Backend
public string ProtocolKey => "webdav";
///<inheritdoc/>
public IEnumerable<IFileEntry> List()
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancelToken)
{
using var timeoutToken = new CancellationTokenSource();
timeoutToken.CancelAfter(TimeSpan.FromSeconds(LIST_OPERATION_TIMEOUT_SECONDS));
using var requestResources = CreateRequest(string.Empty, new HttpMethod("PROPFIND"));
requestResources.RequestMessage.Headers.Add("Depth", "1");
requestResources.RequestMessage.Content = new StreamContent(new MemoryStream(PROPFIND_BODY));
requestResources.RequestMessage.Content.Headers.ContentLength = PROPFIND_BODY.Length;
var doc = new System.Xml.XmlDocument();
try
{
return ListWithoutExceptionCatch();
using var response = await requestResources.HttpClient.SendAsync(requestResources.RequestMessage, HttpCompletionOption.ResponseContentRead, timeoutToken.Token).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
using var stream = await response.Content.ReadAsStreamAsync(cancelToken).ConfigureAwait(false);
doc.Load(stream);
}
catch (HttpRequestException wex)
{
@@ -189,32 +203,11 @@ namespace Duplicati.Library.Backend
throw;
}
}
private IEnumerable<IFileEntry> ListWithoutExceptionCatch()
{
using var timeoutToken = new CancellationTokenSource();
timeoutToken.CancelAfter(TimeSpan.FromSeconds(LIST_OPERATION_TIMEOUT_SECONDS));
using var requestResources = CreateRequest(string.Empty, new HttpMethod("PROPFIND"));
requestResources.RequestMessage.Headers.Add("Depth", "1");
requestResources.RequestMessage.Content = new StreamContent(new MemoryStream(PROPFIND_BODY));
requestResources.RequestMessage.Content.Headers.ContentLength = PROPFIND_BODY.Length;
using var response = requestResources.HttpClient.SendAsync(requestResources.RequestMessage, HttpCompletionOption.ResponseContentRead, timeoutToken.Token).ConfigureAwait(false).GetAwaiter().GetResult();
response.EnsureSuccessStatusCode(); // This replaces the if needed when Mono was used.
var doc = new System.Xml.XmlDocument();
doc.Load(response.Content.ReadAsStream());
System.Xml.XmlNamespaceManager nm = new System.Xml.XmlNamespaceManager(doc.NameTable);
var nm = new System.Xml.XmlNamespaceManager(doc.NameTable);
nm.AddNamespace("D", "DAV:");
List<IFileEntry> files = new List<IFileEntry>();
m_filenamelist = new List<string>();
var filenamelist = new HashSet<string>();
foreach (System.Xml.XmlNode n in doc.SelectNodes("D:multistatus/D:response/D:href", nm))
{
//IIS uses %20 for spaces and %2B for +
@@ -274,13 +267,16 @@ namespace Duplicati.Library.Backend
isCollection = stat.SelectSingleNode("D:resourcetype/D:collection", nm) != null;
}
FileEntry fe = new FileEntry(name, size, lastAccess, lastModified);
fe.IsFolder = isCollection;
files.Add(fe);
m_filenamelist.Add(name);
var fe = new FileEntry(name, size, lastAccess, lastModified)
{
IsFolder = isCollection
};
filenamelist.Add(name);
yield return fe;
}
return files;
m_filenamelist = filenamelist;
}
///<inheritdoc/>
@@ -311,7 +307,7 @@ namespace Duplicati.Library.Backend
var response = await requestResources.HttpClient.SendAsync(requestResources.RequestMessage, combinedTokens.Token).ConfigureAwait(false);
response.EnsureSuccessStatusCode(); // This replaces the if needed when Mono was used.
response.EnsureSuccessStatusCode();
}
catch (HttpRequestException wex)
@@ -347,10 +343,7 @@ namespace Duplicati.Library.Backend
///<inheritdoc/>
public Task TestAsync(CancellationToken cancelToken)
{
List();
return Task.CompletedTask;
}
=> this.TestListAsync(cancelToken);
///<inheritdoc/>
public async Task CreateFolderAsync(CancellationToken cancelToken)
@@ -363,7 +356,7 @@ namespace Duplicati.Library.Backend
using var response = await requestResources.HttpClient.SendAsync(requestResources.RequestMessage, combinedTokens.Token).ConfigureAwait(false);
response.EnsureSuccessStatusCode(); // This replaces the if needed when Mono was used.
response.EnsureSuccessStatusCode();
}
#endregion
@@ -409,7 +402,7 @@ namespace Duplicati.Library.Backend
var request = new HttpRequestMessage(HttpMethod.Get, $"{m_url}{Utility.Uri.UrlEncode(remotename).Replace("+", "%20")}");
request.Headers.Add(HttpRequestHeader.UserAgent.ToString(), "Duplicati WEBDAV Client v" + System.Reflection.Assembly.GetExecutingAssembly().GetName().Version);
request.Headers.ConnectionClose = !m_useIntegratedAuthentication; // ConnectionClose is incompatible with integrated authentication
if (method != null)
@@ -435,7 +428,7 @@ namespace Duplicati.Library.Backend
using var response = await requestResources.HttpClient.SendAsync(requestResources.RequestMessage, HttpCompletionOption.ResponseHeadersRead, cancelToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode(); // This replaces the if needed when Mono was used.
response.EnsureSuccessStatusCode();
}
catch (HttpRequestException wex)
{
Duplicati/Library/Backend/pCloud/Model/pCloudFileEntry.cs
+5
View File
@@ -44,6 +44,11 @@ internal class pCloudFileEntry : IFileEntry
/// </summary>
public DateTime LastModification { get; set; }
/// <summary>
/// Creation of the file
/// </summary>
public DateTime Created { get; set; }
/// <summary>
/// Filename or folder name
/// </summary>
Duplicati/Library/Backend/pCloud/pCloudBackend.cs
+4 -9
View File
@@ -33,6 +33,7 @@ using System.Threading.Tasks;
using Duplicati.Library.Backend.pCloud;
using Duplicati.Library.Utility;
using Uri = System.Uri;
using System.Runtime.CompilerServices;
namespace Duplicati.Library.Backend;
@@ -222,19 +223,13 @@ public class pCloudBackend : IStreamingBackend
/// Implementation of interface method for listing remote folder contents
/// </summary>
/// <returns>List of IFileEntry with directory listing result</returns>
public async Task<IEnumerable<IFileEntry>> ListAsync(CancellationToken cancellationToken)
public async IAsyncEnumerable<IFileEntry> ListAsync([EnumeratorCancellation] CancellationToken cancellationToken)
{
_CachedFolderID ??= await GetFolderId(cancellationToken).ConfigureAwait(false);
return await List(_CachedFolderID.Value, cancellationToken).ConfigureAwait(false);
foreach (var v in await List(_CachedFolderID.Value, cancellationToken).ConfigureAwait(false))
yield return v;
}
/// <summary>
/// Wrapper method of legacy non async call to list files in the remote folder
/// </summary>
/// <returns></returns>
public IEnumerable<IFileEntry> List() => ListAsync(CancellationToken.None).Await();
/// <summary>
/// Upload files to remote location
/// </summary>
Duplicati/Library/Common/IO/FileEntry.cs
+5
View File
@@ -44,6 +44,11 @@ namespace Duplicati.Library.Common.IO
/// </summary>
public DateTime LastModification { get; set; } = new DateTime();
/// <summary>
/// Gets or sets the time the file or folder was created
/// </summary>
public DateTime Created { get; set; } = new DateTime();
/// <summary>
/// Gets or sets the size of the file or folder
/// </summary>
Duplicati/Library/Common/IO/ISystemIO.cs
+29 -19
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
@@ -82,6 +82,16 @@ namespace Duplicati.Library.Common.IO
void SetMetadata(string path, Dictionary<string, string> metdata, bool restorePermissions);
Dictionary<string, string> GetMetadata(string path, bool isSymlink, bool followSymlink);
/// <summary>
/// Sets the permission to read-write only for the current user.
/// </summary>
/// <param name="path">The file to set permissions on.</param>
void FileSetPermissionUserRWOnly(string path);
/// <summary>
/// Sets the permission to read-write only for the current user.
/// </summary>
/// <param name="path">The directory to set permissions on.</param>
void DirectorySetPermissionUserRWOnly(string path);
}
}
Duplicati/Library/Common/IO/SystemIO.cs
+25 -42
View File
@@ -1,61 +1,44 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Runtime.Versioning;
namespace Duplicati.Library.Common.IO
{
public static class SystemIO
{
/// <summary>
/// A cached lookup for windows methods for dealing with long filenames
/// </summary>
[SupportedOSPlatform("windows")]
public static readonly ISystemIO IO_WIN;
[SupportedOSPlatform("linux")]
[SupportedOSPlatform("macOS")]
public static readonly ISystemIO IO_SYS;
public static readonly ISystemIO IO_OS;
static SystemIO()
{
// TODO: These interfaces cannot be properly guarded by the supported platform attribute in this form.
// They are used in static methods of USNJournal on all platforms.
// Since that is the case, the warnings will be suppressed with pragma.
#pragma warning disable CA1416
IO_WIN = new SystemIOWindows();
IO_SYS = new SystemIOLinux();
#pragma warning restore CA1416
if (OperatingSystem.IsWindows())
{
IO_OS = IO_WIN;
IO_OS = new SystemIOWindows();
}
else if (OperatingSystem.IsMacOS() || OperatingSystem.IsLinux())
{
IO_OS = IO_SYS;
IO_OS = new SystemIOLinux();
}
else
{
throw new PlatformNotSupportedException("The current platform is not supported");
}
}
}
Duplicati/Library/Common/IO/SystemIOLinux.cs
+78 -22
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
@@ -25,6 +25,7 @@ using System.Linq;
using System.Collections.Generic;
using Duplicati.Library.Interface;
using System.Runtime.Versioning;
using System.Runtime.InteropServices;
namespace Duplicati.Library.Common.IO
{
@@ -32,6 +33,26 @@ namespace Duplicati.Library.Common.IO
[SupportedOSPlatform("macOS")]
public struct SystemIOLinux : ISystemIO
{
/// <summary>
/// PInvoke methods
/// </summary>
private static class PInvoke
{
/// <summary>
/// Gets the user ID of the current user
/// </summary>
/// <returns>The user ID</returns>
[DllImport("libc")]
public static extern uint getuid();
/// <summary>
/// Gets the group ID of the current user
/// </summary>
/// <returns></returns>
[DllImport("libc")]
public static extern uint getgid();
}
#region ISystemIO implementation
public void DirectoryCreate(string path)
@@ -64,7 +85,8 @@ namespace Duplicati.Library.Common.IO
File.SetLastWriteTimeUtc(path, time);
var gtime = FileGetLastWriteTimeUtc(path);
if(gtime != time) {
if (gtime != time)
{
Console.Error.WriteLine($"DISS: {path} {gtime.ToFileTimeUtc() - time.ToFileTimeUtc()}");
}
}
@@ -214,7 +236,7 @@ namespace Duplicati.Library.Common.IO
var n = PosixFile.GetExtendedAttributes(f, isSymlink, followSymlink);
if (n != null)
foreach(var x in n)
foreach (var x in n)
dict["unix-ext:" + x.Key] = Convert.ToBase64String(x.Value);
var fse = PosixFile.GetUserGroupAndPermissions(f);
@@ -238,7 +260,7 @@ namespace Duplicati.Library.Common.IO
var f = NormalizePath(file);
foreach(var x in data.Where(x => x.Key.StartsWith("unix-ext:", StringComparison.Ordinal)).Select(x => new KeyValuePair<string, byte[]>(x.Key.Substring("unix-ext:".Length), Convert.FromBase64String(x.Value))))
foreach (var x in data.Where(x => x.Key.StartsWith("unix-ext:", StringComparison.Ordinal)).Select(x => new KeyValuePair<string, byte[]>(x.Key.Substring("unix-ext:".Length), Convert.FromBase64String(x.Value))))
PosixFile.SetExtendedAttribute(f, x.Key, x.Value);
if (restorePermissions && data.ContainsKey("unix:uid-gid-perm"))
@@ -346,6 +368,40 @@ namespace Duplicati.Library.Common.IO
{
return new FileEntry(fileInfo.Name, fileInfo.Length, fileInfo.LastAccessTime, fileInfo.LastWriteTime);
}
private static (long uid, long gid) GetOwnerAndGroup(string path)
{
var fi = PosixFile.GetUserGroupAndPermissions(path);
return (fi.UID, fi.GID);
}
/// <summary>
/// Sets the unix permission user read-write Only.
/// </summary>
/// <param name="path">The file to set permissions on.</param>
public void FileSetPermissionUserRWOnly(string path)
{
PosixFile.SetUserGroupAndPermissions(
path,
PInvoke.getuid(),
PInvoke.getgid(),
Convert.ToInt32("600", 8) /* FilePermissions.S_IRUSR | FilePermissions.S_IWUSR*/
);
}
/// <summary>
/// Sets the permission to read-write only for the current user.
/// </summary>
/// <param name="path">The directory to set permissions on.</param>
public void DirectorySetPermissionUserRWOnly(string path)
{
PosixFile.SetUserGroupAndPermissions(
path,
PInvoke.getuid(),
PInvoke.getgid(),
Convert.ToInt32("700", 8) /* FilePermissions.S_IRUSR | FilePermissions.S_IWUSR | FilePermissions.S_IXUSR */
);
}
}
}
Duplicati/Library/Common/IO/SystemIOWindows.cs
+93 -19
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
@@ -28,9 +28,11 @@ using System.Linq;
using Duplicati.Library.Interface;
using Newtonsoft.Json;
using System.Runtime.Versioning;
using System.Security.Principal;
namespace Duplicati.Library.Common.IO
{
[SupportedOSPlatform("windows")]
public struct SystemIOWindows : ISystemIO
{
// Based on the constant names used in
@@ -42,6 +44,16 @@ namespace Duplicati.Library.Common.IO
private static readonly string DIRSEP = Util.DirectorySeparatorString;
/// <summary>
/// The current user name
/// </summary>
private static readonly string CURRENT_USERNAME = WindowsIdentity.GetCurrent().Name;
/// <summary>
/// The LocalSystem user name
/// </summary>
private static readonly string LOCAL_SYSTEM_NAME = new SecurityIdentifier(WellKnownSidType.LocalSystemSid, null).Translate(typeof(NTAccount)).Value;
/// <summary>
/// Prefix path with one of the extended device path prefixes
/// (@"\\?\" or @"\\?\UNC\") but only if it's a fully qualified
@@ -671,6 +683,68 @@ namespace Duplicati.Library.Common.IO
if ((attr & System.IO.FileAttributes.ReparsePoint) == 0)
throw new System.IO.IOException(string.Format("Unable to create symlink, check account permissions: {0}", symlinkfile));
}
/// <summary>
/// Sets the unix permission user read-write Only.
/// </summary>
/// <param name="path">The file to set permissions on.</param>
public void FileSetPermissionUserRWOnly(string path)
{
// Create directory security settings
var security = new FileSecurity();
// Remove inherited permissions to ensure only the current user has access
security.SetAccessRuleProtection(true, false);
// Grant the current user read access
security.AddAccessRule(new FileSystemAccessRule(
CURRENT_USERNAME,
FileSystemRights.FullControl,
AccessControlType.Allow
));
security.AddAccessRule(new FileSystemAccessRule(
LOCAL_SYSTEM_NAME,
FileSystemRights.FullControl,
AccessControlType.Allow
));
// Adjust with the new security settings
new FileInfo(path).SetAccessControl(security);
}
/// <summary>
/// Sets the permission to read-write only for the current user.
/// </summary>
/// <param name="path">The directory to set permissions on.</param>
public void DirectorySetPermissionUserRWOnly(string path)
{
// Create directory security settings
var security = new DirectorySecurity();
// Remove inherited permissions to ensure only the current user has access
security.SetAccessRuleProtection(true, false);
// Grant the current user read access
security.AddAccessRule(new FileSystemAccessRule(
CURRENT_USERNAME,
FileSystemRights.FullControl,
InheritanceFlags.ContainerInherit | InheritanceFlags.ObjectInherit, // Apply to subfolders & files
PropagationFlags.None, // Keeps inheritance settings intact
AccessControlType.Allow
));
security.AddAccessRule(new FileSystemAccessRule(
LOCAL_SYSTEM_NAME,
FileSystemRights.FullControl,
InheritanceFlags.ContainerInherit | InheritanceFlags.ObjectInherit, // Apply to subfolders & files
PropagationFlags.None, // Keeps inheritance settings intact
AccessControlType.Allow
));
// Adjust with the new security settings
new DirectoryInfo(path).SetAccessControl(security);
}
#endregion
}
}
Duplicati/Library/Common/IO/Util.cs
+40 -19
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
@@ -31,8 +31,16 @@ namespace Duplicati.Library.Common.IO
/// </summary>
public static readonly string DirectorySeparatorString = Path.DirectorySeparatorChar.ToString();
/// <summary>
/// A cached instance of the alternate directory separator as a string
/// </summary>
public static readonly string AltDirectorySeparatorString = Path.AltDirectorySeparatorChar.ToString();
/// <summary>
/// Filename of a marker file that can be put inside the data folder to prevent Duplicati from fixing lax permissions
/// </summary>
public const string InsecurePermissionsMarkerFile = "insecure-permissions.txt";
/// <summary>
/// Appends the appropriate directory separator to paths, depending on OS.
/// Does not append the separator if the path already ends with it.
@@ -66,5 +74,18 @@ namespace Duplicati.Library.Common.IO
{
return string.IsNullOrWhiteSpace(path) || path.StartsWith("/", StringComparison.Ordinal) ? "/" : "\\";
}
/// <summary>
/// Checks if the path is inside the Windows folder
/// </summary>
/// <param name="path">The path to check</param>
/// <returns><c>true</c> if the path is inside the Windows folder, <c>false</c> otherwise</returns>
public static bool IsPathUnderWindowsFolder(string path)
{
if (!OperatingSystem.IsWindows())
return false;
return path.StartsWith(AppendDirSeparator(Environment.GetFolderPath(Environment.SpecialFolder.Windows)), StringComparison.OrdinalIgnoreCase);
}
}
}
Duplicati/Library/Compression/ZipCompression/FileArchiveZip.cs
+47 -2
View File
@@ -103,6 +103,16 @@ namespace Duplicati.Library.Compression.ZipCompression
/// </summary>
private readonly IZipArchive m_archive;
/// <summary>
/// The fallback archive to use, if any
/// </summary>
private IZipArchive? m_fallbackArchive = null;
/// <summary>
/// The parameters used to create the archive
/// </summary>
private readonly (Stream Stream, ParsedZipOptions Options, CompressionLibrary Library) m_creationParams;
/// <summary>
/// Default constructor, used to read file extension and supported commands
/// </summary>
@@ -142,6 +152,7 @@ namespace Duplicati.Library.Compression.ZipCompression
var unittestMode = Utility.Utility.ParseBoolOption(options.AsReadOnly(), "unittest-mode");
var parsedOptions = new ParsedZipOptions(compressionLevel, compressionType, usingZip64, unittestMode);
var userCompressionLibrary = compressionLibrary;
if (compressionLibrary == CompressionLibrary.Auto)
{
if (compressionType != CompressionType.Deflate || usingZip64)
@@ -165,6 +176,7 @@ namespace Duplicati.Library.Compression.ZipCompression
}
m_archive = archive ?? new SharpCompressZipArchive(stream, mode, parsedOptions);
m_creationParams = (stream, parsedOptions, userCompressionLibrary);
}
#region IFileArchive Members
@@ -219,7 +231,9 @@ namespace Duplicati.Library.Compression.ZipCompression
public void Dispose()
{
m_archive.Dispose();
if (m_fallbackArchive == null)
m_archive.Dispose();
m_fallbackArchive?.Dispose();
}
public string[] ListFiles(string prefix)
@@ -229,7 +243,38 @@ namespace Duplicati.Library.Compression.ZipCompression
=> m_archive.ListFilesWithSize(prefix);
public Stream? OpenRead(string file)
=> m_archive.OpenRead(file);
{
// If we have started the fallback archive, we should continue using it
if (m_fallbackArchive != null)
return m_fallbackArchive.OpenRead(file);
try
{
return m_archive.OpenRead(file);
}
catch (Exception ex)
{
if (m_creationParams.Library == CompressionLibrary.Auto && m_archive is not SharpCompressZipArchive && m_creationParams.Stream.CanSeek)
{
Log.WriteWarningMessage(LOGTAG, "CompressionReadErrorFallback", ex, "Failed to open file with built-in ZIP archive, falling back to SharpCompress");
try
{
m_creationParams.Item1.Seek(0, SeekOrigin.Begin);
m_archive.Dispose();
}
catch
{
}
m_fallbackArchive = new SharpCompressZipArchive(m_creationParams.Stream, ArchiveMode.Read, m_creationParams.Options);
return m_fallbackArchive.OpenRead(file);
}
throw;
}
}
public DateTime GetLastWriteTime(string file)
=> m_archive.GetLastWriteTime(file);
Duplicati/Library/DynamicLoader/Duplicati.Library.DynamicLoader.csproj
+1
View File
@@ -17,6 +17,7 @@
<ProjectReference Include="..\Modules\Builtin\Duplicati.Library.Modules.Builtin.csproj" />
<ProjectReference Include="..\AutoUpdater\Duplicati.Library.AutoUpdater.csproj" />
<ProjectReference Include="..\SecretProvider\Duplicati.Library.SecretProvider.csproj" />
<ProjectReference Include="..\SourceProviders\Duplicati.Library.SourceProviders.csproj" />
</ItemGroup>
<ItemGroup>
Duplicati/Library/DynamicLoader/SourceProviderLoader.cs
+219
View File
@@ -0,0 +1,219 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using Duplicati.Library.Interface;
using System.Linq;
using System.Runtime.ExceptionServices;
using Duplicati.Library.SourceProviders;
using Duplicati.Library.SourceProvider;
using System.Threading.Tasks;
using System.Threading;
namespace Duplicati.Library.DynamicLoader
{
/// <summary>
/// Loads all SourceProviders dynamically and exposes a list of those
/// </summary>
public class SourceProviderLoader
{
/// <summary>
/// Implementation overrides specific to SourceProviders
/// </summary>
private class SourceProviderLoaderSub : DynamicLoader<ISourceProviderModule>
{
/// <summary>
/// Returns the protocol key
/// </summary>
/// <param name="item">The item to load the key for</param>
/// <returns>The protocol key</returns>
protected override string GetInterfaceKey(ISourceProviderModule item)
{
return item.Key;
}
/// <summary>
/// Returns the subfolders searched for SourceProviders
/// </summary>
protected override string[] Subfolders => ["SourceProviders"];
/// <summary>
/// The built-in modules
/// </summary>
protected override IEnumerable<ISourceProviderModule> BuiltInModules => SourceProviderModules.BuiltInSourceProviderModules;
/// <summary>
/// Instanciates a specific SourceProvider, given the url and options
/// </summary>
/// <param name="url">The url to create the instance for</param>
/// <param name="options">The options to pass to the instance constructor</param>
/// <returns>The instanciated SourceProvider or null if the url is not supported</returns>
public ISourceProviderModule GetSourceProvider(string url, string mountPoint, Dictionary<string, string> options)
{
var uri = new Utility.Uri(url);
LoadInterfaces();
var newOpts = new Dictionary<string, string>(options);
foreach (var key in uri.QueryParameters.AllKeys)
newOpts[key] = uri.QueryParameters[key];
lock (m_lock)
{
try
{
if (m_interfaces.ContainsKey(uri.Scheme))
return (ISourceProviderModule)Activator.CreateInstance(m_interfaces[uri.Scheme].GetType(), url, mountPoint, newOpts);
}
catch (System.Reflection.TargetInvocationException tex)
{
if (tex.InnerException != null)
{
// Unwrap exceptions for nicer display. The ExceptionDispatchInfo class allows us to
// rethrow an exception without changing the stack trace.
ExceptionDispatchInfo.Capture(tex.InnerException).Throw();
}
throw;
}
return null;
}
}
/// <summary>
/// Gets the supported commands for a certain url
/// </summary>
/// <param name="url">The url to find commands for</param>
/// <returns>The supported commands or null if the url scheme was not supported</returns>
public IReadOnlyList<ICommandLineArgument> GetSupportedCommands(string url)
{
var uri = new Utility.Uri(url);
LoadInterfaces();
// TODO: The loading logic is replicated in the "GetSourceProvider" method, should be refactored
lock (m_lock)
{
ISourceProviderModule b;
if (m_interfaces.TryGetValue(uri.Scheme, out b) && b != null)
return GetSupportedCommandsCached(b).ToList();
else if (uri.Scheme.EndsWith("s", StringComparison.Ordinal))
{
var tmpscheme = uri.Scheme.Substring(0, uri.Scheme.Length - 1);
if (m_interfaces.TryGetValue(tmpscheme, out b) && b != null)
return GetSupportedCommandsCached(b).ToList();
}
return null;
}
}
}
/// <summary>
/// The static instance used to access SourceProvider information
/// </summary>
private static readonly SourceProviderLoaderSub _SourceProviderLoader = new SourceProviderLoaderSub();
#region Public static API
/// <summary>
/// Gets a list of loaded SourceProviders, the instances can be used to extract interface information, not used to interact with the SourceProvider.
/// </summary>
public static ISourceProviderModule[] SourceProviders { get { return _SourceProviderLoader.Interfaces; } }
/// <summary>
/// Gets a list of keys supported
/// </summary>
public static string[] Keys { get { return _SourceProviderLoader.Keys; } }
/// <summary>
/// Gets the supported commands for a given SourceProvider
/// </summary>
/// <param name="url">The url to find the commands for</param>
/// <returns>The supported commands or null if the url is not supported</returns>
public static IReadOnlyList<ICommandLineArgument> GetSupportedCommands(string url)
{
if (string.IsNullOrEmpty(url))
throw new ArgumentNullException(nameof(url));
var commands = _SourceProviderLoader.GetSupportedCommands(url);
if (commands != null)
return commands;
var backend = BackendLoader.GetBackend(url, []);
if (backend is IFolderEnabledBackend folderBackend)
commands = folderBackend.SupportedCommands.AsReadOnly();
backend?.Dispose();
return commands;
}
/// <summary>
/// Instanciates a specific SourceProvider, given the url and options
/// </summary>
/// <param name="url">The url to create the instance for</param>
/// <param name="mountPoint">The mount point to use</param>
/// <param name="options">The options to pass to the instance constructor</param>
/// <param name="cancellationToken">The cancellation token</param>
/// <returns>The instanciated SourceProvider or null if the url is not supported</returns>
public static async Task<ISourceProviderModule> GetSourceProvider(string url, string mountPoint, Dictionary<string, string> options, CancellationToken cancellationToken)
{
// Source providers are preferred over backends
var provider = _SourceProviderLoader.GetSourceProvider(url, mountPoint, options);
if (provider == null)
{
// See if there is a backend that can also be a source
var backend = BackendLoader.GetBackend(url, options);
if (backend is IFolderEnabledBackend folderBackend)
provider = new BackendSourceProvider(folderBackend, mountPoint);
else
backend?.Dispose();
}
if (provider == null)
return null;
try
{
await provider.Initialize(cancellationToken).ConfigureAwait(false);
return provider;
}
catch
{
provider.Dispose();
throw;
}
}
#endregion
/// <summary>
/// Adds a SourceProvider to the loader
/// </summary>
/// <param name="SourceProvider">The SourceProvider to add</param>
public static void AddSourceProvider(ISourceProviderModule SourceProvider)
{
_SourceProviderLoader.AddModule(SourceProvider);
}
}
}
Duplicati/Library/Interface/BackendExtensions.cs
+8 -7
View File
@@ -18,10 +18,8 @@
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Interface
@@ -36,12 +34,15 @@ namespace Duplicati.Library.Interface
/// As long as the iteration can either complete or find at least one file without throwing, the test is successful
/// </summary>
/// <param name="backend">Backend to test</param>
public static void TestList(this IBackend backend)
/// <param name="cancelToken">Cancellation token</param>
public static async Task TestListAsync(this IBackend backend, CancellationToken cancelToken)
{
using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancelToken);
// If we can iterate successfully, even if it's empty, then the backend test is successful
foreach (IFileEntry file in backend.List())
await foreach (IFileEntry file in backend.ListAsync(cts.Token).ConfigureAwait(false))
{
break;
cts.Cancel();
return;
}
}
}

Some files were not shown because too many files have changed in this diff Show More