haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into feature/alternate-source-provider

Browse Source
This commit is contained in:
Kenneth Skovhede
2025-02-09 16:15:06 +01:00
parent 8c935a7344 5645b8ae79
commit 3caaa78336
18 changed files with 1593 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati.sln
+9
View File
@@ -172,6 +172,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SourceProviders", "SourcePr
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.SourceProvider.Builtin", "Duplicati\Library\SourceProvider\Builtin\Duplicati.Library.SourceProvider.Builtin.csproj", "{6D9742A7-6F32-4571-8276-79716168B525}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tools", "Tools", "{15388C37-9218-4818-972E-738EEA8F1602}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RemoteSynchronization", "Tools\RemoteSynchronization\RemoteSynchronization.csproj", "{D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -486,6 +490,10 @@ Global
{6D9742A7-6F32-4571-8276-79716168B525}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6D9742A7-6F32-4571-8276-79716168B525}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6D9742A7-6F32-4571-8276-79716168B525}.Release|Any CPU.Build.0 = Release|Any CPU
{D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@@ -547,6 +555,7 @@ Global
{8ACA2736-3C69-4FA5-BCF9-5EDF50CAF332} = {566EBBDA-19A4-4056-A615-D901D57D2439}
{836E0557-B40C-4DC7-9A2A-5C062F9ACC6B} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4}
{6D9742A7-6F32-4571-8276-79716168B525} = {424CEF73-4984-430E-9C8B-E61CC0F22074}
{D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D} = {15388C37-9218-4818-972E-738EEA8F1602}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {8B40BAFE-D862-4397-9495-8F5EAF5CE80C}
Duplicati/CommandLine/BackendTester/Program.cs
+63 -50
View File
@@ -118,16 +118,17 @@ namespace Duplicati.CommandLine.BackendTester
for (int i = 0; i < reruns; i++)
{
Console.WriteLine("Starting run no {0}", i);
Console.WriteLine(DateTime.Now.ToShortDateString() + " " + DateTime.Now.ToLongTimeString());
Console.WriteLine(LogTimeStamp + "Starting run no {0}", i);
if (!Run(args, options, i == 0))
return 1;
}
Console.WriteLine("Unittest complete!");
Console.WriteLine(LogTimeStamp + "Unittest complete!");
return 0;
}
catch (Exception ex)
{
Console.WriteLine("Unittest failed: " + ex);
Console.WriteLine(LogTimeStamp + "Unittest failed: " + ex);
}
return 1;
@@ -185,7 +186,7 @@ namespace Duplicati.CommandLine.BackendTester
}
catch (Exception ex)
{
Console.WriteLine("Autocreate folder failed with message: " + ex.Message);
Console.WriteLine(LogTimeStamp + "Autocreate folder failed with message: " + ex.Message);
}
}
@@ -199,7 +200,7 @@ namespace Duplicati.CommandLine.BackendTester
if (Library.Utility.Utility.ParseBoolOption(options, "auto-clean") && first)
if (Library.Utility.Utility.ParseBoolOption(options, "force"))
{
Console.WriteLine("Auto clean, removing file: {0}", fe.Name);
Console.WriteLine(LogTimeStamp + "Auto clean, removing file: {0}", fe.Name);
backend.DeleteAsync(fe.Name, CancellationToken.None).Await();
continue;
}
@@ -208,10 +209,10 @@ namespace Duplicati.CommandLine.BackendTester
var fileCount = curlist.Where(x => !x.IsFolder).Count();
var filenames = curlist.Where(x => !x.IsFolder).Select(x => x.Name).Take(10).ToList();
Console.WriteLine($"*** Remote folder contains {fileCount} file(s), aborting");
Console.WriteLine($"*** First {filenames.Count} file(s): {Environment.NewLine}{string.Join(Environment.NewLine, filenames)}");
Console.WriteLine(LogTimeStamp + $"*** Remote folder contains {fileCount} file(s), aborting");
Console.WriteLine(LogTimeStamp + $"*** First {filenames.Count} file(s): {Environment.NewLine}{string.Join(Environment.NewLine, filenames)}");
if (fileCount > filenames.Count)
Console.WriteLine($"*** ... and {fileCount - filenames.Count} more file(s)");
Console.WriteLine(LogTimeStamp + $"*** ... and {fileCount - filenames.Count} more file(s)");
return false;
}
@@ -232,7 +233,7 @@ namespace Duplicati.CommandLine.BackendTester
{
if (!(backend is IStreamingBackend) || disableStreaming)
{
Console.WriteLine("Warning: Throttling is only supported in this tool on streaming backends");
Console.WriteLine(LogTimeStamp + "Warning: Throttling is only supported in this tool on streaming backends");
}
throttleUpload = Duplicati.Library.Utility.Sizeparser.ParseSize(throttleUploadString, "kb");
@@ -243,16 +244,26 @@ namespace Duplicati.CommandLine.BackendTester
{
if (!(backend is IStreamingBackend) || disableStreaming)
{
Console.WriteLine("Warning: Throttling is only supported in this tool on streaming backends");
Console.WriteLine(LogTimeStamp + "Warning: Throttling is only supported in this tool on streaming backends");
}
throttleDownload = Duplicati.Library.Utility.Sizeparser.ParseSize(throttleDownloadString, "kb");
}
var readWriteTimeout = (int)TimeSpan.FromSeconds(10).TotalMilliseconds;
if (backend is ITimeoutExemptBackend)
readWriteTimeout = Timeout.Infinite;
int readWriteTimeout = backend is ITimeoutExemptBackend ||
(Environment.GetEnvironmentVariable("READ_WRITE_TIMEOUT_SECONDS") is { } timeout
&& int.TryParse(timeout, out var seconds)
&& seconds == -1)
? Timeout.Infinite
: Environment.GetEnvironmentVariable("READ_WRITE_TIMEOUT_SECONDS") is { } timeoutRetry
&& int.TryParse(timeoutRetry, out var secondsRetry)
? (int)TimeSpan.FromSeconds(secondsRetry).TotalMilliseconds
: (int)TimeSpan.FromSeconds(60).TotalMilliseconds;
Console.WriteLine(LogTimeStamp + "Read Write Timeout set to {0}{1}",
readWriteTimeout == Timeout.Infinite ? "infinite" : readWriteTimeout + " ms",
backend is ITimeoutExemptBackend ? " (because implements ITimeoutExemptBackend)" : "");
// Allow overriding the timeout for the backend here, even if timeouts are disabled
if (options.TryGetValue("read-write-timeout", out var readWriteTimeoutString))
readWriteTimeout = (int)Timeparser.ParseTimeSpan(readWriteTimeoutString).TotalMilliseconds;
@@ -298,7 +309,7 @@ namespace Duplicati.CommandLine.BackendTester
byte[] dummyFileHash = null;
if (!skipOverwriteTest)
{
Console.WriteLine("Uploading wrong files ...");
Console.WriteLine(LogTimeStamp + "Uploading wrong files ...");
using (var dummy = Library.Utility.TempFile.WrapExistingFile(CreateRandomFile(tf, files.Count, 1024, 2048, rnd)))
{
using (var fs = new System.IO.FileStream(dummy, System.IO.FileMode.Open, System.IO.FileAccess.Read))
@@ -313,7 +324,7 @@ namespace Duplicati.CommandLine.BackendTester
}
Console.WriteLine("Uploading files ...");
Console.WriteLine(LogTimeStamp + "Uploading files ...");
for (int i = 0; i < files.Count; i++)
Uploadfile(files[i].localfilename, i, files[i].remotefilename, backend, disableStreaming, throttleUpload, readWriteTimeout);
@@ -328,7 +339,7 @@ namespace Duplicati.CommandLine.BackendTester
renamedFileNewName = CreateRandomRemoteFileName(min_filename_size, max_filename_size, allowedChars, trimFilenameSpaces, rnd);
Console.WriteLine("Renaming file {0} from {1} to {2}", renameIndex, originalRenamedFile.remotefilename, renamedFileNewName);
Console.WriteLine(LogTimeStamp + "Renaming file {0} from {1} to {2}", renameIndex, originalRenamedFile.remotefilename, renamedFileNewName);
renameEnabledBackend.RenameAsync(originalRenamedFile.remotefilename, renamedFileNewName, CancellationToken.None).Await();
files[renameIndex] = new TempFile(renamedFileNewName, originalRenamedFile.localfilename, originalRenamedFile.hash, originalRenamedFile.length);
@@ -336,11 +347,11 @@ namespace Duplicati.CommandLine.BackendTester
if (waitAfterUpload > TimeSpan.Zero)
{
Console.WriteLine("Waiting {0} after upload", waitAfterUpload);
Console.WriteLine(LogTimeStamp + "Waiting {0} after upload", waitAfterUpload);
Thread.Sleep(waitAfterUpload);
}
Console.WriteLine("Verifying file list ...");
Console.WriteLine(LogTimeStamp + "Verifying file list ...");
curlist = backend.List();
foreach (var fe in curlist)
@@ -351,12 +362,12 @@ namespace Duplicati.CommandLine.BackendTester
if (tx.remotefilename == fe.Name)
{
if (tx.found)
Console.WriteLine("*** File with name {0} was found more than once", tx.remotefilename);
Console.WriteLine(LogTimeStamp + "*** File with name {0} was found more than once", tx.remotefilename);
found = true;
tx.found = true;
if (fe.Size > 0 && tx.length != fe.Size)
Console.WriteLine("*** File with name {0} has size {1} but the size was reported as {2}", tx.remotefilename, tx.length, fe.Size);
Console.WriteLine(LogTimeStamp + "*** File with name {0} has size {1} but the size was reported as {2}", tx.remotefilename, tx.length, fe.Size);
break;
}
@@ -364,26 +375,26 @@ namespace Duplicati.CommandLine.BackendTester
if (!found)
if (originalRenamedFile != null && renamedFileNewName != null && originalRenamedFile.remotefilename == fe.Name)
{
Console.WriteLine("*** File with name {0} was found on server but was supposed to have been renamed to {1}!", fe.Name, renamedFileNewName);
Console.WriteLine(LogTimeStamp + "*** File with name {0} was found on server but was supposed to have been renamed to {1}!", fe.Name, renamedFileNewName);
}
else
{
Console.WriteLine("*** File with name {0} was found on server but not uploaded!", fe.Name);
Console.WriteLine(LogTimeStamp + "*** File with name {0} was found on server but not uploaded!", fe.Name);
}
}
foreach (var tx in files)
if (!tx.found)
Console.WriteLine("*** File with name {0} was uploaded but not found afterwards", tx.remotefilename);
Console.WriteLine(LogTimeStamp + "*** File with name {0} was uploaded but not found afterwards", tx.remotefilename);
Console.WriteLine("Downloading files");
Console.WriteLine(LogTimeStamp + "Downloading files");
for (int i = 0; i < files.Count; i++)
{
using (var cf = new Duplicati.Library.Utility.TempFile())
{
Exception e = null;
Console.Write("Downloading file {0} ... ", i);
Console.Write(LogTimeStamp + "Downloading file {0} ... ", i);
var sw = Stopwatch.StartNew();
@@ -410,22 +421,22 @@ namespace Duplicati.CommandLine.BackendTester
if (e != null)
{
failAfterFinished = true;
Console.WriteLine($"failed\n*** Error: {e} after {sw.ElapsedMilliseconds} ms");
Console.WriteLine(LogTimeStamp + $"failed\n*** Error: {e} after {sw.ElapsedMilliseconds} ms");
}
else
Console.WriteLine($"done in {sw.ElapsedMilliseconds} ms");
Console.WriteLine($" done in {sw.ElapsedMilliseconds} ms");
Console.Write("Checking hash ... ");
Console.Write(LogTimeStamp + "Checking hash ... ");
using (var fs = new System.IO.FileStream(cf, System.IO.FileMode.Open, System.IO.FileAccess.Read))
if (Convert.ToBase64String(sha.ComputeHash(fs)) != Convert.ToBase64String(files[i].hash))
{
if (dummyFileHash != null && Convert.ToBase64String(sha.ComputeHash(fs)) == Convert.ToBase64String(dummyFileHash))
Console.WriteLine("failed\n*** Downloaded file was the dummy file"); // Should this be failed?
Console.WriteLine(LogTimeStamp + "failed\n*** Downloaded file was the dummy file"); // Should this be failed?
else
{
failAfterFinished = true;
Console.WriteLine("failed\n*** Downloaded file was corrupt");
Console.WriteLine(LogTimeStamp + "failed\n*** Downloaded file was corrupt");
}
}
else
@@ -433,19 +444,19 @@ namespace Duplicati.CommandLine.BackendTester
}
}
Console.WriteLine("Deleting files...");
Console.WriteLine(LogTimeStamp + "Deleting files...");
foreach (var tx in files)
try { backend.DeleteAsync(tx.remotefilename, CancellationToken.None).Await(); }
catch (Exception ex)
{
failAfterFinished = true;
Console.WriteLine("*** Failed to delete file {0}, message: {1}", tx.remotefilename, ex);
Console.WriteLine(LogTimeStamp + "*** Failed to delete file {0}, message: {1}", tx.remotefilename, ex);
}
if (waitAfterDelete > TimeSpan.Zero)
{
Console.WriteLine("Waiting {0} after delete", waitAfterDelete);
Console.WriteLine(LogTimeStamp + "Waiting {0} after delete", waitAfterDelete);
Thread.Sleep(waitAfterDelete);
}
@@ -453,11 +464,11 @@ namespace Duplicati.CommandLine.BackendTester
foreach (var fe in curlist)
if (!fe.IsFolder)
{
Console.WriteLine("*** Remote folder contains {0} after cleanup", fe.Name);
Console.WriteLine(LogTimeStamp + "*** Remote folder contains {0} after cleanup", fe.Name);
}
// Test some error cases
Console.WriteLine("Checking retrieval of non-existent file...");
Console.WriteLine(LogTimeStamp + "Checking retrieval of non-existent file...");
var caughtExpectedException = false;
try
{
@@ -468,25 +479,25 @@ namespace Duplicati.CommandLine.BackendTester
}
catch (FileMissingException)
{
Console.WriteLine("Caught expected FileMissingException");
Console.WriteLine(LogTimeStamp + "Caught expected FileMissingException");
caughtExpectedException = true;
}
catch (Exception ex)
{
Console.WriteLine("*** Retrieval of non-existent file failed: {0}", ex);
Console.WriteLine(LogTimeStamp + "*** Retrieval of non-existent file failed: {0}", ex);
}
if (!caughtExpectedException)
{
failAfterFinished = true;
Console.WriteLine("*** Retrieval of non-existent file should have failed with FileMissingException");
Console.WriteLine(LogTimeStamp + "*** Retrieval of non-existent file should have failed with FileMissingException");
}
}
// Test quota retrieval
if (backend is IQuotaEnabledBackend quotaEnabledBackend)
{
Console.WriteLine("Checking quota...");
Console.WriteLine(LogTimeStamp + "Checking quota...");
IQuotaInfo quota = null;
bool noException;
try
@@ -496,7 +507,7 @@ namespace Duplicati.CommandLine.BackendTester
}
catch (Exception ex)
{
Console.WriteLine("*** Checking quota information failed: {0}", ex);
Console.WriteLine(LogTimeStamp + "*** Checking quota information failed: {0}", ex);
noException = false;
}
@@ -504,18 +515,18 @@ namespace Duplicati.CommandLine.BackendTester
{
if (quota != null)
{
Console.WriteLine("Free Space: {0}", Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace));
Console.WriteLine("Total Space: {0}", Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace));
Console.WriteLine(LogTimeStamp + "Free Space: {0}", Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace));
Console.WriteLine(LogTimeStamp + "Total Space: {0}", Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace));
}
else
{
Console.WriteLine("Unable to retrieve quota information");
Console.WriteLine(LogTimeStamp + "Unable to retrieve quota information");
}
}
}
// Test DNSName lookup
Console.WriteLine("Checking DNS names used by this backend...");
Console.WriteLine(LogTimeStamp + "Checking DNS names used by this backend...");
try
{
var dnsNames = backend.GetDNSNamesAsync(CancellationToken.None).Await();
@@ -528,12 +539,12 @@ namespace Duplicati.CommandLine.BackendTester
}
else
{
Console.WriteLine("No DNS names reported");
Console.WriteLine(LogTimeStamp + "No DNS names reported");
}
}
catch (Exception ex)
{
Console.WriteLine("*** Checking DNSName failed: {0}", ex);
Console.WriteLine(LogTimeStamp + "*** Checking DNSName failed: {0}", ex);
}
}
finally
@@ -548,7 +559,7 @@ namespace Duplicati.CommandLine.BackendTester
private static void Uploadfile(string localfilename, int i, string remotefilename, IBackend backend, bool disableStreaming, long throttle, int readWriteTimeout)
{
Console.Write("Uploading file {0}, {1} ... ", i, Duplicati.Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(localfilename).Length));
Console.Write(LogTimeStamp + "Uploading file {0}, {1} ... ", i, Duplicati.Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(localfilename).Length));
Exception e = null;
var sw = Stopwatch.StartNew();
@@ -575,11 +586,11 @@ namespace Duplicati.CommandLine.BackendTester
if (e != null)
{
Console.WriteLine($"Failed to upload file {i}, error message: {e}, remote name: {remotefilename} after {sw.ElapsedMilliseconds} ms");
Console.WriteLine(LogTimeStamp + $"Failed to upload file {i}, error message: {e}, remote name: {remotefilename} after {sw.ElapsedMilliseconds} ms");
while (e.InnerException != null)
{
e = e.InnerException;
Console.WriteLine(" Inner exception: {0}", e);
Console.WriteLine(LogTimeStamp + " Inner exception: {0}", e);
}
}
else
@@ -604,7 +615,7 @@ namespace Duplicati.CommandLine.BackendTester
private static string CreateRandomFile(Library.Utility.TempFolder tf, int i, int min_file_size, int max_file_size, Random rnd)
{
Console.Write("Generating file {0}", i);
Console.Write(LogTimeStamp + "Generating file {0}", i);
string filename = System.IO.Path.Combine(tf, i.ToString());
using (System.IO.FileStream fs = new System.IO.FileStream(filename, System.IO.FileMode.CreateNew, System.IO.FileAccess.Write))
{
@@ -625,6 +636,8 @@ namespace Duplicati.CommandLine.BackendTester
return filename;
}
private static string LogTimeStamp => $"[{DateTime.Now:HH:mm:ss fff}] ";
public static IList<ICommandLineArgument> SupportedCommands
{
get
Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs
+2 -4
View File
@@ -123,16 +123,14 @@ partial class BackendManager
{
// extended to use stacked streams
using (var fs = System.IO.File.OpenWrite(dlTarget))
using (var act = new StreamUtil.TimeoutObservingStream(fs) { WriteTimeout = backend is ITimeoutExemptBackend ? Timeout.Infinite : Context.Options.ReadWriteTimeout })
using (var hasher = HashFactory.CreateHasher(Context.Options.FileHashAlgorithm))
using (var hs = new HashCalculatingStream(act, hasher))
using (var hs = new HashCalculatingStream(fs, hasher))
using (var ss = new ShaderStream(hs, true))
using (var linkedToken = CancellationTokenSource.CreateLinkedTokenSource(cancelToken, act.TimeoutToken))
{
using (var ts = new ThrottledStream(ss, 0, Context.Options.MaxDownloadPrSecond))
using (var pgs = new ProgressReportingStream(ts, pg => Context.HandleProgress(ts, pg, RemoteFilename)))
{
await streamingBackend.GetAsync(RemoteFilename, pgs, linkedToken.Token).ConfigureAwait(false);
await streamingBackend.GetAsync(RemoteFilename, pgs, cancelToken).ConfigureAwait(false);
}
ss.Flush();
retDownloadSize = ss.TotalBytesWritten;
Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs
+4 -6
View File
@@ -268,12 +268,10 @@ partial class BackendManager
if (backend is IStreamingBackend streamingBackend && !Context.Options.DisableStreamingTransfers)
{
using (var fs = System.IO.File.OpenRead(LocalFilename))
using (var act = new StreamUtil.TimeoutObservingStream(fs) { ReadTimeout = backend is ITimeoutExemptBackend ? Timeout.Infinite : Context.Options.ReadWriteTimeout })
using (var ts = new ThrottledStream(act, Context.Options.MaxUploadPrSecond, 0))
using (var pgs = new ProgressReportingStream(ts, pg => Context.HandleProgress(ts, pg, RemoteFilename)))
using (var linkedToken = CancellationTokenSource.CreateLinkedTokenSource(cancelToken, act.TimeoutToken))
await streamingBackend.PutAsync(RemoteFilename, pgs, linkedToken.Token).ConfigureAwait(false);
using var fs = System.IO.File.OpenRead(LocalFilename);
using var ts = new ThrottledStream(fs, Context.Options.MaxUploadPrSecond, 0);
using var pgs = new ProgressReportingStream(ts,pg => Context.HandleProgress(ts, pg, RemoteFilename));
await streamingBackend.PutAsync(RemoteFilename, pgs, cancelToken).ConfigureAwait(false);
}
else
await backend.PutAsync(RemoteFilename, LocalFilename, cancelToken).ConfigureAwait(false);
Duplicati/Library/Main/Operation/RepairHandler.cs
+7
View File
@@ -141,6 +141,13 @@ namespace Duplicati.Library.Main.Operation
var progress = 0;
var targetProgess = tp.ExtraVolumes.Count() + tp.MissingVolumes.Count() + tp.VerificationRequiredVolumes.Count();
var mostRecentLocal = db.FilesetTimes.Select(x => x.Value).Append(DateTime.MinValue).Max();
var mostRecentRemote = tp.ParsedVolumes.Select(x => x.Time).Append(DateTime.MinValue).Max();
if (mostRecentLocal < DateTime.UnixEpoch)
throw new UserInformationException("The local database has no fileset times. Consider deleting the local database and run the repair operation again.", "LocalDatabaseHasNoFilesetTimes");
if (mostRecentRemote > mostRecentLocal)
throw new UserInformationException($"The remote files are newer ({mostRecentRemote}) than the local database ({mostRecentLocal}), this is likely because the database is outdated. Consider deleting the local database and run the repair operation again.", "RemoteFilesNewerThanLocalDatabase");
if (m_options.Dryrun)
{
if (!tp.ParsedVolumes.Any() && tp.OtherVolumes.Any())
Duplicati/Library/Modules/Builtin/SendTelegramMessage.cs
+32 -14
View File
@@ -56,6 +56,10 @@ public class SendTelegramMessage : ReportHelper
/// </summary>
private const string OPTION_CHANNEL = "send-telegram-channel-id";
/// <summary>
/// Option used to specify the topic ID to be used in telegram groups.
/// </summary>
private const string OPTION_TOPICID = "send-telegram-topid-id";
/// <summary>
/// Option used to specify report body
/// </summary>
private const string OPTION_MESSAGE = "send-telegram-message";
@@ -133,20 +137,19 @@ public class SendTelegramMessage : ReportHelper
/// </summary>
public override IList<ICommandLineArgument> SupportedCommands
=> [
new CommandLineArgument(OPTION_CHANNEL, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendtelegramchannelShort, Strings.SendTelegramMessage.SendtelegramchannelLong),
new CommandLineArgument(OPTION_MESSAGE, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendtelegrammessageShort, Strings.SendTelegramMessage.SendtelegrammessageLong, DEFAULT_BODY),
new CommandLineArgument(OPTION_BOTID, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendtelegrambotidShort, Strings.SendTelegramMessage.SendtelegrambotidLong),
new CommandLineArgument(OPTION_APIKEY, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendtelegramapikeyShort, Strings.SendTelegramMessage.SendtelegramapikeyLong),
new CommandLineArgument(OPTION_SENDLEVEL, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendtelegramlevelShort, Strings.SendTelegramMessage.SendtelegramlevelLong(ParsedResultType.Success.ToString(), ParsedResultType.Warning.ToString(), ParsedResultType.Error.ToString(), ParsedResultType.Fatal.ToString(), "All"), DEFAULT_LEVEL, null, Enum.GetNames(typeof(ParsedResultType)).Union(new string[] { "All" } ).ToArray()),
new CommandLineArgument(OPTION_SENDALL, CommandLineArgument.ArgumentType.Boolean, Strings.SendTelegramMessage.SendtelegramanyoperationShort, Strings.SendTelegramMessage.SendtelegramanyoperationLong),
new CommandLineArgument(OPTION_CHANNEL, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramChannelShort, Strings.SendTelegramMessage.SendTelegramChannelLong),
new CommandLineArgument(OPTION_MESSAGE, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramMessageShort, Strings.SendTelegramMessage.SendTelegramMessageLong, DEFAULT_BODY),
new CommandLineArgument(OPTION_BOTID, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramBotIdShort, Strings.SendTelegramMessage.SendTelegramBotIdLong),
new CommandLineArgument(OPTION_APIKEY, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramApiKeyShort, Strings.SendTelegramMessage.SendTelegramApiKeyLong),
new CommandLineArgument(OPTION_SENDLEVEL, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramLevelShort, Strings.SendTelegramMessage.SendTelegramlevelLong(ParsedResultType.Success.ToString(), ParsedResultType.Warning.ToString(), ParsedResultType.Error.ToString(), ParsedResultType.Fatal.ToString(), "All"), DEFAULT_LEVEL, null, Enum.GetNames(typeof(ParsedResultType)).Union(new string[] { "All" } ).ToArray()),
new CommandLineArgument(OPTION_SENDALL, CommandLineArgument.ArgumentType.Boolean, Strings.SendTelegramMessage.SendTelegramManyOperationShort, Strings.SendTelegramMessage.SendTelegramManyOperationLong),
new CommandLineArgument(OPTION_LOG_LEVEL, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.OptionLoglevelShort, Strings.ReportHelper.OptionLoglevelLong, DEFAULT_LOG_LEVEL.ToString(), null, Enum.GetNames(typeof(Logging.LogMessageType))),
new CommandLineArgument(OPTION_LOG_FILTER, CommandLineArgument.ArgumentType.String, Strings.ReportHelper.OptionLogfilterShort, Strings.ReportHelper.OptionLogfilterLong),
new CommandLineArgument(OPTION_MAX_LOG_LINES, CommandLineArgument.ArgumentType.Integer, Strings.ReportHelper.OptionmaxloglinesShort, Strings.ReportHelper.OptionmaxloglinesLong, DEFAULT_LOGLINES.ToString()),
new CommandLineArgument(OPTION_RESULT_FORMAT, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.ResultFormatShort, Strings.ReportHelper.ResultFormatLong(Enum.GetNames(typeof(ResultExportFormat))), DEFAULT_EXPORT_FORMAT.ToString(), null, Enum.GetNames(typeof(ResultExportFormat))),
new CommandLineArgument(OPTION_TOPICID, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramTopicShort, Strings.SendTelegramMessage.SendTelegramTopicLong),
];
protected override string SubjectOptionName => OPTION_MESSAGE;
protected override string BodyOptionName => OPTION_MESSAGE;
protected override string ActionLevelOptionName => OPTION_SENDLEVEL;
@@ -168,6 +171,10 @@ public class SendTelegramMessage : ReportHelper
/// The Telegram ChannelID
/// </summary>
private string m_channelId;
/// <summary>
/// The telegram TopicID
/// </summary>
private string m_topicId;
/// <summary>
/// This method is the interception where the module can interact with the execution environment and modify the settings.
@@ -182,12 +189,14 @@ public class SendTelegramMessage : ReportHelper
commandlineOptions.TryGetValue(OPTION_BOTID, out m_botid);
commandlineOptions.TryGetValue(OPTION_APIKEY, out m_apikey);
commandlineOptions.TryGetValue(OPTION_TOPICID, out m_topicId);
return true;
}
#endregion
/// <inheritdoc />
protected override string ReplaceTemplate(string input, object result, Exception exception, bool subjectline)
{
// No need to do the expansion as we throw away the result
@@ -220,7 +229,7 @@ public class SendTelegramMessage : ReportHelper
// Send all chunks sequentially
for (var i = 0; i < messages.Length; i++)
await SendMessageChunk(messages[i], i + 1, messages.Length);
await SendMessageChunk(messages[i], i + 1, messages.Length).ConfigureAwait(false);
}
catch (Exception e)
{
@@ -239,9 +248,15 @@ public class SendTelegramMessage : ReportHelper
{
try
{
if (string.IsNullOrWhiteSpace(m_botid))
throw new Exception("Telegram Bot ID is required and not set");
if (string.IsNullOrWhiteSpace(m_apikey))
throw new Exception("Telegram API Key is required and not set");
var p = new
{
chat_id = Uri.EscapeDataString(m_channelId),
message_thread_id = string.IsNullOrWhiteSpace(m_topicId) ? null : Uri.EscapeDataString(m_topicId),
parse_mode = "Markdown",
text = Uri.EscapeDataString(totalParts > 1
? $"Part {partNumber}/{totalParts}:\n{message}"
@@ -250,13 +265,16 @@ public class SendTelegramMessage : ReportHelper
apiKey = Uri.EscapeDataString(m_apikey)
};
var url = $"https://api.telegram.org/bot{p.botId}:{p.apiKey}/sendMessage?chat_id={p.chat_id}&parse_mode={p.parse_mode}&text={p.text}";
var baseUrl = $"https://api.telegram.org/bot{p.botId}:{p.apiKey}/sendMessage";
var url = $"{baseUrl}?chat_id={p.chat_id}&parse_mode={p.parse_mode}&text={p.text}" +
$"{(!string.IsNullOrWhiteSpace(m_topicId) ? $"&message_thread_id={m_topicId}" : string.Empty)}";
using var timeoutToken = new CancellationTokenSource();
timeoutToken.CancelAfter(REQUEST_TIMEOUT);
using var client = HttpClientHelper.CreateClient();
var response = await client.GetAsync(url, timeoutToken.Token);
var responseContent = await response.Content.ReadAsStringAsync(timeoutToken.Token);
var response = await client.GetAsync(url, timeoutToken.Token).ConfigureAwait(false);
var responseContent = await response.Content.ReadAsStringAsync(timeoutToken.Token).ConfigureAwait(false);
if (!responseContent.Contains("\"ok\":true"))
Logging.Log.WriteWarningMessage(LOGTAG, "telegramSendError", null,
Duplicati/Library/Modules/Builtin/Strings.cs
+37 -35
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Localization.Short;
using System;
@@ -167,11 +167,12 @@ You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The sp
internal static class SendTelegramMessage
{
public static string Description { get { return LC.L(@"This module provides support for sending status reports via Telegram messages"); } }
public static string DisplayName { get { return LC.L(@"Telegram report module"); } }
public static string SendtelegramchannelLong { get { return LC.L(@"Use this option to set the channel ID."); } }
public static string SendtelegramchannelShort { get { return LC.L(@"Telegram channel ID"); } }
public static string SendtelegrammessageLong { get { return LC.L(@"This value can be a filename. If the file exists, the file contents will be used as the message.
public static string Description => LC.L(@"This module provides support for sending status reports via Telegram messages");
public static string DisplayName => LC.L(@"Telegram report module");
public static string SendTelegramChannelLong => LC.L(@"Use this option to set the channel ID.");
public static string SendTelegramChannelShort => LC.L(@"Telegram channel ID");
public static string SendTelegramMessageLong =>
LC.L(@"This value can be a filename. If the file exists, the file contents will be used as the message.
In the message, certain tokens are replaced:
%OPERATIONNAME% - The name of the operation, normally ""Backup""
@@ -179,19 +180,20 @@ In the message, certain tokens are replaced:
%LOCALPATH% - The path to the local files or folders involved in the operation (if any)
%PARSEDRESULT% - The parsed result, if the operation is a backup. Possible values are: Error, Warning, Success
All command line options are also reported within %value%, e.g. %volsize%. Any unknown/unset value is removed."); } }
public static string SendtelegrammessageShort { get { return LC.L(@"The message template"); } }
public static string SendtelegrambotidLong { get { return LC.L(@"Use this option to set a bot ID for the bot that will send the message."); } }
All command line options are also reported within %value%, e.g. %volsize%. Any unknown/unset value is removed.");
public static string SendtelegrambotidShort { get { return LC.L(@"The Telegram bot ID"); } }
public static string SendtelegramapikeyLong { get { return LC.L(@"Use this option to set a API key for the bot that will send the message."); } }
public static string SendtelegramapikeyShort { get { return LC.L(@"The Telegram API key"); } }
public static string SendtelegramlevelLong(string success, string warning, string error, string fatal, string all) { return LC.L(@"You can specify one of ""{0}"", ""{1}"", ""{2}"", ""{3}"".
public static string SendTelegramMessageShort => LC.L(@"The message template");
public static string SendTelegramBotIdLong => LC.L(@"Use this option to set a bot ID for the bot that will send the message.");
public static string SendTelegramBotIdShort => LC.L(@"The Telegram bot ID");
public static string SendTelegramApiKeyLong => LC.L(@"Use this option to set a API key for the bot that will send the message.");
public static string SendTelegramApiKeyShort => LC.L(@"The Telegram API key");
public static string SendTelegramTopicShort => LC.L(@"The Telegram topic ID");
public static string SendTelegramTopicLong => LC.L(@"Topic ID for the Topic in the telegram group. For more information on Telegram setup, refer to documentation.");
public static string SendTelegramlevelLong(string success, string warning, string error, string fatal, string all) { return LC.L(@"You can specify one of ""{0}"", ""{1}"", ""{2}"", ""{3}"".
You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The special value ""{4}"" is a shorthand for ""{0},{1},{2},{3}"" and will cause all backup operations to send a message.", success, warning, error, fatal, all); }
public static string SendtelegramlevelShort { get { return LC.L(@"The messages to send"); } }
public static string SendtelegramanyoperationLong { get { return LC.L(@"By default, messages will only be sent after a backup operation. Use this option to send messages for all operations."); } }
public static string SendtelegramanyoperationShort { get { return LC.L(@"Send messages for all operations"); } }
public static string LoginTimeoutError { get { return LC.L(@"Timeout occurred while sending to Telegram server"); } }
public static string SendTelegramLevelShort => LC.L(@"The messages to send");
public static string SendTelegramManyOperationLong => LC.L(@"By default, messages will only be sent after a backup operation. Use this option to send messages for all operations.");
public static string SendTelegramManyOperationShort => LC.L(@"Send messages for all operations");
}
internal static class SendHttpMessage
Duplicati/Library/RestAPI/Strings.cs
+2
View File
@@ -59,6 +59,8 @@ Error message: {0}", error); }
public static string WebserverAllowedhostnamesDescription { get { return LC.L(@"The hostnames that are accepted, separated with semicolons. If any of the hostnames are ""*"", all hostnames are allowed and the hostname checking is disabled."); } }
public static string PingpongkeepaliveLong { get { return LC.L(@"When running as a server, the service daemon must verify that the process is responding. If this option is enabled, the server reads stdin and writes a reply to each line read."); } }
public static string PingpongkeepaliveShort { get { return LC.L(@"Enable the ping-pong responder"); } }
public static string DisableupdatecheckShort { get { return LC.L(@"Disable the automatic update check"); } }
public static string DisableupdatecheckLong { get { return LC.L(@"Use this option to disable the automatic update check. Manual update checks can still be performed."); } }
public static string LogretentionLong { get { return LC.L(@"Set the time after which log data will be purged from the database."); } }
public static string LogretentionShort { get { return LC.L(@"Clean up old log data"); } }
public static string ServerdatafolderLong(string envname) { return LC.L(@"Duplicati needs to store a small database with all settings. Use this option to choose where the settings are stored. This option can also be set with the environment variable {0}.", envname); }
Duplicati/Library/RestAPI/UpdatePollThread.cs
+11 -5
View File
@@ -39,6 +39,7 @@ namespace Duplicati.Server
private readonly object m_lock = new object();
private AutoResetEvent m_waitSignal;
private double m_downloadProgress;
private bool m_disableChecks = false;
public bool IsUpdateRequested { get; private set; } = false;
@@ -56,8 +57,9 @@ namespace Duplicati.Server
}
}
public void Init()
public void Init(bool disableChecks)
{
m_disableChecks = disableChecks;
m_waitSignal = new AutoResetEvent(false);
ThreadState = UpdatePollerStates.Waiting;
m_thread = new Thread(Run)
@@ -115,16 +117,20 @@ namespace Duplicati.Server
if (nextCheck < DateTime.UtcNow || m_forceCheck)
{
DateTime started = DateTime.UtcNow;
connection.ApplicationSettings.LastUpdateCheck = started;
nextCheck = connection.ApplicationSettings.NextUpdateCheck;
// If this is not forced, and we have disabled checks, we just update the next check time
if (m_disableChecks && !m_forceCheck)
continue;
lock (m_lock)
m_forceCheck = false;
ThreadState = UpdatePollerStates.Checking;
eventPollNotify.SignalNewEvent();
DateTime started = DateTime.UtcNow;
connection.ApplicationSettings.LastUpdateCheck = started;
nextCheck = connection.ApplicationSettings.NextUpdateCheck;
Library.AutoUpdater.ReleaseType rt;
if (!Enum.TryParse<Library.AutoUpdater.ReleaseType>(connection.ApplicationSettings.UpdateChannel, true, out rt))
rt = Duplicati.Library.AutoUpdater.ReleaseType.Unknown;
Duplicati/Server/Program.cs
+3 -1
View File
@@ -51,6 +51,7 @@ namespace Duplicati.Server
private const string DISABLE_DB_ENCRYPTION_OPTION = "disable-db-encryption";
private const string REQUIRE_DB_ENCRYPTION_KEY_OPTION = "require-db-encryption-key";
private const string SETTINGS_ENCRYPTION_KEY_OPTION = "settings-encryption-key";
private const string DISABLE_UPDATE_CHECK_OPTION = "disable-update-check";
#if DEBUG
private const bool DEBUG_MODE = true;
@@ -266,7 +267,7 @@ namespace Duplicati.Server
DuplicatiWebserver = StartWebServer(commandlineOptions, DataConnection).ConfigureAwait(false).GetAwaiter().GetResult();
UpdatePoller.Init();
UpdatePoller.Init(Library.Utility.Utility.ParseBoolOption(commandlineOptions, DISABLE_UPDATE_CHECK_OPTION));
SetPurgeTempFilesTimer(commandlineOptions);
@@ -961,6 +962,7 @@ namespace Duplicati.Server
new Duplicati.Library.Interface.CommandLineArgument(WebServerLoader.OPTION_WEBSERVICE_TIMEZONE, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.String, Strings.Program.WebserverTimezoneDescription, Strings.Program.WebserverTimezoneDescription, TimeZoneHelper.GetLocalTimeZone(), null, TimeZoneHelper.GetTimeZones().Select(x => x.Id).ToArray()),
new Duplicati.Library.Interface.CommandLineArgument(WebServerLoader.OPTION_WEBSERVICE_CORS_ORIGINS, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.Program.WebserverCorsOriginsDescription, Strings.Program.WebserverCorsOriginsDescription, WebServerLoader.DEFAULT_OPTION_SPAPATHS),
new Duplicati.Library.Interface.CommandLineArgument(PING_PONG_KEEPALIVE_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.PingpongkeepaliveShort, Strings.Program.PingpongkeepaliveLong),
new Duplicati.Library.Interface.CommandLineArgument(DISABLE_UPDATE_CHECK_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.DisableupdatecheckShort, Strings.Program.DisableupdatecheckLong),
new Duplicati.Library.Interface.CommandLineArgument("log-retention", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Timespan, Strings.Program.LogretentionShort, Strings.Program.LogretentionLong, DEFAULT_LOG_RETENTION),
new Duplicati.Library.Interface.CommandLineArgument(DataFolderManager.SERVER_DATAFOLDER_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.Program.ServerdatafolderShort, Strings.Program.ServerdatafolderLong(DataFolderManager.DATAFOLDER_ENV_NAME), DataFolderManager.DATAFOLDER),
new Duplicati.Library.Interface.CommandLineArgument(DISABLE_DB_ENCRYPTION_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.DisabledbencryptionShort, Strings.Program.DisabledbencryptionLong),
Duplicati/Server/webroot/ngax/templates/settings.html
+1 -1
View File
@@ -138,7 +138,7 @@
<div class="input checkbox">
<input type="radio" class="checkbox" id="updateChannelBeta" value="beta" name="updateChannel" ng-model="updateChannel" />
<label for="updateChannelBeta" translate>Beta</label>
<div class="sublabel" translate>Try out the new features that we are working on. Currently the most stable version available. Test Restore data before using this in production environments.</div>
<div class="sublabel" translate>Try out the new features that we are working on. Test Backup &amp; Restore before using this in production environments.</div>
</div>
<div class="input checkbox">
<input type="radio" class="checkbox" id="updateChannelExperimental" value="experimental" name="updateChannel" ng-model="updateChannel" />
Duplicati/Tools/Duplicati.Tools.csproj
+3 -3
View File
@@ -5,7 +5,7 @@
<OutputType>Library</OutputType>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
</PropertyGroup>
<ItemGroup>
<Content Include="..\..\Tools\Verification\DuplicatiVerify.ps1">
<Link>utility-scripts\DuplicatiVerify.ps1</Link>
@@ -24,12 +24,12 @@
<Link>utility-scripts\pause-resume.py</Link>
</None>
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.DotNet.Analyzers.Compatibility" Version="0.2.12-alpha">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup>
</Project>
Duplicati/UnitTest/Duplicati.UnitTest.csproj
+4 -3
View File
@@ -8,7 +8,7 @@
<RootNamespace>Duplicati.UnitTest</RootNamespace>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="nunit" Version="3.13.3" />
<PackageReference Include="NUnit3TestAdapter" Version="4.2.1" />
@@ -35,14 +35,15 @@
<ProjectReference Include="..\CommandLine\RecoveryTool\Duplicati.CommandLine.RecoveryTool.csproj" />
<ProjectReference Include="..\Library\UsageReporter\Duplicati.Library.UsageReporter.csproj" />
<ProjectReference Include="..\Tools\Duplicati.Tools.csproj" />
<ProjectReference Include="..\..\Tools\RemoteSynchronization\RemoteSynchronization.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.DotNet.Analyzers.Compatibility" Version="0.2.12-alpha">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup>
</Project>
Duplicati/UnitTest/ToolTests.cs
+529
View File
@@ -0,0 +1,529 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Main;
using NUnit.Framework;
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
namespace Duplicati.UnitTest
{
/// <summary>
/// Tests the tools.
/// </summary>
public class ToolTests : BasicSetupHelper
{
/// <summary>
/// Tests that the remote synchronization tool doesn't do anything when the dry run option is used.
/// </summary>
[Test]
[Category("Tools")]
public void TestDryRun()
{
var l1 = Path.Combine(TARGETFOLDER, "l1");
var l2 = Path.Combine(TARGETFOLDER, "l2");
Directory.CreateDirectory(l1);
Directory.CreateDirectory(l2);
GenerateTestData(l1, 5, 0, 0, 1024).Wait();
var args = new string[] { $"file://{l1}", $"file://{l2}", "--confirm", "--dry-run" };
var async_call = RemoteSynchronization.Program.Main(args);
var return_code = async_call.ConfigureAwait(false).GetAwaiter().GetResult();
Assert.AreEqual(0, return_code, "Remote synchronization tool did not return 0.");
Assert.IsFalse(DirectoriesAndContentsAreEqual(l1, l2), "Synchronized directories are equal");
Assert.IsTrue(!Directory.EnumerateFiles(l2).Any(), "Destination directory is not empty");
}
/// <summary>
/// Tests that the remote synchronization tool works with an empty source to an empty destination.
/// </summary>
[Test]
[Category("Tools")]
public void TestEmptySourceAndDestination()
{
var l1 = Path.Combine(TARGETFOLDER, "empty_src");
var l2 = Path.Combine(TARGETFOLDER, "l2");
Directory.CreateDirectory(l1);
Directory.CreateDirectory(l2);
var args = new string[] { $"file://{l1}", $"file://{l2}", "--confirm" };
var async_call = RemoteSynchronization.Program.Main(args);
var return_code = async_call.ConfigureAwait(false).GetAwaiter().GetResult();
Assert.AreEqual(0, return_code, "Remote synchronization tool did not return 0.");
Assert.IsTrue(DirectoriesAndContentsAreEqual(l1, l2), "Synchronized directories are not equal");
}
/// <summary>
/// Test that remote synchronizing an empty source to a non-empty destination deletes the destination files.
/// </summary>
[Test]
[Category("Tools")]
public void TestEmptySourceDeletesDestination()
{
var l1 = Path.Combine(TARGETFOLDER, "empty_src");
var l2 = Path.Combine(TARGETFOLDER, "l2");
Directory.CreateDirectory(l1);
Directory.CreateDirectory(l2);
GenerateTestData(l2, 5, 0, 0, 1024).Wait();
var args = new string[] { $"file://{l1}", $"file://{l2}", "--confirm" };
var async_call = RemoteSynchronization.Program.Main(args);
var return_code = async_call.ConfigureAwait(false).GetAwaiter().GetResult();
Assert.AreEqual(0, return_code, "Remote synchronization tool did not return 0.");
Assert.IsTrue(DirectoriesAndContentsAreEqual(l1, l2), "Synchronized directories are not equal");
}
/// <summary>
/// Test that remote synchronizing an empty source to a non-empty destination renames the destination files when the `--retention` option is used.
/// </summary>
[Test]
[Category("Tools")]
public void TestEmptySourceRenamesDestination()
{
var l1 = Path.Combine(TARGETFOLDER, "empty_src");
var l2 = Path.Combine(TARGETFOLDER, "l2");
Directory.CreateDirectory(l1);
Directory.CreateDirectory(l2);
GenerateTestData(l2, 5, 0, 0, 1024).Wait();
var filelist = Directory.EnumerateFiles(l2).ToList();
var files = filelist.Select(x => File.ReadAllBytes(x)).ToList();
var args = new string[] { $"file://{l1}", $"file://{l2}", "--confirm", "--retention" };
var async_call = RemoteSynchronization.Program.Main(args);
var return_code = async_call.ConfigureAwait(false).GetAwaiter().GetResult();
Assert.AreEqual(0, return_code, "Remote synchronization tool did not return 0.");
var newfilelist = Directory.EnumerateFiles(l2).ToList();
foreach (var (name, contents) in filelist.Zip(files))
{
var filename = Path.GetFileName(name);
var newcontents = File.ReadAllBytes(newfilelist.FirstOrDefault(x => x.EndsWith(filename)));
Assert.AreEqual(contents, newcontents, "File contents are not equal");
}
}
/// <summary>
/// Tests passing all arguments to the main method of the remote synchronization tool.
/// </summary>
[Test]
public void TestMainMethodParsesArgumentsCorrectly()
{
string[][] testCases =
{
["source", "destination", "--parse-arguments-only"],
["source", "destination", "--parse-arguments-only", "--dry-run"],
["source", "destination", "--parse-arguments-only", "--force"],
["source", "destination", "--parse-arguments-only", "--dry-run", "--force"],
["source", "destination", "--parse-arguments-only", "--verify-contents"],
["source", "destination", "--parse-arguments-only", "--verify-get-after-put"],
["source", "destination", "--parse-arguments-only", "--retry", "3"],
["source", "destination", "--parse-arguments-only", "--log-level", "Debug"],
["source", "destination", "--parse-arguments-only", "--log-level", "Information"],
["source", "destination", "--parse-arguments-only", "--log-level", "Profiling"],
["source", "destination", "--parse-arguments-only", "--log-level", "Verbose"],
["source", "destination", "--parse-arguments-only", "--log-file", "somefile.log"],
["source", "destination", "--parse-arguments-only", "--progress"],
["source", "destination", "--parse-arguments-only", "--retention"],
["source", "destination", "--parse-arguments-only", "--confirm"],
["source", "destination", "--parse-arguments-only", "--global-options", "someglobalkey=someglobalvalue", "anotherglobalkey=anotherglobalvalue"],
["source", "destination", "--parse-arguments-only", "--src-options", "somesrckey=somesrcvalue", "anothersrckey=anothersrcvalue"],
["source", "destination", "--parse-arguments-only", "--dst-options", "somedstkey=somedstvalue", "anotherdstkey=anotherdstvalue"],
[
"source", "destination", "--parse-arguments-only",
"--dry-run", "--force", "--verify-contents", "--retry", "3", "--log-level", "Debug", "--log-file", "somefile.log", "--progress", "--retention", "--confirm",
"--global-options", "someglobalkey=someglobalvalue", "anotherglobalkey=anotherglobalvalue",
"--src-options", "somesrckey=somesrcvalue", "anothersrckey=anothersrcvalue",
"--dst-options", "somedstkey=somedstvalue", "anotherdstkey=anotherdstvalue"
],
};
foreach (var args in testCases)
{
int result = RemoteSynchronization.Program.Main(args).ConfigureAwait(false).GetAwaiter().GetResult();
Assert.AreEqual(0, result, $"Failed for args: {string.Join(" ", args)}");
}
int failed_result = RemoteSynchronization.Program.Main(["source", "destination", "--bogus-option"]).ConfigureAwait(false).GetAwaiter().GetResult();
Assert.AreEqual(1, failed_result, "Invalid option did not return 1");
}
/// <summary>
/// Tests the original inded use of the remote synchronization tool on an empty destination.
/// </summary>
[Test]
[Category("Tools")]
public void TestRemoteSynchronization()
{
var l1 = Path.Combine(TARGETFOLDER, "l1");
var l2 = Path.Combine(TARGETFOLDER, "l2");
var l1r = Path.Combine(RESTOREFOLDER, "l1_restore");
var l2r = Path.Combine(RESTOREFOLDER, "l2_restore");
var options = TestOptions;
var now = DateTime.Now;
GenerateTestData(DATAFOLDER, 10, 3, 3, 1024 * 1024).Wait();
Console.WriteLine($"Generated test data in {DATAFOLDER} in {DateTime.Now - now}");
// Create the directories if they do not exist
foreach (var p in new string[] { l1, l2, l1r, l2r })
{
if (!SystemIO.IO_OS.DirectoryExists(p))
SystemIO.IO_OS.DirectoryCreate(p);
}
// Backup the first level
using (var c = new Controller($"file://{l1}", options, null))
{
now = DateTime.Now;
var results = c.Backup([DATAFOLDER]);
Assert.AreEqual(0, results.Errors.Count());
Assert.AreEqual(0, results.Warnings.Count());
Console.WriteLine($"Backed up {results.AddedFiles} files to {l1} in {DateTime.Now - now}");
}
// Call the tool
now = DateTime.Now;
var exe = RemoteSynchronization.Program.Main;
string[] args = [
$"file://{l1}", $"file://{l2}",
"--global-options", ..options.Select(x => $"{x.Key}={x.Value}"),
// Pass along multi token options to test that the parser won't fail
"--src-options", "ssh-accept-any-fingerprints=true", "ssh-keyfile=/path/to/keyfile",
"--dst-options", "some-other-key=value", "another-key=value2",
"--confirm"
];
var async_call = exe(args);
async_call.Wait();
Assert.AreEqual(0, async_call.Result, "Remote synchronization tool did not return 0.");
Console.WriteLine($"Remote synchronization tool returned 0 in {DateTime.Now - now}");
// Verify that the directories are equal
Assert.IsTrue(DirectoriesAndContentsAreEqual(l1, l2), "Synchronized directories are not equal");
// Try to restore the first level
options["restore-path"] = l1r;
using (var c = new Controller($"file://{l1}", options, null))
{
now = DateTime.Now;
var results = c.Restore([Path.Combine(DATAFOLDER, "*")]);
Assert.AreEqual(0, results.Errors.Count());
Assert.AreEqual(0, results.Warnings.Count());
Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]} in {DateTime.Now - now}");
}
Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l1r), "Restored first level files is not equal to original files");
// Try to restore the second level
options["restore-path"] = l2r;
using (var c = new Controller($"file://{l2}", options, null))
{
now = DateTime.Now;
var results = c.Restore([]);
Assert.AreEqual(0, results.Errors.Count());
Assert.AreEqual(0, results.Warnings.Count());
Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]} in {DateTime.Now - now}");
}
Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l2r), "Restored second level files is not equal to original files");
// Delete the l2r directory
SystemIO.IO_OS.DirectoryDelete(l2r, true);
// Delete one file from the l2 backup
var files = Directory.EnumerateFiles(l2).ToList();
File.Delete(files.First());
// Check that the restore fails
options["restore-path"] = l2r;
using (var c = new Controller($"file://{l2}", options, null))
{
now = DateTime.Now;
try
{
var results = c.Restore([]);
}
catch (RemoteListVerificationException)
{
Console.WriteLine($"Failed (as expected) to restore files to {options["restore-path"]} in {DateTime.Now - now}");
}
}
// Run the tool again to copy the missing file
now = DateTime.Now;
async_call = exe(args);
async_call.Wait();
Assert.AreEqual(0, async_call.Result, "Remote synchronization tool did not return 0.");
Console.WriteLine($"Remote synchronization tool returned 0 in {DateTime.Now - now}");
// Try to restore the second level again
options["restore-path"] = l2r;
using (var c = new Controller($"file://{l2}", options, null))
{
now = DateTime.Now;
var results = c.Restore([]);
Assert.AreEqual(0, results.Errors.Count());
Assert.AreEqual(0, results.Warnings.Count());
Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]} in {DateTime.Now - now}");
}
Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l2r), "Restored second level files is not equal to original files");
// Add some more files to the source
GenerateTestData(Path.Combine(DATAFOLDER, "brand_new_files"), 5, 2, 2, 1024 * 1024).Wait();
// Backup the new files to l1
using (var c = new Controller($"file://{l1}", options, null))
{
now = DateTime.Now;
var results = c.Backup([DATAFOLDER]);
Assert.AreEqual(0, results.Errors.Count());
Assert.AreEqual(0, results.Warnings.Count());
Console.WriteLine($"Backed up {results.AddedFiles} files to {l1} in {DateTime.Now - now}");
}
// Run the tool again to copy the new files
now = DateTime.Now;
async_call = exe(args);
async_call.Wait();
Assert.AreEqual(0, async_call.Result, "Remote synchronization tool did not return 0.");
Console.WriteLine($"Remote synchronization tool returned 0 in {DateTime.Now - now}");
// Try to restore the second level again
options["restore-path"] = l2r;
using (var c = new Controller($"file://{l2}", options, null))
{
now = DateTime.Now;
var results = c.Restore([]);
Assert.AreEqual(0, results.Errors.Count());
Assert.AreEqual(0, results.Warnings.Count());
Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]} in {DateTime.Now - now}");
}
Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l2r), "Restored second level files is not equal to original files");
// Delete the l2r directory
SystemIO.IO_OS.DirectoryDelete(l2r, true);
// Remove a directory from the source
SystemIO.IO_OS.DirectoryDelete(Path.Combine(DATAFOLDER, "dir_0"), true);
// Backup the new files to l1
using (var c = new Controller($"file://{l1}", options, null))
{
now = DateTime.Now;
var results = c.Backup([DATAFOLDER]);
Assert.AreEqual(0, results.Errors.Count());
Assert.AreEqual(0, results.Warnings.Count());
Console.WriteLine($"Backed up {results.AddedFiles} files to {l1} in {DateTime.Now - now}");
}
// Compact the backup
using (var c = new Controller($"file://{l1}", options, null))
{
now = DateTime.Now;
var results = c.Compact();
Assert.AreEqual(0, results.Errors.Count());
Assert.AreEqual(0, results.Warnings.Count());
Console.WriteLine($"Compacted backup in {DateTime.Now - now}");
}
// Run the tool again to copy the new files
now = DateTime.Now;
async_call = exe(args);
async_call.Wait();
Assert.AreEqual(0, async_call.Result, "Remote synchronization tool did not return 0.");
// Try to restore the second level again
options["restore-path"] = l2r;
using (var c = new Controller($"file://{l2}", options, null))
{
now = DateTime.Now;
var results = c.Restore([]);
Assert.AreEqual(0, results.Errors.Count());
Assert.AreEqual(0, results.Warnings.Count());
Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]} in {DateTime.Now - now}");
}
Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l2r), "Restored second level files is not equal to original files");
// Delete the l2r directory
SystemIO.IO_OS.DirectoryDelete(l2r, true);
// Perform a forced synchronization with retention to check that retention doesn't break a restore
now = DateTime.Now;
async_call = exe([.. args, "--force", "--retention"]);
async_call.Wait();
Assert.AreEqual(0, async_call.Result, "Remote synchronization tool did not return 0.");
// Try to restore the second level again
options["restore-path"] = l2r;
using (var c = new Controller($"file://{l2}", options, null))
{
now = DateTime.Now;
var results = c.Restore([]);
Assert.AreEqual(0, results.Errors.Count());
Assert.AreEqual(0, results.Warnings.Count());
Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]} in {DateTime.Now - now}");
}
Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l2r), "Restored second level files is not equal to original files");
}
/// <summary>
/// Tests that the remote synchronization tool verifies the contents of the files.
/// </summary>
[Test]
[Category("Tools")]
public void TestVerifies()
{
var l1 = Path.Combine(TARGETFOLDER, "l1");
var l2 = Path.Combine(TARGETFOLDER, "l2");
Directory.CreateDirectory(l1);
Directory.CreateDirectory(l2);
GenerateTestData(l1, 5, 0, 0, 1024).Wait();
var filenames = Directory.EnumerateFiles(l1).Take(2).ToList();
var first_file = filenames.First();
var second_file = filenames.Skip(1).First();
File.Copy(first_file, Path.Combine(l2, Path.GetFileName(first_file)));
File.Copy(second_file, Path.Combine(l2, Path.GetFileName(second_file)));
// Touch the first file to give it a different timestamp
File.SetLastWriteTime(first_file, DateTime.Now.AddMinutes(1));
// Modify the second file to give it different contents, but the same size
var second_file_contents = File.ReadAllBytes(second_file);
second_file_contents[0] = (byte)(second_file_contents[0] + 1);
File.WriteAllBytes(second_file, second_file_contents);
var args = new string[] { $"file://{l1}", $"file://{l2}", "--confirm", "--verify-contents", "--verify-get-after-put" };
var async_call = RemoteSynchronization.Program.Main(args);
var return_code = async_call.ConfigureAwait(false).GetAwaiter().GetResult();
Assert.AreEqual(0, return_code, "Remote synchronization tool did not return 0.");
Assert.IsTrue(DirectoriesAndContentsAreEqual(l1, l2), "Synchronized directories are not equal");
}
//
// Helper methods
//
/// <summary>
/// Compares two directories and their contents.
/// </summary>
/// <param name="d1">The first directory.</param>
/// <param name="d2">The second directory.</param>
/// <returns>`true` if the two directories contain exactly the same files and if the contents of the files are equivalent. `false` otherwise.</returns>
public static bool DirectoriesAndContentsAreEqual(string d1, string d2)
{
// Recursively get the files in the two directories
var f1s = Directory.EnumerateFiles(d1, "*", SearchOption.AllDirectories).Select(x => x[(d1.Length + 1)..]).OrderDescending();
var f2s = Directory.EnumerateFiles(d2, "*", SearchOption.AllDirectories).Select(x => x[(d2.Length + 1)..]).OrderDescending();
// If the two directories do not contain the same files, return false
if (!f1s.SequenceEqual(f2s))
return false;
// Check that each file pair exist and have the same content
foreach (var f1 in f1s)
{
var f1full = Path.Combine(d1, f1);
if (!File.Exists(f1full))
return false;
var f2full = Path.Combine(d2, f1);
if (!File.Exists(f2full))
return false;
var c1 = File.ReadAllText(f1full);
var c2 = File.ReadAllText(f2full);
if (!c1.SequenceEqual(c2))
return false;
}
return true;
}
/// <summary>
/// Generates test data in the specified directory.
/// </summary>
/// <param name="dir">The directory to fill with the generated data.</param>
/// <param name="n_files">How many files the directory should have.</param>
/// <param name="n_dirs">How many subdirectories the directory should have.</param>
/// <param name="n_levels">How deep the number of subdirectories within subdirectories should go.</param>
/// <param name="max_file_size">The maximum size of the files to generate.</param>
public static async Task GenerateTestData(string dir, int n_files, int n_dirs, int n_levels, int max_file_size)
{
if (!SystemIO.IO_OS.DirectoryExists(dir))
SystemIO.IO_OS.DirectoryCreate(dir);
var fs = Enumerable.Range(0, n_files)
.Select(i => GenerateTestFile(dir, i, max_file_size));
var ds = n_levels > 0 ?
Enumerable.Range(0, n_dirs)
.Select(i =>
{
var subdir = Path.Combine(dir, $"dir_{i}");
return GenerateTestData(subdir, n_files, n_dirs, n_levels - 1, max_file_size);
})
: [];
await Task.WhenAll([.. fs, .. ds]);
}
public static async Task GenerateTestFile(string dir, int i, int max_file_size)
{
var rnd = new Random();
var file = Path.Combine(dir, $"file_{i}.txt");
var size = rnd.Next(1, max_file_size);
var data = new byte[size];
rnd.NextBytes(data);
await File.WriteAllBytesAsync(file, data);
}
}
}
ReleaseBuilder/Build/Command.Upload.cs
+24 -4
View File
@@ -106,8 +106,9 @@ public static partial class Command
/// </summary>
/// <param name="files">The files to upload</param>
/// <param name="rtcfg">The runtime configuration</param>
/// <param name="propagateTo">The release channels to propagate to</param>
/// <returns>An awaitable task</returns>
public static async Task UploadToS3(IEnumerable<UploadFile> files, RuntimeConfig rtcfg)
public static async Task UploadToS3(IEnumerable<UploadFile> files, RuntimeConfig rtcfg, IEnumerable<ReleaseChannel> propagateTo)
{
var totalSize = files.Sum(f => new FileInfo(f.Path).Length);
Console.WriteLine($"Uploading {files.Count()} files ({Duplicati.Library.Utility.Utility.FormatSizeString(totalSize)}) to S3...");
@@ -141,6 +142,22 @@ public static partial class Command
size += filesize;
Console.WriteLine($"{size / (double)totalSize * 100:F1}% - Uploaded {file.Name} ({Duplicati.Library.Utility.Utility.FormatSizeString(filesize)})");
}
foreach (var channel in propagateTo)
{
if (channel == rtcfg.ReleaseInfo.Channel)
continue;
var target = $"{channel.ToString().ToLowerInvariant()}/latest-v2.manifest";
var source = $"{rtcfg.ReleaseInfo.Channel.ToString().ToLowerInvariant()}/latest-v2.manifest";
await Duplicati.Library.Utility.RetryHelper.Retry(() =>
client.CopyObjectAsync(
Program.Configuration.ConfigFiles.AwsUploadBucket,
source,
Program.Configuration.ConfigFiles.AwsUploadBucket,
target
), 3, TimeSpan.FromSeconds(1), CancellationToken.None);
}
}
/// <summary>
@@ -151,6 +168,9 @@ public static partial class Command
/// <returns>An awaitable task</returns>
public static async Task UploadToGithub(IEnumerable<UploadFile> files, RuntimeConfig rtcfg)
{
var commithash = (await ProcessHelper.ExecuteWithOutput(["git", "rev-parse", "HEAD"]))?.Trim()
?? throw new Exception("Failed to get the current commit hash");
using var httpClient = new HttpClient();
var totalSize = files.Sum(f => new FileInfo(f.Path).Length);
Console.WriteLine($"Uploading {files.Count()} files ({Duplicati.Library.Utility.Utility.FormatSizeString(totalSize)}) to Github...");
@@ -166,7 +186,7 @@ public static partial class Command
request.Headers.Add("User-Agent", "Duplicati Release Builder v1");
request.Content = JsonContent.Create(new GhReleaseInfo(
tag_name: $"v{rtcfg.ReleaseInfo.ReleaseName}",
target_commitish: "master",
target_commitish: commithash,
name: $"v{rtcfg.ReleaseInfo.ReleaseName}",
body: rtcfg.ChangelogNews,
draft: false,
@@ -211,7 +231,7 @@ public static partial class Command
/// </summary>
/// <param name="rtcfg">The runtime configuration</param>
/// <returns>An awaitable task</returns>
public static async Task ReloadUpdateServer(RuntimeConfig rtcfg)
public static async Task ReloadUpdateServer(RuntimeConfig rtcfg, IEnumerable<ReleaseChannel> propagateTo)
{
using var client = new HttpClient();
var req = new HttpRequestMessage(HttpMethod.Post, "https://updates.duplicati.com/reload");
@@ -222,7 +242,7 @@ public static partial class Command
$"{rtcfg.ReleaseInfo.Channel.ToString().ToLowerInvariant()}/latest-v2.json",
$"{rtcfg.ReleaseInfo.Channel.ToString().ToLowerInvariant()}/latest-v2.js",
$"{rtcfg.ReleaseInfo.Channel.ToString().ToLowerInvariant()}/latest-v2.manifest",
});
}.Concat(propagateTo.Select(c => $"{c.ToString().ToLowerInvariant()}/latest-v2.manifest")));
var response = await client.SendAsync(req);
response.EnsureSuccessStatusCode();
ReleaseBuilder/Build/Command.cs
+20 -4
View File
@@ -299,6 +299,12 @@ public static partial class Command
getDefaultValue: () => false
);
var propagateReleaseOption = new Option<bool>(
name: "--propagate-release",
description: "Propagate the release to the next channel",
getDefaultValue: () => false
);
var command = new System.CommandLine.Command("build", "Builds the packages for a release") {
gitStashPushOption,
releaseChannelArgument,
@@ -322,7 +328,8 @@ public static partial class Command
disableUpdateServerReloadOption,
disableDiscordAnnounceOption,
useHostedBuildsOption,
resumeFromUploadOption
resumeFromUploadOption,
propagateReleaseOption
};
command.Handler = CommandHandler.Create<CommandInput>(DoBuild);
@@ -355,6 +362,7 @@ public static partial class Command
/// <param name="DisableDiscordAnnounce">If forum posting should be disabled</param>
/// <param name="UseHostedBuilds">If hosted builds should be used</param>
/// <param name="ResumeFromUpload">If the process should resume from the upload step</param>
/// <param name="PropagateRelease">If the release should be propagated to the next channel</param>
record CommandInput(
PackageTarget[] Targets,
DirectoryInfo BuildPath,
@@ -378,7 +386,8 @@ public static partial class Command
bool DisableUpdateServerReload,
bool DisableDiscordAnnounce,
bool UseHostedBuilds,
bool ResumeFromUpload
bool ResumeFromUpload,
bool PropagateRelease
);
static async Task DoBuild(CommandInput input)
@@ -619,6 +628,13 @@ public static partial class Command
if (input.GitStashPush && !input.ResumeFromUpload)
await GitPush.TagAndPush(baseDir, releaseInfo);
// Propagate the release to the next channel, if selected
var nextChannels = new[] { ReleaseChannel.Experimental, ReleaseChannel.Beta, ReleaseChannel.Stable }
.Where(x => x > releaseInfo.Channel)
.Where(x => rtcfg.ReleaseInfo.Channel != ReleaseChannel.Canary)
.Where(x => input.PropagateRelease)
.ToArray();
if (rtcfg.UseS3Upload || rtcfg.UseGithubUpload)
{
Console.WriteLine("Build completed, uploading packages ...");
@@ -636,7 +652,7 @@ public static partial class Command
.Append(new Upload.UploadFile(packageJson, Path.GetFileName(packageJson)))
.Append(new Upload.UploadFile(packageJson, $"latest-v2-{releaseInfo.Version}.json"));
await Upload.UploadToS3(uploads, rtcfg);
await Upload.UploadToS3(uploads, rtcfg, nextChannels);
}
if (rtcfg.UseGithubUpload)
@@ -657,7 +673,7 @@ public static partial class Command
if (rtcfg.UseUpdateServerReload)
{
Console.WriteLine("Release completed, reloading update server ...");
await Upload.ReloadUpdateServer(rtcfg);
await Upload.ReloadUpdateServer(rtcfg, nextChannels);
}
if (rtcfg.UseForumPosting)
Tools/RemoteSynchronization/Program.cs
+826
View File
@@ -0,0 +1,826 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using System;
using System.Collections.Generic;
using System.CommandLine;
using System.CommandLine.NamingConventionBinder;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace RemoteSynchronization
{
/// <summary>
/// Remote synchronization tool.
/// </summary>
public class Program
{
/// <summary>
/// Global configuration for the tool. Should be set after parsing the commandline arguments.
/// </summary>
private sealed record Config
(
// Arguments
string Src,
string Dst,
// Options
bool Confirm,
bool DryRun,
List<string> DstOptions,
bool Force,
List<string> GlobalOptions,
string LogFile,
string LogLevel,
bool ParseArgumentsOnly,
bool Progress,
bool Retention,
int Retry,
List<string> SrcOptions,
bool VerifyContents,
bool VerifyGetAfterPut
);
/// <summary>
/// The log tag for this tool.
/// </summary>
private static readonly string LOGTAG = Duplicati.Library.Logging.Log.LogTagFromType<Program>();
/// <summary>
/// Main entry point for the tool.
/// </summary>
/// <param name="args">The commandline arguments</param>
/// <returns>0 on success, -1 on abort, and the number of errors encountered otherwise.</returns>
public static async Task<int> Main(string[] args)
{
var arg_src = new Argument<string>(name: "backend_src", description: "The source backend string");
var arg_dst = new Argument<string>(name: "backend_dst", description: "The destination backend string");
var root_cmd = new RootCommand(@"Remote Synchronization Tool
This tool synchronizes two remote backends. The tool assumes that the intent is
to have the destination match the source.
If the destination has files that are not in the source, they will be deleted
(or renamed if the retention option is set).
If the destination has files that are in the source but are different in size or
has a newer timestamp, they will be overwritten by the source files.
If the force option is set, the destination will be overwritten by the source,
regardless of the state of the files. It will also skip the initial comparison,
and delete (or rename) all files in the destination.
If the verify option is set, the files will be downloaded and compared after
uploading to ensure that the files are correct. Files that already exist in the
destination will be verified before being overwritten (if they seemingly match).
")
{
arg_src,
arg_dst,
new Option<bool>(aliases: ["--confirm", "--yes", "-y"], description: "Automatically confirm the operation", getDefaultValue: () => false),
new Option<bool>(aliases: ["--dry-run", "-d"], description: "Do not actually write or delete files. If not set here, the global options will be checked", getDefaultValue: () => false),
OptionWithMultipleTokens(aliases: ["--dst-options"], description: "Options for the destination backend. Each option is a key-value pair separated by an equals sign, e.g. --dst-options key1=value1 key2=value2 [default: empty]", getDefaultValue: () => []),
new Option<bool>(aliases: ["--force", "-f"], description: "Force the synchronization", getDefaultValue: () => false),
OptionWithMultipleTokens(aliases: ["--global-options"], description: "Global options all backends. May be overridden by backend specific options (src-options, dst-options). Each option is a key-value pair separated by an equals sign, e.g. --global-options key1=value1 key2=value2 [default: empty]", getDefaultValue: () => []),
new Option<string>(aliases: ["--log-file"], description: "The log file to write to. If not set here, global options will be checked [default: \"\"]", getDefaultValue: () => "") { Arity = ArgumentArity.ExactlyOne },
new Option<string>(aliases: ["--log-level"], description: "The log level to use. If not set here, global options will be checked", getDefaultValue: () => "Information") { Arity = ArgumentArity.ExactlyOne },
new Option<bool>(aliases: ["--parse-arguments-only"], description: "Only parse the arguments and then exit", getDefaultValue: () => false),
new Option<bool>(aliases: ["--progress"], description: "Print progress to STDOUT", getDefaultValue: () => false),
new Option<bool>(aliases: ["--retention"], description: "Toggles whether to keep old files. Any deletes will be renames instead", getDefaultValue: () => false),
new Option<int>(aliases: ["--retry"], description: "Number of times to retry on errors", getDefaultValue: () => 3) { Arity = ArgumentArity.ExactlyOne },
OptionWithMultipleTokens(aliases: ["--src-options"], description: "Options for the source backend. Each option is a key-value pair separated by an equals sign, e.g. --src-options key1=value1 key2=value2 [default: empty]", getDefaultValue: () => []),
new Option<bool>(aliases: ["--verify-contents"], description: "Verify the contents of the files to decide whether the pre-existing destination files should be overwritten", getDefaultValue: () => false),
new Option<bool>(aliases: ["--verify-get-after-put"], description: "Verify the files after uploading them to ensure that they were uploaded correctly", getDefaultValue: () => false),
};
root_cmd.Handler = CommandHandler.Create((string backend_src, string backend_dst, Config config) =>
{
var config_with_args = config with { Dst = backend_dst, Src = backend_src };
return Run(config_with_args);
});
return await root_cmd.InvokeAsync(args);
}
/// <summary>
/// The main logic of the tool.
/// </summary>
/// <param name="src">The connection string for the source backend.</param>
/// <param name="dst">The connection string for the destination backend.</param>
/// <param name="options">Various options for the tool</param>
/// <returns>The return code for the main entry; 0 on success.</returns>
private static async Task<int> Run(Config config)
{
// Unpack and parse the multi token options
Dictionary<string, string> global_options = config.GlobalOptions
.Select(x => x.Split("="))
.ToDictionary(x => x[0], x => x[1]);
// Parse the log level
var log_level_parsed = Enum.TryParse<Duplicati.Library.Logging.LogMessageType>(config.LogLevel, true, out var log_level_enum);
log_level_enum = log_level_parsed ? log_level_enum : Duplicati.Library.Logging.LogMessageType.Information;
using var console_sink = new Duplicati.CommandLine.ConsoleOutput(Console.Out, global_options);
using var multi_sink = new Duplicati.Library.Main.ControllerMultiLogTarget(console_sink, log_level_enum, null);
// Parse the log file
// The log file sink doesn't have to be disposed, as the multi_sink will take care of it
Duplicati.Library.Logging.StreamLogDestination? log_file_sink = null;
if (!string.IsNullOrEmpty(config.LogFile))
{
string log_file_dir = SystemIO.IO_OS.PathGetDirectoryName(config.LogFile);
if (!string.IsNullOrEmpty(log_file_dir) && !SystemIO.IO_OS.DirectoryExists(log_file_dir))
SystemIO.IO_OS.DirectoryCreate(log_file_dir);
log_file_sink = new Duplicati.Library.Logging.StreamLogDestination(config.LogFile);
}
multi_sink.AddTarget(log_file_sink, log_level_enum, null);
// Start the logging scope
using var _ = Duplicati.Library.Logging.Log.StartScope(multi_sink, log_level_enum);
Dictionary<string, string> src_opts = config.SrcOptions
.Select(x => x.Split("="))
.ToDictionary(x => x[0], x => x[1]);
Dictionary<string, string> dst_opts = config.DstOptions
.Select(x => x.Split("="))
.ToDictionary(x => x[0], x => x[1]);
// Merge the global options into the source and destination options. The global options will be overridden by the source and destination options.
foreach (var x in global_options)
{
if (!src_opts.ContainsKey(x.Key))
src_opts[x.Key] = x.Value;
if (!dst_opts.ContainsKey(x.Key))
dst_opts[x.Key] = x.Value;
}
// Check if we only had to parse the arguments
if (config.ParseArgumentsOnly)
{
Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Arguments parsed successfully; exiting.");
return 0;
}
// Load the backends
static Exception throw_message(string target)
{
var message = $"The {target} backend does not support streaming operations.";
var ex = new Exception(message);
Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", ex, message);
return ex;
}
using var b1 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(config.Src, src_opts);
var b1s = b1 as IStreamingBackend ?? throw throw_message("source");
using var b2 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(config.Dst, dst_opts);
var b2s = b2 as IStreamingBackend ?? throw throw_message("destination");
// Prepare the operations
var (to_copy, to_delete, to_verify) = PrepareFileLists(b1s, b2s, config);
// Verify the files if requested. If the files are not verified, they will be deleted and copied again.
long verified = 0, failed_verify = 0;
if (config.VerifyContents)
{
// As this is a potentially slow operation, ask for confirmation of the verification)
if (!config.Confirm)
{
Console.WriteLine($"This will verify {to_verify.Count()} files before copying them. Do you want to continue? [y/N]");
var response = Console.ReadLine();
if (!response?.Equals("y", StringComparison.CurrentCultureIgnoreCase) ?? true)
{
Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Aborted");
return -1;
}
}
var not_verified = await VerifyAsync(b1s, b2s, to_verify, config);
failed_verify = not_verified.Count();
verified = to_verify.Count() - failed_verify;
if (not_verified.Any())
{
Duplicati.Library.Logging.Log.WriteWarningMessage(LOGTAG, "rsync", null,
"{0} files failed verification. They will be deleted and copied again.",
not_verified.Count());
to_delete = to_delete.Concat(not_verified);
to_copy = to_copy.Concat(not_verified);
}
}
Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync",
"The remote synchronization plan is to {0} {1} files from {2}, then copy {3} files from {4} to {2}.",
config.Retention ? "rename" : "delete",
to_delete.Count(), b2s.DisplayName, to_copy.Count(), b1s.DisplayName);
// As this is a potentially destructive operation, ask for confirmation
if (!config.Confirm)
{
var delete_rename = config.Retention ? "Rename" : "Delete";
Console.WriteLine($"This will perform the following actions (in order):");
Console.WriteLine($" - {delete_rename} {to_delete.Count()} files from {config.Dst}");
Console.WriteLine($" - Copy {to_copy.Count()} files from {config.Src} to {config.Dst}");
Console.WriteLine();
Console.WriteLine("Do you want to continue? [y/N]");
var response = Console.ReadLine();
if (!response?.Equals("y", StringComparison.CurrentCultureIgnoreCase) ?? true)
{
Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Aborted");
return -1;
}
}
// Delete or rename the files that are not needed
long renamed = 0, deleted = 0;
if (config.Retention)
{
renamed = await RenameAsync(b2, to_delete, config);
Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync",
"Renamed {0} files in {1}", renamed, b2s.DisplayName);
}
else
{
deleted = await DeleteAsync(b2s, to_delete, config);
Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync",
"Deleted {0} files from {1}", deleted, b2s.DisplayName);
}
// Copy the files
var (copied, copy_errors) = await CopyAsync(b1s, b2s, to_copy, config);
Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync",
"Copied {0} files from {1} to {2}", copied, b1s, b2s);
// If there are still errors, retry a few times
if (copy_errors.Any())
{
Duplicati.Library.Logging.Log.WriteWarningMessage(LOGTAG, "rsync", null,
"Could not copy {0} files.", copy_errors.Count());
if (config.Retry > 0)
{
Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync",
"Retrying {0} more times to copy the {1} files that failed",
config.Retry, copy_errors.Count());
for (int i = 0; i < config.Retry; i++)
{
await Task.Delay(5000); // Wait 5 seconds before retrying
(copied, copy_errors) = await CopyAsync(b1s, b2s, copy_errors, config);
Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync",
"Copied {0} files from {1} to {2}", copied, b1s, b2s);
if (!copy_errors.Any())
break;
}
}
if (copy_errors.Any())
{
Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", null,
"Could not copy {0} files. Not retrying any more.", copy_errors.Count());
return copy_errors.Count();
}
}
// Results reporting
if (verified > 0)
Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync",
"Verified {0} files in {1} that didn't need to be copied",
verified, b2s.DisplayName);
if (failed_verify > 0)
Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync",
"Failed to verify {0} files in {1}, which were then attempted to be copied",
failed_verify, b2s.DisplayName);
if (copied > 0)
Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync",
"Copied {0} files from {1} to {2}", copied, b1s.DisplayName, b2s.DisplayName);
if (deleted > 0)
Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync",
"Deleted {0} files from {1}", deleted, b2s.DisplayName);
if (renamed > 0)
Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync",
"Renamed {0} files in {1}", renamed, b2s.DisplayName);
Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync",
"Remote synchronization completed successfully");
return 0;
}
// TODO have concurrency parameters: uploaders, downloaders
// TODO low memory mode, where things aren't kept in memory. Maybe utilize SQLite?
// TODO For convenience, have the option to launch a "duplicati test" on the destination backend after the synchronization
// TODO Save hash to minimize redownload
// TODO Duplicati Results
/// <summary>
/// Copies the files from one backend to another.
/// The files are copied one by one, and each file is verified after uploading if the verify flag is set.
/// </summary>
/// <param name="b_src">The source backend.</param>
/// <param name="b_dst">The destination backend.</param>
/// <param name="files">The files that will be copied.</param>
/// <param name="config">The parsed configuration for the tool.</param>
/// <returns>A tuple holding the number of succesful copies and a List of the files that failed.</returns>
private static async Task<(long, IEnumerable<IFileEntry>)> CopyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable<IFileEntry> files, Config config)
{
long successful_copies = 0;
List<IFileEntry> errors = [];
using var s_src = new MemoryStream();
using var s_dst = new MemoryStream();
long i = 0, n = files.Count();
var sw_get_src = new System.Diagnostics.Stopwatch();
var sw_put_dst = new System.Diagnostics.Stopwatch();
var sw_get_dst = new System.Diagnostics.Stopwatch();
var sw_get_cmp = new System.Diagnostics.Stopwatch();
foreach (var f in files)
{
if (config.Progress)
Console.Write($"\rCopying: {i}/{n}");
Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync",
"Copying {0} from {1} to {2}", f.Name, b_src.DisplayName, b_dst.DisplayName);
try
{
sw_get_src.Start();
await b_src.GetAsync(f.Name, s_src, CancellationToken.None);
sw_get_src.Stop();
if (config.DryRun)
{
Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync",
"Would write {0} bytes of {1} to {2}",
Duplicati.Library.Utility.Utility.FormatSizeString(s_src.Length),
f.Name, b_dst.DisplayName);
}
else
{
sw_put_dst.Start();
await b_dst.PutAsync(f.Name, s_src, CancellationToken.None);
sw_put_dst.Stop();
if (config.VerifyGetAfterPut)
{
sw_get_dst.Start();
await b_dst.GetAsync(f.Name, s_dst, CancellationToken.None);
sw_get_dst.Stop();
sw_get_cmp.Start();
string? err_string = null;
if (s_src.Length != s_dst.Length)
{
err_string = $"The sizes of the files do not match: {s_src.Length} != {s_dst.Length}.";
}
if (!s_src.ToArray().SequenceEqual(s_dst.ToArray()))
{
err_string = (err_string is null ? "" : err_string + " ") + "The contents of the files do not match.";
}
sw_get_cmp.Stop();
if (err_string is not null)
{
throw new Exception(err_string);
}
}
}
successful_copies++;
}
catch (Exception e)
{
Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e,
"Error copying {0}: {1}", f.Name, e.Message);
errors.Add(f);
}
finally
{
s_src.SetLength(0);
s_dst.SetLength(0);
i++;
// Stop any running timers
sw_get_src.Stop();
sw_put_dst.Stop();
sw_get_dst.Stop();
sw_get_cmp.Stop();
}
}
if (config.Progress)
Console.WriteLine($"\rCopying: {n}/{n}");
Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync",
"Copy | Get source: {0} ms, Put destination: {1} ms, Get destination: {2} ms, Get compare: {3} ms",
TimeSpan.FromMilliseconds(sw_get_src.ElapsedMilliseconds),
TimeSpan.FromMilliseconds(sw_put_dst.ElapsedMilliseconds),
TimeSpan.FromMilliseconds(sw_get_dst.ElapsedMilliseconds),
TimeSpan.FromMilliseconds(sw_get_cmp.ElapsedMilliseconds));
return (successful_copies, errors);
}
/// <summary>
/// Deletes the files from a backend.
/// </summary>
/// <param name="b">The backend to delete the files from.</param>
/// <param name="files">The files to delete.</param>
/// <param name="config">The parsed configuration for the tool.</param>
/// <returns>The number of successful deletions.</returns>
private static async Task<long> DeleteAsync(IStreamingBackend b, IEnumerable<IFileEntry> files, Config config)
{
long successful_deletes = 0;
long i = 0, n = files.Count();
using var timer = new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Delete operation");
foreach (var f in files)
{
if (n > 1 && config.Progress)
{
Console.Write($"\rDeleting: {i}/{n}");
}
Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync",
"Deleting {0} from {1}", f.Name, b.DisplayName);
try
{
if (config.DryRun)
{
Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync",
"Would delete {0} from {1}", f.Name, b.DisplayName);
}
else
{
await b.DeleteAsync(f.Name, CancellationToken.None);
}
successful_deletes++;
}
catch (Exception e)
{
Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e,
"Error deleting {0}: {1}", f.Name, e.Message);
}
i++;
}
if (config.Progress)
Console.WriteLine($"\rDeleting: {n}/{n}");
return successful_deletes;
}
/// <summary>
/// Creates an option that allows multiple tokens and multiple arguments per token.
/// </summary>
/// <param name="aliases">The aliases for the option.</param>
/// <param name="description">The description for the option.</param>
/// <returns>The created option.</returns>
private static Option<List<string>> OptionWithMultipleTokens(string[] aliases, string description, Func<List<string>> getDefaultValue)
{
return new Option<List<string>>(aliases: aliases, description: description, getDefaultValue: getDefaultValue)
{
Arity = ArgumentArity.OneOrMore,
AllowMultipleArgumentsPerToken = true
};
}
/// <summary>
/// Prepares the lists of files to copy, delete and verify.
/// The files to copy are the files that are not in the destination, have a different size or have a more recent modification date.
/// The files to delete are the files that are found in the destination but not found in the source.
/// The files to verify are the files that are found in both the source and the destination, and that have the same size and modification date.
/// </summary>
/// <param name="b_src">The source backend.</param>
/// <param name="b_dst">The destination backend.</param>
/// <param name="config">The parsed configuration for the tool.</param>
/// <returns>A tuple of Lists each holding the files to copy, delete and verify.</returns>
private static (IEnumerable<IFileEntry>, IEnumerable<IFileEntry>, IEnumerable<IFileEntry>) PrepareFileLists(IStreamingBackend b_src, IStreamingBackend b_dst, Config config)
{
IEnumerable<IFileEntry> files_src, files_dst;
using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | List source"))
files_src = b_src.List();
using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | List destination"))
files_dst = b_dst.List();
// Shortcut for force
if (config.Force)
{
return (files_src, files_dst, []);
}
// Shortcut for empty destination
if (!files_dst.Any())
{
return (files_src, [], []);
}
Dictionary<string, IFileEntry> lookup_src, lookup_dst;
using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | Build lookup for source and destination"))
{
lookup_src = files_src.ToDictionary(x => x.Name);
lookup_dst = files_dst.ToDictionary(x => x.Name);
}
var to_copy = new List<IFileEntry>();
var to_delete = new HashSet<string>();
var to_verify = new List<IFileEntry>();
// Find all of the files in src that are not in dst, have a different size or have a more recent modification date
using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | Check the files that are present in source against destination"))
foreach (var f_src in files_src)
{
if (lookup_dst.TryGetValue(f_src.Name, out var f_dst))
{
if (f_src.Size != f_dst.Size || f_src.LastModification > f_dst.LastModification)
{
// The file is different, so we need to copy it
to_copy.Add(f_src);
to_delete.Add(f_dst.Name);
}
else
{
// The file seems to be the same, so we need to verify it if the user wants to
to_verify.Add(f_src);
}
}
else
{
// The file is not in the destination, so we need to copy it
to_copy.Add(f_src);
}
}
// Find all of the files in dst that are not in src
using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | Check the files that are present in destination against source"))
foreach (var f_dst in files_dst)
{
if (to_delete.Contains(f_dst.Name))
continue;
if (!lookup_src.ContainsKey(f_dst.Name))
{
to_delete.Add(f_dst.Name);
}
}
List<IFileEntry> to_delete_lookedup;
using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | Lookup the files to delete"))
to_delete_lookedup = [.. to_delete.Select(x => lookup_dst[x])];
return (to_copy, to_delete_lookedup, to_verify);
}
/// <summary>
/// Renames the files in a backend.
/// The renaming is done by deleting the file and re-uploading it with a new name.
/// </summary>
/// <param name="b">The backend to rename the files in.</param>
/// <param name="files">The files to rename.</param>
/// <param name="config">The parsed configuration for the tool.</param>
/// <returns>The number of successful renames.</returns>
private static async Task<long> RenameAsync(IBackend b, IEnumerable<IFileEntry> files, Config config)
{
long successful_renames = 0;
string prefix = $"{System.DateTime.UtcNow:yyyyMMddHHmmss}.old";
using var downloaded = new MemoryStream();
long i = 0, n = files.Count();
switch (b)
{
case IStreamingBackend sb:
{
var sw_get_src = new System.Diagnostics.Stopwatch();
var sw_put_dst = new System.Diagnostics.Stopwatch();
var sw_del_src = new System.Diagnostics.Stopwatch();
foreach (var f in files)
{
if (config.Progress)
Console.Write($"\rRenaming: {i}/{n}");
Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync",
"Renaming {0} to {1}.{0} by deleting and re-uploading {2} bytes to {3}",
f.Name, prefix,
Duplicati.Library.Utility.Utility.FormatSizeString(downloaded.Length),
sb.DisplayName);
try
{
sw_get_src.Start();
await sb.GetAsync(f.Name, downloaded, CancellationToken.None);
sw_get_src.Stop();
if (config.DryRun)
{
Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync",
"Would rename {0} to {1}.{0} by deleting and re-uploading {2} bytes to {3}",
f.Name, prefix,
Duplicati.Library.Utility.Utility.FormatSizeString(downloaded.Length),
sb.DisplayName);
}
else
{
sw_put_dst.Start();
await sb.PutAsync($"{prefix}.{f.Name}", downloaded, CancellationToken.None);
sw_put_dst.Stop();
sw_del_src.Start();
await sb.DeleteAsync(f.Name, CancellationToken.None);
sw_del_src.Stop();
}
successful_renames++;
}
catch (Exception e)
{
Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e,
"Error renaming {0}: {1}", f.Name, e.Message);
}
finally
{
// Reset the stream
downloaded.SetLength(0);
// Stop any running timers
sw_get_src.Stop();
sw_put_dst.Stop();
sw_del_src.Stop();
}
i++;
}
Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync",
"Rename | Get source: {0} ms, Put destination: {1} ms, Delete source: {2} ms",
TimeSpan.FromMilliseconds(sw_get_src.ElapsedMilliseconds),
TimeSpan.FromMilliseconds(sw_put_dst.ElapsedMilliseconds),
TimeSpan.FromMilliseconds(sw_del_src.ElapsedMilliseconds));
if (config.Progress)
Console.WriteLine($"\rRenaming: {n}/{n}");
return successful_renames;
}
case IRenameEnabledBackend rb:
{
var sw = new System.Diagnostics.Stopwatch();
foreach (var f in files)
{
if (config.Progress)
Console.Write($"\rRenaming: {i}/{n}");
Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync",
"Renaming {0} to {1}.{0} by calling Rename on {2}",
f.Name, prefix, rb.DisplayName);
try
{
if (config.DryRun)
{
Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync",
"Would rename {0} to {1}.{0} by calling Rename on {2}",
f.Name, prefix, rb.DisplayName);
}
else
{
sw.Start();
await rb.RenameAsync(f.Name, $"{prefix}.{f.Name}", CancellationToken.None);
sw.Stop();
}
successful_renames++;
}
catch (Exception e)
{
Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e,
"Error renaming {0}: {1}", f.Name, e.Message);
}
finally
{
// Ensure the timer is stopped
sw.Stop();
}
i++;
}
Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync",
"Rename: {0} ms",
TimeSpan.FromMilliseconds(sw.ElapsedMilliseconds));
if (config.Progress)
Console.WriteLine($"\rRenaming: {n}/{n}");
return successful_renames;
}
default:
throw new NotSupportedException("The backend does not support renaming");
}
}
/// <summary>
/// Verifies the files in the destination backend.
/// The verification is done by downloading the files from the destination backend and comparing them to the source files.
/// </summary>
/// <param name="b_src">The source backend.</param>
/// <param name="b_dst">The destination backend.</param>
/// <param name="files">The files to verify.</param>
/// <param name="config">The parsed configuration for the tool.</param>
/// <returns>A list of the files that failed verification.</returns>
private static async Task<IEnumerable<IFileEntry>> VerifyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable<IFileEntry> files, Config config)
{
var errors = new List<IFileEntry>();
using var s_src = new MemoryStream();
using var s_dst = new MemoryStream();
long i = 0, n = files.Count();
var sw_get = new System.Diagnostics.Stopwatch();
var sw_cmp = new System.Diagnostics.Stopwatch();
foreach (var f in files)
{
if (config.Progress)
Console.Write($"\rVerifying: {i}/{n}");
Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync",
"Verifying {0} by downloading and comparing {1} bytes from {2} and {3}",
f.Name,
Duplicati.Library.Utility.Utility.FormatSizeString(s_src.Length),
b_dst.DisplayName, b_src.DisplayName);
try
{
// Get both files
sw_get.Start();
var fs = b_src.GetAsync(f.Name, s_src, CancellationToken.None);
var ds = b_dst.GetAsync(f.Name, s_dst, CancellationToken.None);
await Task.WhenAll(fs, ds);
sw_get.Stop();
// Compare the contents
sw_cmp.Start();
if (s_src.Length != s_dst.Length || !s_src.ToArray().SequenceEqual(s_dst.ToArray()))
{
errors.Add(f);
}
sw_cmp.Stop();
}
catch (Exception e)
{
errors.Add(f);
Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e,
"Error during verification of {0}: {1}", f.Name, e.Message);
}
finally
{
// Reset the streams
s_src.SetLength(0);
s_dst.SetLength(0);
// Stop any running timers
sw_get.Stop();
sw_cmp.Stop();
}
i++;
}
Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync",
"Verify | Get: {0} ms, Compare: {1} ms",
TimeSpan.FromMilliseconds(sw_get.ElapsedMilliseconds),
TimeSpan.FromMilliseconds(sw_cmp.ElapsedMilliseconds));
if (config.Progress)
Console.WriteLine($"\rVerifying: {n}/{n}");
return errors;
}
}
}
Tools/RemoteSynchronization/RemoteSynchronization.csproj
+16
View File
@@ -0,0 +1,16 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>disable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\Duplicati\Library\Main\Duplicati.Library.Main.csproj" />
<ProjectReference Include="..\..\Duplicati\CommandLine\CLI\Duplicati.CommandLine.csproj" />
<PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
<PackageReference Include="System.CommandLine.NamingConventionBinder" Version="2.0.0-beta4.22272.1" />
</ItemGroup>
</Project>