diff --git a/Duplicati.sln b/Duplicati.sln index eaf3aee27..ff3f49463 100644 --- a/Duplicati.sln +++ b/Duplicati.sln @@ -172,6 +172,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SourceProviders", "SourcePr EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.SourceProvider.Builtin", "Duplicati\Library\SourceProvider\Builtin\Duplicati.Library.SourceProvider.Builtin.csproj", "{6D9742A7-6F32-4571-8276-79716168B525}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tools", "Tools", "{15388C37-9218-4818-972E-738EEA8F1602}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RemoteSynchronization", "Tools\RemoteSynchronization\RemoteSynchronization.csproj", "{D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -486,6 +490,10 @@ Global {6D9742A7-6F32-4571-8276-79716168B525}.Debug|Any CPU.Build.0 = Debug|Any CPU {6D9742A7-6F32-4571-8276-79716168B525}.Release|Any CPU.ActiveCfg = Release|Any CPU {6D9742A7-6F32-4571-8276-79716168B525}.Release|Any CPU.Build.0 = Release|Any CPU + {D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -547,6 +555,7 @@ Global {8ACA2736-3C69-4FA5-BCF9-5EDF50CAF332} = {566EBBDA-19A4-4056-A615-D901D57D2439} {836E0557-B40C-4DC7-9A2A-5C062F9ACC6B} = {E1A9B303-F281-45C5-A4F6-CADD9DE3F3C4} {6D9742A7-6F32-4571-8276-79716168B525} = {424CEF73-4984-430E-9C8B-E61CC0F22074} + {D3A7E41E-279D-4E0A-A2DB-5E8002E02B6D} = {15388C37-9218-4818-972E-738EEA8F1602} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {8B40BAFE-D862-4397-9495-8F5EAF5CE80C} diff --git a/Duplicati/CommandLine/BackendTester/Program.cs b/Duplicati/CommandLine/BackendTester/Program.cs index d794c149f..52a60857b 100644 --- a/Duplicati/CommandLine/BackendTester/Program.cs +++ b/Duplicati/CommandLine/BackendTester/Program.cs @@ -118,16 +118,17 @@ namespace Duplicati.CommandLine.BackendTester for (int i = 0; i < reruns; i++) { - Console.WriteLine("Starting run no {0}", i); + Console.WriteLine(DateTime.Now.ToShortDateString() + " " + DateTime.Now.ToLongTimeString()); + Console.WriteLine(LogTimeStamp + "Starting run no {0}", i); if (!Run(args, options, i == 0)) return 1; } - Console.WriteLine("Unittest complete!"); + Console.WriteLine(LogTimeStamp + "Unittest complete!"); return 0; } catch (Exception ex) { - Console.WriteLine("Unittest failed: " + ex); + Console.WriteLine(LogTimeStamp + "Unittest failed: " + ex); } return 1; @@ -185,7 +186,7 @@ namespace Duplicati.CommandLine.BackendTester } catch (Exception ex) { - Console.WriteLine("Autocreate folder failed with message: " + ex.Message); + Console.WriteLine(LogTimeStamp + "Autocreate folder failed with message: " + ex.Message); } } @@ -199,7 +200,7 @@ namespace Duplicati.CommandLine.BackendTester if (Library.Utility.Utility.ParseBoolOption(options, "auto-clean") && first) if (Library.Utility.Utility.ParseBoolOption(options, "force")) { - Console.WriteLine("Auto clean, removing file: {0}", fe.Name); + Console.WriteLine(LogTimeStamp + "Auto clean, removing file: {0}", fe.Name); backend.DeleteAsync(fe.Name, CancellationToken.None).Await(); continue; } @@ -208,10 +209,10 @@ namespace Duplicati.CommandLine.BackendTester var fileCount = curlist.Where(x => !x.IsFolder).Count(); var filenames = curlist.Where(x => !x.IsFolder).Select(x => x.Name).Take(10).ToList(); - Console.WriteLine($"*** Remote folder contains {fileCount} file(s), aborting"); - Console.WriteLine($"*** First {filenames.Count} file(s): {Environment.NewLine}{string.Join(Environment.NewLine, filenames)}"); + Console.WriteLine(LogTimeStamp + $"*** Remote folder contains {fileCount} file(s), aborting"); + Console.WriteLine(LogTimeStamp + $"*** First {filenames.Count} file(s): {Environment.NewLine}{string.Join(Environment.NewLine, filenames)}"); if (fileCount > filenames.Count) - Console.WriteLine($"*** ... and {fileCount - filenames.Count} more file(s)"); + Console.WriteLine(LogTimeStamp + $"*** ... and {fileCount - filenames.Count} more file(s)"); return false; } @@ -232,7 +233,7 @@ namespace Duplicati.CommandLine.BackendTester { if (!(backend is IStreamingBackend) || disableStreaming) { - Console.WriteLine("Warning: Throttling is only supported in this tool on streaming backends"); + Console.WriteLine(LogTimeStamp + "Warning: Throttling is only supported in this tool on streaming backends"); } throttleUpload = Duplicati.Library.Utility.Sizeparser.ParseSize(throttleUploadString, "kb"); @@ -243,16 +244,26 @@ namespace Duplicati.CommandLine.BackendTester { if (!(backend is IStreamingBackend) || disableStreaming) { - Console.WriteLine("Warning: Throttling is only supported in this tool on streaming backends"); + Console.WriteLine(LogTimeStamp + "Warning: Throttling is only supported in this tool on streaming backends"); } throttleDownload = Duplicati.Library.Utility.Sizeparser.ParseSize(throttleDownloadString, "kb"); } - var readWriteTimeout = (int)TimeSpan.FromSeconds(10).TotalMilliseconds; - if (backend is ITimeoutExemptBackend) - readWriteTimeout = Timeout.Infinite; + int readWriteTimeout = backend is ITimeoutExemptBackend || + (Environment.GetEnvironmentVariable("READ_WRITE_TIMEOUT_SECONDS") is { } timeout + && int.TryParse(timeout, out var seconds) + && seconds == -1) + ? Timeout.Infinite + : Environment.GetEnvironmentVariable("READ_WRITE_TIMEOUT_SECONDS") is { } timeoutRetry + && int.TryParse(timeoutRetry, out var secondsRetry) + ? (int)TimeSpan.FromSeconds(secondsRetry).TotalMilliseconds + : (int)TimeSpan.FromSeconds(60).TotalMilliseconds; + Console.WriteLine(LogTimeStamp + "Read Write Timeout set to {0}{1}", + readWriteTimeout == Timeout.Infinite ? "infinite" : readWriteTimeout + " ms", + backend is ITimeoutExemptBackend ? " (because implements ITimeoutExemptBackend)" : ""); + // Allow overriding the timeout for the backend here, even if timeouts are disabled if (options.TryGetValue("read-write-timeout", out var readWriteTimeoutString)) readWriteTimeout = (int)Timeparser.ParseTimeSpan(readWriteTimeoutString).TotalMilliseconds; @@ -298,7 +309,7 @@ namespace Duplicati.CommandLine.BackendTester byte[] dummyFileHash = null; if (!skipOverwriteTest) { - Console.WriteLine("Uploading wrong files ..."); + Console.WriteLine(LogTimeStamp + "Uploading wrong files ..."); using (var dummy = Library.Utility.TempFile.WrapExistingFile(CreateRandomFile(tf, files.Count, 1024, 2048, rnd))) { using (var fs = new System.IO.FileStream(dummy, System.IO.FileMode.Open, System.IO.FileAccess.Read)) @@ -313,7 +324,7 @@ namespace Duplicati.CommandLine.BackendTester } - Console.WriteLine("Uploading files ..."); + Console.WriteLine(LogTimeStamp + "Uploading files ..."); for (int i = 0; i < files.Count; i++) Uploadfile(files[i].localfilename, i, files[i].remotefilename, backend, disableStreaming, throttleUpload, readWriteTimeout); @@ -328,7 +339,7 @@ namespace Duplicati.CommandLine.BackendTester renamedFileNewName = CreateRandomRemoteFileName(min_filename_size, max_filename_size, allowedChars, trimFilenameSpaces, rnd); - Console.WriteLine("Renaming file {0} from {1} to {2}", renameIndex, originalRenamedFile.remotefilename, renamedFileNewName); + Console.WriteLine(LogTimeStamp + "Renaming file {0} from {1} to {2}", renameIndex, originalRenamedFile.remotefilename, renamedFileNewName); renameEnabledBackend.RenameAsync(originalRenamedFile.remotefilename, renamedFileNewName, CancellationToken.None).Await(); files[renameIndex] = new TempFile(renamedFileNewName, originalRenamedFile.localfilename, originalRenamedFile.hash, originalRenamedFile.length); @@ -336,11 +347,11 @@ namespace Duplicati.CommandLine.BackendTester if (waitAfterUpload > TimeSpan.Zero) { - Console.WriteLine("Waiting {0} after upload", waitAfterUpload); + Console.WriteLine(LogTimeStamp + "Waiting {0} after upload", waitAfterUpload); Thread.Sleep(waitAfterUpload); } - Console.WriteLine("Verifying file list ..."); + Console.WriteLine(LogTimeStamp + "Verifying file list ..."); curlist = backend.List(); foreach (var fe in curlist) @@ -351,12 +362,12 @@ namespace Duplicati.CommandLine.BackendTester if (tx.remotefilename == fe.Name) { if (tx.found) - Console.WriteLine("*** File with name {0} was found more than once", tx.remotefilename); + Console.WriteLine(LogTimeStamp + "*** File with name {0} was found more than once", tx.remotefilename); found = true; tx.found = true; if (fe.Size > 0 && tx.length != fe.Size) - Console.WriteLine("*** File with name {0} has size {1} but the size was reported as {2}", tx.remotefilename, tx.length, fe.Size); + Console.WriteLine(LogTimeStamp + "*** File with name {0} has size {1} but the size was reported as {2}", tx.remotefilename, tx.length, fe.Size); break; } @@ -364,26 +375,26 @@ namespace Duplicati.CommandLine.BackendTester if (!found) if (originalRenamedFile != null && renamedFileNewName != null && originalRenamedFile.remotefilename == fe.Name) { - Console.WriteLine("*** File with name {0} was found on server but was supposed to have been renamed to {1}!", fe.Name, renamedFileNewName); + Console.WriteLine(LogTimeStamp + "*** File with name {0} was found on server but was supposed to have been renamed to {1}!", fe.Name, renamedFileNewName); } else { - Console.WriteLine("*** File with name {0} was found on server but not uploaded!", fe.Name); + Console.WriteLine(LogTimeStamp + "*** File with name {0} was found on server but not uploaded!", fe.Name); } } foreach (var tx in files) if (!tx.found) - Console.WriteLine("*** File with name {0} was uploaded but not found afterwards", tx.remotefilename); + Console.WriteLine(LogTimeStamp + "*** File with name {0} was uploaded but not found afterwards", tx.remotefilename); - Console.WriteLine("Downloading files"); + Console.WriteLine(LogTimeStamp + "Downloading files"); for (int i = 0; i < files.Count; i++) { using (var cf = new Duplicati.Library.Utility.TempFile()) { Exception e = null; - Console.Write("Downloading file {0} ... ", i); + Console.Write(LogTimeStamp + "Downloading file {0} ... ", i); var sw = Stopwatch.StartNew(); @@ -410,22 +421,22 @@ namespace Duplicati.CommandLine.BackendTester if (e != null) { failAfterFinished = true; - Console.WriteLine($"failed\n*** Error: {e} after {sw.ElapsedMilliseconds} ms"); + Console.WriteLine(LogTimeStamp + $"failed\n*** Error: {e} after {sw.ElapsedMilliseconds} ms"); } else - Console.WriteLine($"done in {sw.ElapsedMilliseconds} ms"); + Console.WriteLine($" done in {sw.ElapsedMilliseconds} ms"); - Console.Write("Checking hash ... "); + Console.Write(LogTimeStamp + "Checking hash ... "); using (var fs = new System.IO.FileStream(cf, System.IO.FileMode.Open, System.IO.FileAccess.Read)) if (Convert.ToBase64String(sha.ComputeHash(fs)) != Convert.ToBase64String(files[i].hash)) { if (dummyFileHash != null && Convert.ToBase64String(sha.ComputeHash(fs)) == Convert.ToBase64String(dummyFileHash)) - Console.WriteLine("failed\n*** Downloaded file was the dummy file"); // Should this be failed? + Console.WriteLine(LogTimeStamp + "failed\n*** Downloaded file was the dummy file"); // Should this be failed? else { failAfterFinished = true; - Console.WriteLine("failed\n*** Downloaded file was corrupt"); + Console.WriteLine(LogTimeStamp + "failed\n*** Downloaded file was corrupt"); } } else @@ -433,19 +444,19 @@ namespace Duplicati.CommandLine.BackendTester } } - Console.WriteLine("Deleting files..."); + Console.WriteLine(LogTimeStamp + "Deleting files..."); foreach (var tx in files) try { backend.DeleteAsync(tx.remotefilename, CancellationToken.None).Await(); } catch (Exception ex) { failAfterFinished = true; - Console.WriteLine("*** Failed to delete file {0}, message: {1}", tx.remotefilename, ex); + Console.WriteLine(LogTimeStamp + "*** Failed to delete file {0}, message: {1}", tx.remotefilename, ex); } if (waitAfterDelete > TimeSpan.Zero) { - Console.WriteLine("Waiting {0} after delete", waitAfterDelete); + Console.WriteLine(LogTimeStamp + "Waiting {0} after delete", waitAfterDelete); Thread.Sleep(waitAfterDelete); } @@ -453,11 +464,11 @@ namespace Duplicati.CommandLine.BackendTester foreach (var fe in curlist) if (!fe.IsFolder) { - Console.WriteLine("*** Remote folder contains {0} after cleanup", fe.Name); + Console.WriteLine(LogTimeStamp + "*** Remote folder contains {0} after cleanup", fe.Name); } // Test some error cases - Console.WriteLine("Checking retrieval of non-existent file..."); + Console.WriteLine(LogTimeStamp + "Checking retrieval of non-existent file..."); var caughtExpectedException = false; try { @@ -468,25 +479,25 @@ namespace Duplicati.CommandLine.BackendTester } catch (FileMissingException) { - Console.WriteLine("Caught expected FileMissingException"); + Console.WriteLine(LogTimeStamp + "Caught expected FileMissingException"); caughtExpectedException = true; } catch (Exception ex) { - Console.WriteLine("*** Retrieval of non-existent file failed: {0}", ex); + Console.WriteLine(LogTimeStamp + "*** Retrieval of non-existent file failed: {0}", ex); } if (!caughtExpectedException) { failAfterFinished = true; - Console.WriteLine("*** Retrieval of non-existent file should have failed with FileMissingException"); + Console.WriteLine(LogTimeStamp + "*** Retrieval of non-existent file should have failed with FileMissingException"); } } // Test quota retrieval if (backend is IQuotaEnabledBackend quotaEnabledBackend) { - Console.WriteLine("Checking quota..."); + Console.WriteLine(LogTimeStamp + "Checking quota..."); IQuotaInfo quota = null; bool noException; try @@ -496,7 +507,7 @@ namespace Duplicati.CommandLine.BackendTester } catch (Exception ex) { - Console.WriteLine("*** Checking quota information failed: {0}", ex); + Console.WriteLine(LogTimeStamp + "*** Checking quota information failed: {0}", ex); noException = false; } @@ -504,18 +515,18 @@ namespace Duplicati.CommandLine.BackendTester { if (quota != null) { - Console.WriteLine("Free Space: {0}", Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace)); - Console.WriteLine("Total Space: {0}", Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace)); + Console.WriteLine(LogTimeStamp + "Free Space: {0}", Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace)); + Console.WriteLine(LogTimeStamp + "Total Space: {0}", Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace)); } else { - Console.WriteLine("Unable to retrieve quota information"); + Console.WriteLine(LogTimeStamp + "Unable to retrieve quota information"); } } } // Test DNSName lookup - Console.WriteLine("Checking DNS names used by this backend..."); + Console.WriteLine(LogTimeStamp + "Checking DNS names used by this backend..."); try { var dnsNames = backend.GetDNSNamesAsync(CancellationToken.None).Await(); @@ -528,12 +539,12 @@ namespace Duplicati.CommandLine.BackendTester } else { - Console.WriteLine("No DNS names reported"); + Console.WriteLine(LogTimeStamp + "No DNS names reported"); } } catch (Exception ex) { - Console.WriteLine("*** Checking DNSName failed: {0}", ex); + Console.WriteLine(LogTimeStamp + "*** Checking DNSName failed: {0}", ex); } } finally @@ -548,7 +559,7 @@ namespace Duplicati.CommandLine.BackendTester private static void Uploadfile(string localfilename, int i, string remotefilename, IBackend backend, bool disableStreaming, long throttle, int readWriteTimeout) { - Console.Write("Uploading file {0}, {1} ... ", i, Duplicati.Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(localfilename).Length)); + Console.Write(LogTimeStamp + "Uploading file {0}, {1} ... ", i, Duplicati.Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(localfilename).Length)); Exception e = null; var sw = Stopwatch.StartNew(); @@ -575,11 +586,11 @@ namespace Duplicati.CommandLine.BackendTester if (e != null) { - Console.WriteLine($"Failed to upload file {i}, error message: {e}, remote name: {remotefilename} after {sw.ElapsedMilliseconds} ms"); + Console.WriteLine(LogTimeStamp + $"Failed to upload file {i}, error message: {e}, remote name: {remotefilename} after {sw.ElapsedMilliseconds} ms"); while (e.InnerException != null) { e = e.InnerException; - Console.WriteLine(" Inner exception: {0}", e); + Console.WriteLine(LogTimeStamp + " Inner exception: {0}", e); } } else @@ -604,7 +615,7 @@ namespace Duplicati.CommandLine.BackendTester private static string CreateRandomFile(Library.Utility.TempFolder tf, int i, int min_file_size, int max_file_size, Random rnd) { - Console.Write("Generating file {0}", i); + Console.Write(LogTimeStamp + "Generating file {0}", i); string filename = System.IO.Path.Combine(tf, i.ToString()); using (System.IO.FileStream fs = new System.IO.FileStream(filename, System.IO.FileMode.CreateNew, System.IO.FileAccess.Write)) { @@ -625,6 +636,8 @@ namespace Duplicati.CommandLine.BackendTester return filename; } + private static string LogTimeStamp => $"[{DateTime.Now:HH:mm:ss fff}] "; + public static IList SupportedCommands { get diff --git a/Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs b/Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs index 14bd9a2bf..3d3855ef4 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.GetOperation.cs @@ -123,16 +123,14 @@ partial class BackendManager { // extended to use stacked streams using (var fs = System.IO.File.OpenWrite(dlTarget)) - using (var act = new StreamUtil.TimeoutObservingStream(fs) { WriteTimeout = backend is ITimeoutExemptBackend ? Timeout.Infinite : Context.Options.ReadWriteTimeout }) using (var hasher = HashFactory.CreateHasher(Context.Options.FileHashAlgorithm)) - using (var hs = new HashCalculatingStream(act, hasher)) + using (var hs = new HashCalculatingStream(fs, hasher)) using (var ss = new ShaderStream(hs, true)) - using (var linkedToken = CancellationTokenSource.CreateLinkedTokenSource(cancelToken, act.TimeoutToken)) { using (var ts = new ThrottledStream(ss, 0, Context.Options.MaxDownloadPrSecond)) using (var pgs = new ProgressReportingStream(ts, pg => Context.HandleProgress(ts, pg, RemoteFilename))) { - await streamingBackend.GetAsync(RemoteFilename, pgs, linkedToken.Token).ConfigureAwait(false); + await streamingBackend.GetAsync(RemoteFilename, pgs, cancelToken).ConfigureAwait(false); } ss.Flush(); retDownloadSize = ss.TotalBytesWritten; diff --git a/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs b/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs index f9733c775..cea59320d 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs @@ -268,12 +268,10 @@ partial class BackendManager if (backend is IStreamingBackend streamingBackend && !Context.Options.DisableStreamingTransfers) { - using (var fs = System.IO.File.OpenRead(LocalFilename)) - using (var act = new StreamUtil.TimeoutObservingStream(fs) { ReadTimeout = backend is ITimeoutExemptBackend ? Timeout.Infinite : Context.Options.ReadWriteTimeout }) - using (var ts = new ThrottledStream(act, Context.Options.MaxUploadPrSecond, 0)) - using (var pgs = new ProgressReportingStream(ts, pg => Context.HandleProgress(ts, pg, RemoteFilename))) - using (var linkedToken = CancellationTokenSource.CreateLinkedTokenSource(cancelToken, act.TimeoutToken)) - await streamingBackend.PutAsync(RemoteFilename, pgs, linkedToken.Token).ConfigureAwait(false); + using var fs = System.IO.File.OpenRead(LocalFilename); + using var ts = new ThrottledStream(fs, Context.Options.MaxUploadPrSecond, 0); + using var pgs = new ProgressReportingStream(ts,pg => Context.HandleProgress(ts, pg, RemoteFilename)); + await streamingBackend.PutAsync(RemoteFilename, pgs, cancelToken).ConfigureAwait(false); } else await backend.PutAsync(RemoteFilename, LocalFilename, cancelToken).ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index be4083989..821e75381 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -141,6 +141,13 @@ namespace Duplicati.Library.Main.Operation var progress = 0; var targetProgess = tp.ExtraVolumes.Count() + tp.MissingVolumes.Count() + tp.VerificationRequiredVolumes.Count(); + var mostRecentLocal = db.FilesetTimes.Select(x => x.Value).Append(DateTime.MinValue).Max(); + var mostRecentRemote = tp.ParsedVolumes.Select(x => x.Time).Append(DateTime.MinValue).Max(); + if (mostRecentLocal < DateTime.UnixEpoch) + throw new UserInformationException("The local database has no fileset times. Consider deleting the local database and run the repair operation again.", "LocalDatabaseHasNoFilesetTimes"); + if (mostRecentRemote > mostRecentLocal) + throw new UserInformationException($"The remote files are newer ({mostRecentRemote}) than the local database ({mostRecentLocal}), this is likely because the database is outdated. Consider deleting the local database and run the repair operation again.", "RemoteFilesNewerThanLocalDatabase"); + if (m_options.Dryrun) { if (!tp.ParsedVolumes.Any() && tp.OtherVolumes.Any()) diff --git a/Duplicati/Library/Modules/Builtin/SendTelegramMessage.cs b/Duplicati/Library/Modules/Builtin/SendTelegramMessage.cs index 394d41c31..b4a73948a 100644 --- a/Duplicati/Library/Modules/Builtin/SendTelegramMessage.cs +++ b/Duplicati/Library/Modules/Builtin/SendTelegramMessage.cs @@ -56,6 +56,10 @@ public class SendTelegramMessage : ReportHelper /// private const string OPTION_CHANNEL = "send-telegram-channel-id"; /// + /// Option used to specify the topic ID to be used in telegram groups. + /// + private const string OPTION_TOPICID = "send-telegram-topid-id"; + /// /// Option used to specify report body /// private const string OPTION_MESSAGE = "send-telegram-message"; @@ -133,20 +137,19 @@ public class SendTelegramMessage : ReportHelper /// public override IList SupportedCommands => [ - new CommandLineArgument(OPTION_CHANNEL, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendtelegramchannelShort, Strings.SendTelegramMessage.SendtelegramchannelLong), - new CommandLineArgument(OPTION_MESSAGE, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendtelegrammessageShort, Strings.SendTelegramMessage.SendtelegrammessageLong, DEFAULT_BODY), - new CommandLineArgument(OPTION_BOTID, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendtelegrambotidShort, Strings.SendTelegramMessage.SendtelegrambotidLong), - new CommandLineArgument(OPTION_APIKEY, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendtelegramapikeyShort, Strings.SendTelegramMessage.SendtelegramapikeyLong), - new CommandLineArgument(OPTION_SENDLEVEL, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendtelegramlevelShort, Strings.SendTelegramMessage.SendtelegramlevelLong(ParsedResultType.Success.ToString(), ParsedResultType.Warning.ToString(), ParsedResultType.Error.ToString(), ParsedResultType.Fatal.ToString(), "All"), DEFAULT_LEVEL, null, Enum.GetNames(typeof(ParsedResultType)).Union(new string[] { "All" } ).ToArray()), - new CommandLineArgument(OPTION_SENDALL, CommandLineArgument.ArgumentType.Boolean, Strings.SendTelegramMessage.SendtelegramanyoperationShort, Strings.SendTelegramMessage.SendtelegramanyoperationLong), - + new CommandLineArgument(OPTION_CHANNEL, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramChannelShort, Strings.SendTelegramMessage.SendTelegramChannelLong), + new CommandLineArgument(OPTION_MESSAGE, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramMessageShort, Strings.SendTelegramMessage.SendTelegramMessageLong, DEFAULT_BODY), + new CommandLineArgument(OPTION_BOTID, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramBotIdShort, Strings.SendTelegramMessage.SendTelegramBotIdLong), + new CommandLineArgument(OPTION_APIKEY, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramApiKeyShort, Strings.SendTelegramMessage.SendTelegramApiKeyLong), + new CommandLineArgument(OPTION_SENDLEVEL, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramLevelShort, Strings.SendTelegramMessage.SendTelegramlevelLong(ParsedResultType.Success.ToString(), ParsedResultType.Warning.ToString(), ParsedResultType.Error.ToString(), ParsedResultType.Fatal.ToString(), "All"), DEFAULT_LEVEL, null, Enum.GetNames(typeof(ParsedResultType)).Union(new string[] { "All" } ).ToArray()), + new CommandLineArgument(OPTION_SENDALL, CommandLineArgument.ArgumentType.Boolean, Strings.SendTelegramMessage.SendTelegramManyOperationShort, Strings.SendTelegramMessage.SendTelegramManyOperationLong), new CommandLineArgument(OPTION_LOG_LEVEL, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.OptionLoglevelShort, Strings.ReportHelper.OptionLoglevelLong, DEFAULT_LOG_LEVEL.ToString(), null, Enum.GetNames(typeof(Logging.LogMessageType))), new CommandLineArgument(OPTION_LOG_FILTER, CommandLineArgument.ArgumentType.String, Strings.ReportHelper.OptionLogfilterShort, Strings.ReportHelper.OptionLogfilterLong), new CommandLineArgument(OPTION_MAX_LOG_LINES, CommandLineArgument.ArgumentType.Integer, Strings.ReportHelper.OptionmaxloglinesShort, Strings.ReportHelper.OptionmaxloglinesLong, DEFAULT_LOGLINES.ToString()), - new CommandLineArgument(OPTION_RESULT_FORMAT, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.ResultFormatShort, Strings.ReportHelper.ResultFormatLong(Enum.GetNames(typeof(ResultExportFormat))), DEFAULT_EXPORT_FORMAT.ToString(), null, Enum.GetNames(typeof(ResultExportFormat))), + new CommandLineArgument(OPTION_TOPICID, CommandLineArgument.ArgumentType.String, Strings.SendTelegramMessage.SendTelegramTopicShort, Strings.SendTelegramMessage.SendTelegramTopicLong), ]; - + protected override string SubjectOptionName => OPTION_MESSAGE; protected override string BodyOptionName => OPTION_MESSAGE; protected override string ActionLevelOptionName => OPTION_SENDLEVEL; @@ -168,6 +171,10 @@ public class SendTelegramMessage : ReportHelper /// The Telegram ChannelID /// private string m_channelId; + /// + /// The telegram TopicID + /// + private string m_topicId; /// /// This method is the interception where the module can interact with the execution environment and modify the settings. @@ -182,12 +189,14 @@ public class SendTelegramMessage : ReportHelper commandlineOptions.TryGetValue(OPTION_BOTID, out m_botid); commandlineOptions.TryGetValue(OPTION_APIKEY, out m_apikey); + commandlineOptions.TryGetValue(OPTION_TOPICID, out m_topicId); return true; } #endregion - + + /// protected override string ReplaceTemplate(string input, object result, Exception exception, bool subjectline) { // No need to do the expansion as we throw away the result @@ -220,7 +229,7 @@ public class SendTelegramMessage : ReportHelper // Send all chunks sequentially for (var i = 0; i < messages.Length; i++) - await SendMessageChunk(messages[i], i + 1, messages.Length); + await SendMessageChunk(messages[i], i + 1, messages.Length).ConfigureAwait(false); } catch (Exception e) { @@ -239,9 +248,15 @@ public class SendTelegramMessage : ReportHelper { try { + if (string.IsNullOrWhiteSpace(m_botid)) + throw new Exception("Telegram Bot ID is required and not set"); + if (string.IsNullOrWhiteSpace(m_apikey)) + throw new Exception("Telegram API Key is required and not set"); + var p = new { chat_id = Uri.EscapeDataString(m_channelId), + message_thread_id = string.IsNullOrWhiteSpace(m_topicId) ? null : Uri.EscapeDataString(m_topicId), parse_mode = "Markdown", text = Uri.EscapeDataString(totalParts > 1 ? $"Part {partNumber}/{totalParts}:\n{message}" @@ -250,13 +265,16 @@ public class SendTelegramMessage : ReportHelper apiKey = Uri.EscapeDataString(m_apikey) }; - var url = $"https://api.telegram.org/bot{p.botId}:{p.apiKey}/sendMessage?chat_id={p.chat_id}&parse_mode={p.parse_mode}&text={p.text}"; + var baseUrl = $"https://api.telegram.org/bot{p.botId}:{p.apiKey}/sendMessage"; + var url = $"{baseUrl}?chat_id={p.chat_id}&parse_mode={p.parse_mode}&text={p.text}" + + $"{(!string.IsNullOrWhiteSpace(m_topicId) ? $"&message_thread_id={m_topicId}" : string.Empty)}"; + using var timeoutToken = new CancellationTokenSource(); timeoutToken.CancelAfter(REQUEST_TIMEOUT); using var client = HttpClientHelper.CreateClient(); - var response = await client.GetAsync(url, timeoutToken.Token); - var responseContent = await response.Content.ReadAsStringAsync(timeoutToken.Token); + var response = await client.GetAsync(url, timeoutToken.Token).ConfigureAwait(false); + var responseContent = await response.Content.ReadAsStringAsync(timeoutToken.Token).ConfigureAwait(false); if (!responseContent.Contains("\"ok\":true")) Logging.Log.WriteWarningMessage(LOGTAG, "telegramSendError", null, diff --git a/Duplicati/Library/Modules/Builtin/Strings.cs b/Duplicati/Library/Modules/Builtin/Strings.cs index e4d9cc974..5d4750cc5 100644 --- a/Duplicati/Library/Modules/Builtin/Strings.cs +++ b/Duplicati/Library/Modules/Builtin/Strings.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Localization.Short; using System; @@ -167,11 +167,12 @@ You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The sp internal static class SendTelegramMessage { - public static string Description { get { return LC.L(@"This module provides support for sending status reports via Telegram messages"); } } - public static string DisplayName { get { return LC.L(@"Telegram report module"); } } - public static string SendtelegramchannelLong { get { return LC.L(@"Use this option to set the channel ID."); } } - public static string SendtelegramchannelShort { get { return LC.L(@"Telegram channel ID"); } } - public static string SendtelegrammessageLong { get { return LC.L(@"This value can be a filename. If the file exists, the file contents will be used as the message. + public static string Description => LC.L(@"This module provides support for sending status reports via Telegram messages"); + public static string DisplayName => LC.L(@"Telegram report module"); + public static string SendTelegramChannelLong => LC.L(@"Use this option to set the channel ID."); + public static string SendTelegramChannelShort => LC.L(@"Telegram channel ID"); + public static string SendTelegramMessageLong => + LC.L(@"This value can be a filename. If the file exists, the file contents will be used as the message. In the message, certain tokens are replaced: %OPERATIONNAME% - The name of the operation, normally ""Backup"" @@ -179,19 +180,20 @@ In the message, certain tokens are replaced: %LOCALPATH% - The path to the local files or folders involved in the operation (if any) %PARSEDRESULT% - The parsed result, if the operation is a backup. Possible values are: Error, Warning, Success -All command line options are also reported within %value%, e.g. %volsize%. Any unknown/unset value is removed."); } } - public static string SendtelegrammessageShort { get { return LC.L(@"The message template"); } } - public static string SendtelegrambotidLong { get { return LC.L(@"Use this option to set a bot ID for the bot that will send the message."); } } +All command line options are also reported within %value%, e.g. %volsize%. Any unknown/unset value is removed."); - public static string SendtelegrambotidShort { get { return LC.L(@"The Telegram bot ID"); } } - public static string SendtelegramapikeyLong { get { return LC.L(@"Use this option to set a API key for the bot that will send the message."); } } - public static string SendtelegramapikeyShort { get { return LC.L(@"The Telegram API key"); } } - public static string SendtelegramlevelLong(string success, string warning, string error, string fatal, string all) { return LC.L(@"You can specify one of ""{0}"", ""{1}"", ""{2}"", ""{3}"". + public static string SendTelegramMessageShort => LC.L(@"The message template"); + public static string SendTelegramBotIdLong => LC.L(@"Use this option to set a bot ID for the bot that will send the message."); + public static string SendTelegramBotIdShort => LC.L(@"The Telegram bot ID"); + public static string SendTelegramApiKeyLong => LC.L(@"Use this option to set a API key for the bot that will send the message."); + public static string SendTelegramApiKeyShort => LC.L(@"The Telegram API key"); + public static string SendTelegramTopicShort => LC.L(@"The Telegram topic ID"); + public static string SendTelegramTopicLong => LC.L(@"Topic ID for the Topic in the telegram group. For more information on Telegram setup, refer to documentation."); + public static string SendTelegramlevelLong(string success, string warning, string error, string fatal, string all) { return LC.L(@"You can specify one of ""{0}"", ""{1}"", ""{2}"", ""{3}"". You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The special value ""{4}"" is a shorthand for ""{0},{1},{2},{3}"" and will cause all backup operations to send a message.", success, warning, error, fatal, all); } - public static string SendtelegramlevelShort { get { return LC.L(@"The messages to send"); } } - public static string SendtelegramanyoperationLong { get { return LC.L(@"By default, messages will only be sent after a backup operation. Use this option to send messages for all operations."); } } - public static string SendtelegramanyoperationShort { get { return LC.L(@"Send messages for all operations"); } } - public static string LoginTimeoutError { get { return LC.L(@"Timeout occurred while sending to Telegram server"); } } + public static string SendTelegramLevelShort => LC.L(@"The messages to send"); + public static string SendTelegramManyOperationLong => LC.L(@"By default, messages will only be sent after a backup operation. Use this option to send messages for all operations."); + public static string SendTelegramManyOperationShort => LC.L(@"Send messages for all operations"); } internal static class SendHttpMessage diff --git a/Duplicati/Library/RestAPI/Strings.cs b/Duplicati/Library/RestAPI/Strings.cs index 115f56e7b..a7e7d3e0c 100644 --- a/Duplicati/Library/RestAPI/Strings.cs +++ b/Duplicati/Library/RestAPI/Strings.cs @@ -59,6 +59,8 @@ Error message: {0}", error); } public static string WebserverAllowedhostnamesDescription { get { return LC.L(@"The hostnames that are accepted, separated with semicolons. If any of the hostnames are ""*"", all hostnames are allowed and the hostname checking is disabled."); } } public static string PingpongkeepaliveLong { get { return LC.L(@"When running as a server, the service daemon must verify that the process is responding. If this option is enabled, the server reads stdin and writes a reply to each line read."); } } public static string PingpongkeepaliveShort { get { return LC.L(@"Enable the ping-pong responder"); } } + public static string DisableupdatecheckShort { get { return LC.L(@"Disable the automatic update check"); } } + public static string DisableupdatecheckLong { get { return LC.L(@"Use this option to disable the automatic update check. Manual update checks can still be performed."); } } public static string LogretentionLong { get { return LC.L(@"Set the time after which log data will be purged from the database."); } } public static string LogretentionShort { get { return LC.L(@"Clean up old log data"); } } public static string ServerdatafolderLong(string envname) { return LC.L(@"Duplicati needs to store a small database with all settings. Use this option to choose where the settings are stored. This option can also be set with the environment variable {0}.", envname); } diff --git a/Duplicati/Library/RestAPI/UpdatePollThread.cs b/Duplicati/Library/RestAPI/UpdatePollThread.cs index 5996c46e6..ee251f244 100644 --- a/Duplicati/Library/RestAPI/UpdatePollThread.cs +++ b/Duplicati/Library/RestAPI/UpdatePollThread.cs @@ -39,6 +39,7 @@ namespace Duplicati.Server private readonly object m_lock = new object(); private AutoResetEvent m_waitSignal; private double m_downloadProgress; + private bool m_disableChecks = false; public bool IsUpdateRequested { get; private set; } = false; @@ -56,8 +57,9 @@ namespace Duplicati.Server } } - public void Init() + public void Init(bool disableChecks) { + m_disableChecks = disableChecks; m_waitSignal = new AutoResetEvent(false); ThreadState = UpdatePollerStates.Waiting; m_thread = new Thread(Run) @@ -115,16 +117,20 @@ namespace Duplicati.Server if (nextCheck < DateTime.UtcNow || m_forceCheck) { + DateTime started = DateTime.UtcNow; + connection.ApplicationSettings.LastUpdateCheck = started; + nextCheck = connection.ApplicationSettings.NextUpdateCheck; + + // If this is not forced, and we have disabled checks, we just update the next check time + if (m_disableChecks && !m_forceCheck) + continue; + lock (m_lock) m_forceCheck = false; ThreadState = UpdatePollerStates.Checking; eventPollNotify.SignalNewEvent(); - DateTime started = DateTime.UtcNow; - connection.ApplicationSettings.LastUpdateCheck = started; - nextCheck = connection.ApplicationSettings.NextUpdateCheck; - Library.AutoUpdater.ReleaseType rt; if (!Enum.TryParse(connection.ApplicationSettings.UpdateChannel, true, out rt)) rt = Duplicati.Library.AutoUpdater.ReleaseType.Unknown; diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index b43feb0f0..25b37a954 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -51,6 +51,7 @@ namespace Duplicati.Server private const string DISABLE_DB_ENCRYPTION_OPTION = "disable-db-encryption"; private const string REQUIRE_DB_ENCRYPTION_KEY_OPTION = "require-db-encryption-key"; private const string SETTINGS_ENCRYPTION_KEY_OPTION = "settings-encryption-key"; + private const string DISABLE_UPDATE_CHECK_OPTION = "disable-update-check"; #if DEBUG private const bool DEBUG_MODE = true; @@ -266,7 +267,7 @@ namespace Duplicati.Server DuplicatiWebserver = StartWebServer(commandlineOptions, DataConnection).ConfigureAwait(false).GetAwaiter().GetResult(); - UpdatePoller.Init(); + UpdatePoller.Init(Library.Utility.Utility.ParseBoolOption(commandlineOptions, DISABLE_UPDATE_CHECK_OPTION)); SetPurgeTempFilesTimer(commandlineOptions); @@ -961,6 +962,7 @@ namespace Duplicati.Server new Duplicati.Library.Interface.CommandLineArgument(WebServerLoader.OPTION_WEBSERVICE_TIMEZONE, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.String, Strings.Program.WebserverTimezoneDescription, Strings.Program.WebserverTimezoneDescription, TimeZoneHelper.GetLocalTimeZone(), null, TimeZoneHelper.GetTimeZones().Select(x => x.Id).ToArray()), new Duplicati.Library.Interface.CommandLineArgument(WebServerLoader.OPTION_WEBSERVICE_CORS_ORIGINS, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.Program.WebserverCorsOriginsDescription, Strings.Program.WebserverCorsOriginsDescription, WebServerLoader.DEFAULT_OPTION_SPAPATHS), new Duplicati.Library.Interface.CommandLineArgument(PING_PONG_KEEPALIVE_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.PingpongkeepaliveShort, Strings.Program.PingpongkeepaliveLong), + new Duplicati.Library.Interface.CommandLineArgument(DISABLE_UPDATE_CHECK_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.DisableupdatecheckShort, Strings.Program.DisableupdatecheckLong), new Duplicati.Library.Interface.CommandLineArgument("log-retention", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Timespan, Strings.Program.LogretentionShort, Strings.Program.LogretentionLong, DEFAULT_LOG_RETENTION), new Duplicati.Library.Interface.CommandLineArgument(DataFolderManager.SERVER_DATAFOLDER_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.Program.ServerdatafolderShort, Strings.Program.ServerdatafolderLong(DataFolderManager.DATAFOLDER_ENV_NAME), DataFolderManager.DATAFOLDER), new Duplicati.Library.Interface.CommandLineArgument(DISABLE_DB_ENCRYPTION_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.DisabledbencryptionShort, Strings.Program.DisabledbencryptionLong), diff --git a/Duplicati/Server/webroot/ngax/templates/settings.html b/Duplicati/Server/webroot/ngax/templates/settings.html index abc79760c..d3a116aab 100644 --- a/Duplicati/Server/webroot/ngax/templates/settings.html +++ b/Duplicati/Server/webroot/ngax/templates/settings.html @@ -138,7 +138,7 @@
-
Try out the new features that we are working on. Currently the most stable version available. Test Restore data before using this in production environments.
+
Try out the new features that we are working on. Test Backup & Restore before using this in production environments.
diff --git a/Duplicati/Tools/Duplicati.Tools.csproj b/Duplicati/Tools/Duplicati.Tools.csproj index 0e771a6e5..ee68d0b16 100644 --- a/Duplicati/Tools/Duplicati.Tools.csproj +++ b/Duplicati/Tools/Duplicati.Tools.csproj @@ -5,7 +5,7 @@ Library Copyright © 2025 Team Duplicati, MIT license - + utility-scripts\DuplicatiVerify.ps1 @@ -24,12 +24,12 @@ utility-scripts\pause-resume.py - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Duplicati/UnitTest/Duplicati.UnitTest.csproj b/Duplicati/UnitTest/Duplicati.UnitTest.csproj index e3623c8a9..9925d51ee 100644 --- a/Duplicati/UnitTest/Duplicati.UnitTest.csproj +++ b/Duplicati/UnitTest/Duplicati.UnitTest.csproj @@ -8,7 +8,7 @@ Duplicati.UnitTest Copyright © 2025 Team Duplicati, MIT license - + @@ -35,14 +35,15 @@ + - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Duplicati/UnitTest/ToolTests.cs b/Duplicati/UnitTest/ToolTests.cs new file mode 100644 index 000000000..e5ec55dad --- /dev/null +++ b/Duplicati/UnitTest/ToolTests.cs @@ -0,0 +1,529 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using Duplicati.Library.Common.IO; +using Duplicati.Library.Interface; +using Duplicati.Library.Main; +using NUnit.Framework; +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; + +namespace Duplicati.UnitTest +{ + + /// + /// Tests the tools. + /// + public class ToolTests : BasicSetupHelper + { + + /// + /// Tests that the remote synchronization tool doesn't do anything when the dry run option is used. + /// + [Test] + [Category("Tools")] + public void TestDryRun() + { + var l1 = Path.Combine(TARGETFOLDER, "l1"); + var l2 = Path.Combine(TARGETFOLDER, "l2"); + + Directory.CreateDirectory(l1); + Directory.CreateDirectory(l2); + + GenerateTestData(l1, 5, 0, 0, 1024).Wait(); + + var args = new string[] { $"file://{l1}", $"file://{l2}", "--confirm", "--dry-run" }; + + var async_call = RemoteSynchronization.Program.Main(args); + var return_code = async_call.ConfigureAwait(false).GetAwaiter().GetResult(); + + Assert.AreEqual(0, return_code, "Remote synchronization tool did not return 0."); + Assert.IsFalse(DirectoriesAndContentsAreEqual(l1, l2), "Synchronized directories are equal"); + Assert.IsTrue(!Directory.EnumerateFiles(l2).Any(), "Destination directory is not empty"); + } + + /// + /// Tests that the remote synchronization tool works with an empty source to an empty destination. + /// + [Test] + [Category("Tools")] + public void TestEmptySourceAndDestination() + { + var l1 = Path.Combine(TARGETFOLDER, "empty_src"); + var l2 = Path.Combine(TARGETFOLDER, "l2"); + + Directory.CreateDirectory(l1); + Directory.CreateDirectory(l2); + + var args = new string[] { $"file://{l1}", $"file://{l2}", "--confirm" }; + + var async_call = RemoteSynchronization.Program.Main(args); + var return_code = async_call.ConfigureAwait(false).GetAwaiter().GetResult(); + + Assert.AreEqual(0, return_code, "Remote synchronization tool did not return 0."); + Assert.IsTrue(DirectoriesAndContentsAreEqual(l1, l2), "Synchronized directories are not equal"); + } + + /// + /// Test that remote synchronizing an empty source to a non-empty destination deletes the destination files. + /// + [Test] + [Category("Tools")] + public void TestEmptySourceDeletesDestination() + { + var l1 = Path.Combine(TARGETFOLDER, "empty_src"); + var l2 = Path.Combine(TARGETFOLDER, "l2"); + + Directory.CreateDirectory(l1); + Directory.CreateDirectory(l2); + + GenerateTestData(l2, 5, 0, 0, 1024).Wait(); + + var args = new string[] { $"file://{l1}", $"file://{l2}", "--confirm" }; + + var async_call = RemoteSynchronization.Program.Main(args); + var return_code = async_call.ConfigureAwait(false).GetAwaiter().GetResult(); + + Assert.AreEqual(0, return_code, "Remote synchronization tool did not return 0."); + Assert.IsTrue(DirectoriesAndContentsAreEqual(l1, l2), "Synchronized directories are not equal"); + } + + /// + /// Test that remote synchronizing an empty source to a non-empty destination renames the destination files when the `--retention` option is used. + /// + [Test] + [Category("Tools")] + public void TestEmptySourceRenamesDestination() + { + var l1 = Path.Combine(TARGETFOLDER, "empty_src"); + var l2 = Path.Combine(TARGETFOLDER, "l2"); + + Directory.CreateDirectory(l1); + Directory.CreateDirectory(l2); + + GenerateTestData(l2, 5, 0, 0, 1024).Wait(); + + var filelist = Directory.EnumerateFiles(l2).ToList(); + var files = filelist.Select(x => File.ReadAllBytes(x)).ToList(); + + var args = new string[] { $"file://{l1}", $"file://{l2}", "--confirm", "--retention" }; + + var async_call = RemoteSynchronization.Program.Main(args); + var return_code = async_call.ConfigureAwait(false).GetAwaiter().GetResult(); + + Assert.AreEqual(0, return_code, "Remote synchronization tool did not return 0."); + + var newfilelist = Directory.EnumerateFiles(l2).ToList(); + foreach (var (name, contents) in filelist.Zip(files)) + { + var filename = Path.GetFileName(name); + var newcontents = File.ReadAllBytes(newfilelist.FirstOrDefault(x => x.EndsWith(filename))); + Assert.AreEqual(contents, newcontents, "File contents are not equal"); + } + } + + /// + /// Tests passing all arguments to the main method of the remote synchronization tool. + /// + [Test] + public void TestMainMethodParsesArgumentsCorrectly() + { + string[][] testCases = + { + ["source", "destination", "--parse-arguments-only"], + ["source", "destination", "--parse-arguments-only", "--dry-run"], + ["source", "destination", "--parse-arguments-only", "--force"], + ["source", "destination", "--parse-arguments-only", "--dry-run", "--force"], + ["source", "destination", "--parse-arguments-only", "--verify-contents"], + ["source", "destination", "--parse-arguments-only", "--verify-get-after-put"], + ["source", "destination", "--parse-arguments-only", "--retry", "3"], + ["source", "destination", "--parse-arguments-only", "--log-level", "Debug"], + ["source", "destination", "--parse-arguments-only", "--log-level", "Information"], + ["source", "destination", "--parse-arguments-only", "--log-level", "Profiling"], + ["source", "destination", "--parse-arguments-only", "--log-level", "Verbose"], + ["source", "destination", "--parse-arguments-only", "--log-file", "somefile.log"], + ["source", "destination", "--parse-arguments-only", "--progress"], + ["source", "destination", "--parse-arguments-only", "--retention"], + ["source", "destination", "--parse-arguments-only", "--confirm"], + ["source", "destination", "--parse-arguments-only", "--global-options", "someglobalkey=someglobalvalue", "anotherglobalkey=anotherglobalvalue"], + ["source", "destination", "--parse-arguments-only", "--src-options", "somesrckey=somesrcvalue", "anothersrckey=anothersrcvalue"], + ["source", "destination", "--parse-arguments-only", "--dst-options", "somedstkey=somedstvalue", "anotherdstkey=anotherdstvalue"], + [ + "source", "destination", "--parse-arguments-only", + "--dry-run", "--force", "--verify-contents", "--retry", "3", "--log-level", "Debug", "--log-file", "somefile.log", "--progress", "--retention", "--confirm", + "--global-options", "someglobalkey=someglobalvalue", "anotherglobalkey=anotherglobalvalue", + "--src-options", "somesrckey=somesrcvalue", "anothersrckey=anothersrcvalue", + "--dst-options", "somedstkey=somedstvalue", "anotherdstkey=anotherdstvalue" + ], + }; + + foreach (var args in testCases) + { + int result = RemoteSynchronization.Program.Main(args).ConfigureAwait(false).GetAwaiter().GetResult(); + Assert.AreEqual(0, result, $"Failed for args: {string.Join(" ", args)}"); + } + + int failed_result = RemoteSynchronization.Program.Main(["source", "destination", "--bogus-option"]).ConfigureAwait(false).GetAwaiter().GetResult(); + Assert.AreEqual(1, failed_result, "Invalid option did not return 1"); + } + + /// + /// Tests the original inded use of the remote synchronization tool on an empty destination. + /// + [Test] + [Category("Tools")] + public void TestRemoteSynchronization() + { + var l1 = Path.Combine(TARGETFOLDER, "l1"); + var l2 = Path.Combine(TARGETFOLDER, "l2"); + var l1r = Path.Combine(RESTOREFOLDER, "l1_restore"); + var l2r = Path.Combine(RESTOREFOLDER, "l2_restore"); + + var options = TestOptions; + + var now = DateTime.Now; + GenerateTestData(DATAFOLDER, 10, 3, 3, 1024 * 1024).Wait(); + Console.WriteLine($"Generated test data in {DATAFOLDER} in {DateTime.Now - now}"); + + // Create the directories if they do not exist + foreach (var p in new string[] { l1, l2, l1r, l2r }) + { + if (!SystemIO.IO_OS.DirectoryExists(p)) + SystemIO.IO_OS.DirectoryCreate(p); + } + + // Backup the first level + using (var c = new Controller($"file://{l1}", options, null)) + { + now = DateTime.Now; + var results = c.Backup([DATAFOLDER]); + Assert.AreEqual(0, results.Errors.Count()); + Assert.AreEqual(0, results.Warnings.Count()); + Console.WriteLine($"Backed up {results.AddedFiles} files to {l1} in {DateTime.Now - now}"); + } + + // Call the tool + now = DateTime.Now; + var exe = RemoteSynchronization.Program.Main; + string[] args = [ + $"file://{l1}", $"file://{l2}", + "--global-options", ..options.Select(x => $"{x.Key}={x.Value}"), + // Pass along multi token options to test that the parser won't fail + "--src-options", "ssh-accept-any-fingerprints=true", "ssh-keyfile=/path/to/keyfile", + "--dst-options", "some-other-key=value", "another-key=value2", + "--confirm" + ]; + var async_call = exe(args); + async_call.Wait(); + Assert.AreEqual(0, async_call.Result, "Remote synchronization tool did not return 0."); + Console.WriteLine($"Remote synchronization tool returned 0 in {DateTime.Now - now}"); + + // Verify that the directories are equal + Assert.IsTrue(DirectoriesAndContentsAreEqual(l1, l2), "Synchronized directories are not equal"); + + // Try to restore the first level + options["restore-path"] = l1r; + using (var c = new Controller($"file://{l1}", options, null)) + { + now = DateTime.Now; + var results = c.Restore([Path.Combine(DATAFOLDER, "*")]); + Assert.AreEqual(0, results.Errors.Count()); + Assert.AreEqual(0, results.Warnings.Count()); + Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]} in {DateTime.Now - now}"); + } + Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l1r), "Restored first level files is not equal to original files"); + + // Try to restore the second level + options["restore-path"] = l2r; + using (var c = new Controller($"file://{l2}", options, null)) + { + now = DateTime.Now; + var results = c.Restore([]); + Assert.AreEqual(0, results.Errors.Count()); + Assert.AreEqual(0, results.Warnings.Count()); + Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]} in {DateTime.Now - now}"); + } + Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l2r), "Restored second level files is not equal to original files"); + + // Delete the l2r directory + SystemIO.IO_OS.DirectoryDelete(l2r, true); + + // Delete one file from the l2 backup + var files = Directory.EnumerateFiles(l2).ToList(); + File.Delete(files.First()); + + // Check that the restore fails + options["restore-path"] = l2r; + using (var c = new Controller($"file://{l2}", options, null)) + { + now = DateTime.Now; + try + { + var results = c.Restore([]); + } + catch (RemoteListVerificationException) + { + Console.WriteLine($"Failed (as expected) to restore files to {options["restore-path"]} in {DateTime.Now - now}"); + } + } + + // Run the tool again to copy the missing file + now = DateTime.Now; + async_call = exe(args); + async_call.Wait(); + Assert.AreEqual(0, async_call.Result, "Remote synchronization tool did not return 0."); + Console.WriteLine($"Remote synchronization tool returned 0 in {DateTime.Now - now}"); + + // Try to restore the second level again + options["restore-path"] = l2r; + using (var c = new Controller($"file://{l2}", options, null)) + { + now = DateTime.Now; + var results = c.Restore([]); + Assert.AreEqual(0, results.Errors.Count()); + Assert.AreEqual(0, results.Warnings.Count()); + Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]} in {DateTime.Now - now}"); + } + + Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l2r), "Restored second level files is not equal to original files"); + + // Add some more files to the source + GenerateTestData(Path.Combine(DATAFOLDER, "brand_new_files"), 5, 2, 2, 1024 * 1024).Wait(); + + // Backup the new files to l1 + using (var c = new Controller($"file://{l1}", options, null)) + { + now = DateTime.Now; + var results = c.Backup([DATAFOLDER]); + Assert.AreEqual(0, results.Errors.Count()); + Assert.AreEqual(0, results.Warnings.Count()); + Console.WriteLine($"Backed up {results.AddedFiles} files to {l1} in {DateTime.Now - now}"); + } + + // Run the tool again to copy the new files + now = DateTime.Now; + async_call = exe(args); + async_call.Wait(); + Assert.AreEqual(0, async_call.Result, "Remote synchronization tool did not return 0."); + Console.WriteLine($"Remote synchronization tool returned 0 in {DateTime.Now - now}"); + + // Try to restore the second level again + options["restore-path"] = l2r; + using (var c = new Controller($"file://{l2}", options, null)) + { + now = DateTime.Now; + var results = c.Restore([]); + Assert.AreEqual(0, results.Errors.Count()); + Assert.AreEqual(0, results.Warnings.Count()); + Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]} in {DateTime.Now - now}"); + } + + Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l2r), "Restored second level files is not equal to original files"); + + // Delete the l2r directory + SystemIO.IO_OS.DirectoryDelete(l2r, true); + + // Remove a directory from the source + SystemIO.IO_OS.DirectoryDelete(Path.Combine(DATAFOLDER, "dir_0"), true); + + // Backup the new files to l1 + using (var c = new Controller($"file://{l1}", options, null)) + { + now = DateTime.Now; + var results = c.Backup([DATAFOLDER]); + Assert.AreEqual(0, results.Errors.Count()); + Assert.AreEqual(0, results.Warnings.Count()); + Console.WriteLine($"Backed up {results.AddedFiles} files to {l1} in {DateTime.Now - now}"); + } + + // Compact the backup + using (var c = new Controller($"file://{l1}", options, null)) + { + now = DateTime.Now; + var results = c.Compact(); + Assert.AreEqual(0, results.Errors.Count()); + Assert.AreEqual(0, results.Warnings.Count()); + Console.WriteLine($"Compacted backup in {DateTime.Now - now}"); + } + + // Run the tool again to copy the new files + now = DateTime.Now; + async_call = exe(args); + async_call.Wait(); + Assert.AreEqual(0, async_call.Result, "Remote synchronization tool did not return 0."); + + // Try to restore the second level again + options["restore-path"] = l2r; + using (var c = new Controller($"file://{l2}", options, null)) + { + now = DateTime.Now; + var results = c.Restore([]); + Assert.AreEqual(0, results.Errors.Count()); + Assert.AreEqual(0, results.Warnings.Count()); + Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]} in {DateTime.Now - now}"); + } + + Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l2r), "Restored second level files is not equal to original files"); + + // Delete the l2r directory + SystemIO.IO_OS.DirectoryDelete(l2r, true); + + // Perform a forced synchronization with retention to check that retention doesn't break a restore + now = DateTime.Now; + async_call = exe([.. args, "--force", "--retention"]); + async_call.Wait(); + Assert.AreEqual(0, async_call.Result, "Remote synchronization tool did not return 0."); + + // Try to restore the second level again + options["restore-path"] = l2r; + using (var c = new Controller($"file://{l2}", options, null)) + { + now = DateTime.Now; + var results = c.Restore([]); + Assert.AreEqual(0, results.Errors.Count()); + Assert.AreEqual(0, results.Warnings.Count()); + Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]} in {DateTime.Now - now}"); + } + + Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l2r), "Restored second level files is not equal to original files"); + } + + /// + /// Tests that the remote synchronization tool verifies the contents of the files. + /// + [Test] + [Category("Tools")] + public void TestVerifies() + { + var l1 = Path.Combine(TARGETFOLDER, "l1"); + var l2 = Path.Combine(TARGETFOLDER, "l2"); + + Directory.CreateDirectory(l1); + Directory.CreateDirectory(l2); + + GenerateTestData(l1, 5, 0, 0, 1024).Wait(); + + var filenames = Directory.EnumerateFiles(l1).Take(2).ToList(); + var first_file = filenames.First(); + var second_file = filenames.Skip(1).First(); + + File.Copy(first_file, Path.Combine(l2, Path.GetFileName(first_file))); + File.Copy(second_file, Path.Combine(l2, Path.GetFileName(second_file))); + + // Touch the first file to give it a different timestamp + File.SetLastWriteTime(first_file, DateTime.Now.AddMinutes(1)); + + // Modify the second file to give it different contents, but the same size + var second_file_contents = File.ReadAllBytes(second_file); + second_file_contents[0] = (byte)(second_file_contents[0] + 1); + File.WriteAllBytes(second_file, second_file_contents); + + var args = new string[] { $"file://{l1}", $"file://{l2}", "--confirm", "--verify-contents", "--verify-get-after-put" }; + + var async_call = RemoteSynchronization.Program.Main(args); + var return_code = async_call.ConfigureAwait(false).GetAwaiter().GetResult(); + + Assert.AreEqual(0, return_code, "Remote synchronization tool did not return 0."); + Assert.IsTrue(DirectoriesAndContentsAreEqual(l1, l2), "Synchronized directories are not equal"); + } + + // + // Helper methods + // + + /// + /// Compares two directories and their contents. + /// + /// The first directory. + /// The second directory. + /// `true` if the two directories contain exactly the same files and if the contents of the files are equivalent. `false` otherwise. + public static bool DirectoriesAndContentsAreEqual(string d1, string d2) + { + // Recursively get the files in the two directories + var f1s = Directory.EnumerateFiles(d1, "*", SearchOption.AllDirectories).Select(x => x[(d1.Length + 1)..]).OrderDescending(); + var f2s = Directory.EnumerateFiles(d2, "*", SearchOption.AllDirectories).Select(x => x[(d2.Length + 1)..]).OrderDescending(); + + // If the two directories do not contain the same files, return false + if (!f1s.SequenceEqual(f2s)) + return false; + + // Check that each file pair exist and have the same content + foreach (var f1 in f1s) + { + var f1full = Path.Combine(d1, f1); + if (!File.Exists(f1full)) + return false; + + var f2full = Path.Combine(d2, f1); + if (!File.Exists(f2full)) + return false; + + var c1 = File.ReadAllText(f1full); + var c2 = File.ReadAllText(f2full); + + if (!c1.SequenceEqual(c2)) + return false; + } + + return true; + } + + /// + /// Generates test data in the specified directory. + /// + /// The directory to fill with the generated data. + /// How many files the directory should have. + /// How many subdirectories the directory should have. + /// How deep the number of subdirectories within subdirectories should go. + /// The maximum size of the files to generate. + public static async Task GenerateTestData(string dir, int n_files, int n_dirs, int n_levels, int max_file_size) + { + if (!SystemIO.IO_OS.DirectoryExists(dir)) + SystemIO.IO_OS.DirectoryCreate(dir); + + var fs = Enumerable.Range(0, n_files) + .Select(i => GenerateTestFile(dir, i, max_file_size)); + var ds = n_levels > 0 ? + Enumerable.Range(0, n_dirs) + .Select(i => + { + var subdir = Path.Combine(dir, $"dir_{i}"); + return GenerateTestData(subdir, n_files, n_dirs, n_levels - 1, max_file_size); + }) + : []; + + await Task.WhenAll([.. fs, .. ds]); + } + + public static async Task GenerateTestFile(string dir, int i, int max_file_size) + { + var rnd = new Random(); + var file = Path.Combine(dir, $"file_{i}.txt"); + var size = rnd.Next(1, max_file_size); + var data = new byte[size]; + rnd.NextBytes(data); + await File.WriteAllBytesAsync(file, data); + } + + } +} \ No newline at end of file diff --git a/ReleaseBuilder/Build/Command.Upload.cs b/ReleaseBuilder/Build/Command.Upload.cs index 6f07cbd5c..c311f5bbb 100644 --- a/ReleaseBuilder/Build/Command.Upload.cs +++ b/ReleaseBuilder/Build/Command.Upload.cs @@ -106,8 +106,9 @@ public static partial class Command ///
/// The files to upload /// The runtime configuration + /// The release channels to propagate to /// An awaitable task - public static async Task UploadToS3(IEnumerable files, RuntimeConfig rtcfg) + public static async Task UploadToS3(IEnumerable files, RuntimeConfig rtcfg, IEnumerable propagateTo) { var totalSize = files.Sum(f => new FileInfo(f.Path).Length); Console.WriteLine($"Uploading {files.Count()} files ({Duplicati.Library.Utility.Utility.FormatSizeString(totalSize)}) to S3..."); @@ -141,6 +142,22 @@ public static partial class Command size += filesize; Console.WriteLine($"{size / (double)totalSize * 100:F1}% - Uploaded {file.Name} ({Duplicati.Library.Utility.Utility.FormatSizeString(filesize)})"); } + + foreach (var channel in propagateTo) + { + if (channel == rtcfg.ReleaseInfo.Channel) + continue; + + var target = $"{channel.ToString().ToLowerInvariant()}/latest-v2.manifest"; + var source = $"{rtcfg.ReleaseInfo.Channel.ToString().ToLowerInvariant()}/latest-v2.manifest"; + await Duplicati.Library.Utility.RetryHelper.Retry(() => + client.CopyObjectAsync( + Program.Configuration.ConfigFiles.AwsUploadBucket, + source, + Program.Configuration.ConfigFiles.AwsUploadBucket, + target + ), 3, TimeSpan.FromSeconds(1), CancellationToken.None); + } } /// @@ -151,6 +168,9 @@ public static partial class Command /// An awaitable task public static async Task UploadToGithub(IEnumerable files, RuntimeConfig rtcfg) { + var commithash = (await ProcessHelper.ExecuteWithOutput(["git", "rev-parse", "HEAD"]))?.Trim() + ?? throw new Exception("Failed to get the current commit hash"); + using var httpClient = new HttpClient(); var totalSize = files.Sum(f => new FileInfo(f.Path).Length); Console.WriteLine($"Uploading {files.Count()} files ({Duplicati.Library.Utility.Utility.FormatSizeString(totalSize)}) to Github..."); @@ -166,7 +186,7 @@ public static partial class Command request.Headers.Add("User-Agent", "Duplicati Release Builder v1"); request.Content = JsonContent.Create(new GhReleaseInfo( tag_name: $"v{rtcfg.ReleaseInfo.ReleaseName}", - target_commitish: "master", + target_commitish: commithash, name: $"v{rtcfg.ReleaseInfo.ReleaseName}", body: rtcfg.ChangelogNews, draft: false, @@ -211,7 +231,7 @@ public static partial class Command /// /// The runtime configuration /// An awaitable task - public static async Task ReloadUpdateServer(RuntimeConfig rtcfg) + public static async Task ReloadUpdateServer(RuntimeConfig rtcfg, IEnumerable propagateTo) { using var client = new HttpClient(); var req = new HttpRequestMessage(HttpMethod.Post, "https://updates.duplicati.com/reload"); @@ -222,7 +242,7 @@ public static partial class Command $"{rtcfg.ReleaseInfo.Channel.ToString().ToLowerInvariant()}/latest-v2.json", $"{rtcfg.ReleaseInfo.Channel.ToString().ToLowerInvariant()}/latest-v2.js", $"{rtcfg.ReleaseInfo.Channel.ToString().ToLowerInvariant()}/latest-v2.manifest", - }); + }.Concat(propagateTo.Select(c => $"{c.ToString().ToLowerInvariant()}/latest-v2.manifest"))); var response = await client.SendAsync(req); response.EnsureSuccessStatusCode(); diff --git a/ReleaseBuilder/Build/Command.cs b/ReleaseBuilder/Build/Command.cs index 84c25fb60..d0de3a4b9 100644 --- a/ReleaseBuilder/Build/Command.cs +++ b/ReleaseBuilder/Build/Command.cs @@ -299,6 +299,12 @@ public static partial class Command getDefaultValue: () => false ); + var propagateReleaseOption = new Option( + name: "--propagate-release", + description: "Propagate the release to the next channel", + getDefaultValue: () => false + ); + var command = new System.CommandLine.Command("build", "Builds the packages for a release") { gitStashPushOption, releaseChannelArgument, @@ -322,7 +328,8 @@ public static partial class Command disableUpdateServerReloadOption, disableDiscordAnnounceOption, useHostedBuildsOption, - resumeFromUploadOption + resumeFromUploadOption, + propagateReleaseOption }; command.Handler = CommandHandler.Create(DoBuild); @@ -355,6 +362,7 @@ public static partial class Command /// If forum posting should be disabled /// If hosted builds should be used /// If the process should resume from the upload step + /// If the release should be propagated to the next channel record CommandInput( PackageTarget[] Targets, DirectoryInfo BuildPath, @@ -378,7 +386,8 @@ public static partial class Command bool DisableUpdateServerReload, bool DisableDiscordAnnounce, bool UseHostedBuilds, - bool ResumeFromUpload + bool ResumeFromUpload, + bool PropagateRelease ); static async Task DoBuild(CommandInput input) @@ -619,6 +628,13 @@ public static partial class Command if (input.GitStashPush && !input.ResumeFromUpload) await GitPush.TagAndPush(baseDir, releaseInfo); + // Propagate the release to the next channel, if selected + var nextChannels = new[] { ReleaseChannel.Experimental, ReleaseChannel.Beta, ReleaseChannel.Stable } + .Where(x => x > releaseInfo.Channel) + .Where(x => rtcfg.ReleaseInfo.Channel != ReleaseChannel.Canary) + .Where(x => input.PropagateRelease) + .ToArray(); + if (rtcfg.UseS3Upload || rtcfg.UseGithubUpload) { Console.WriteLine("Build completed, uploading packages ..."); @@ -636,7 +652,7 @@ public static partial class Command .Append(new Upload.UploadFile(packageJson, Path.GetFileName(packageJson))) .Append(new Upload.UploadFile(packageJson, $"latest-v2-{releaseInfo.Version}.json")); - await Upload.UploadToS3(uploads, rtcfg); + await Upload.UploadToS3(uploads, rtcfg, nextChannels); } if (rtcfg.UseGithubUpload) @@ -657,7 +673,7 @@ public static partial class Command if (rtcfg.UseUpdateServerReload) { Console.WriteLine("Release completed, reloading update server ..."); - await Upload.ReloadUpdateServer(rtcfg); + await Upload.ReloadUpdateServer(rtcfg, nextChannels); } if (rtcfg.UseForumPosting) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs new file mode 100644 index 000000000..3ac9c0eef --- /dev/null +++ b/Tools/RemoteSynchronization/Program.cs @@ -0,0 +1,826 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using Duplicati.Library.Common.IO; +using Duplicati.Library.Interface; +using System; +using System.Collections.Generic; +using System.CommandLine; +using System.CommandLine.NamingConventionBinder; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + + +namespace RemoteSynchronization +{ + /// + /// Remote synchronization tool. + /// + public class Program + { + /// + /// Global configuration for the tool. Should be set after parsing the commandline arguments. + /// + private sealed record Config + ( + // Arguments + string Src, + string Dst, + + // Options + bool Confirm, + bool DryRun, + List DstOptions, + bool Force, + List GlobalOptions, + string LogFile, + string LogLevel, + bool ParseArgumentsOnly, + bool Progress, + bool Retention, + int Retry, + List SrcOptions, + bool VerifyContents, + bool VerifyGetAfterPut + ); + + /// + /// The log tag for this tool. + /// + private static readonly string LOGTAG = Duplicati.Library.Logging.Log.LogTagFromType(); + + /// + /// Main entry point for the tool. + /// + /// The commandline arguments + /// 0 on success, -1 on abort, and the number of errors encountered otherwise. + public static async Task Main(string[] args) + { + var arg_src = new Argument(name: "backend_src", description: "The source backend string"); + var arg_dst = new Argument(name: "backend_dst", description: "The destination backend string"); + + var root_cmd = new RootCommand(@"Remote Synchronization Tool + +This tool synchronizes two remote backends. The tool assumes that the intent is +to have the destination match the source. + +If the destination has files that are not in the source, they will be deleted +(or renamed if the retention option is set). + +If the destination has files that are in the source but are different in size or +has a newer timestamp, they will be overwritten by the source files. + +If the force option is set, the destination will be overwritten by the source, +regardless of the state of the files. It will also skip the initial comparison, +and delete (or rename) all files in the destination. + +If the verify option is set, the files will be downloaded and compared after +uploading to ensure that the files are correct. Files that already exist in the +destination will be verified before being overwritten (if they seemingly match). + ") + { + arg_src, + arg_dst, + + new Option(aliases: ["--confirm", "--yes", "-y"], description: "Automatically confirm the operation", getDefaultValue: () => false), + new Option(aliases: ["--dry-run", "-d"], description: "Do not actually write or delete files. If not set here, the global options will be checked", getDefaultValue: () => false), + OptionWithMultipleTokens(aliases: ["--dst-options"], description: "Options for the destination backend. Each option is a key-value pair separated by an equals sign, e.g. --dst-options key1=value1 key2=value2 [default: empty]", getDefaultValue: () => []), + new Option(aliases: ["--force", "-f"], description: "Force the synchronization", getDefaultValue: () => false), + OptionWithMultipleTokens(aliases: ["--global-options"], description: "Global options all backends. May be overridden by backend specific options (src-options, dst-options). Each option is a key-value pair separated by an equals sign, e.g. --global-options key1=value1 key2=value2 [default: empty]", getDefaultValue: () => []), + new Option(aliases: ["--log-file"], description: "The log file to write to. If not set here, global options will be checked [default: \"\"]", getDefaultValue: () => "") { Arity = ArgumentArity.ExactlyOne }, + new Option(aliases: ["--log-level"], description: "The log level to use. If not set here, global options will be checked", getDefaultValue: () => "Information") { Arity = ArgumentArity.ExactlyOne }, + new Option(aliases: ["--parse-arguments-only"], description: "Only parse the arguments and then exit", getDefaultValue: () => false), + new Option(aliases: ["--progress"], description: "Print progress to STDOUT", getDefaultValue: () => false), + new Option(aliases: ["--retention"], description: "Toggles whether to keep old files. Any deletes will be renames instead", getDefaultValue: () => false), + new Option(aliases: ["--retry"], description: "Number of times to retry on errors", getDefaultValue: () => 3) { Arity = ArgumentArity.ExactlyOne }, + OptionWithMultipleTokens(aliases: ["--src-options"], description: "Options for the source backend. Each option is a key-value pair separated by an equals sign, e.g. --src-options key1=value1 key2=value2 [default: empty]", getDefaultValue: () => []), + new Option(aliases: ["--verify-contents"], description: "Verify the contents of the files to decide whether the pre-existing destination files should be overwritten", getDefaultValue: () => false), + new Option(aliases: ["--verify-get-after-put"], description: "Verify the files after uploading them to ensure that they were uploaded correctly", getDefaultValue: () => false), + }; + + root_cmd.Handler = CommandHandler.Create((string backend_src, string backend_dst, Config config) => + { + var config_with_args = config with { Dst = backend_dst, Src = backend_src }; + + return Run(config_with_args); + }); + + return await root_cmd.InvokeAsync(args); + } + + /// + /// The main logic of the tool. + /// + /// The connection string for the source backend. + /// The connection string for the destination backend. + /// Various options for the tool + /// The return code for the main entry; 0 on success. + private static async Task Run(Config config) + { + // Unpack and parse the multi token options + Dictionary global_options = config.GlobalOptions + .Select(x => x.Split("=")) + .ToDictionary(x => x[0], x => x[1]); + + // Parse the log level + var log_level_parsed = Enum.TryParse(config.LogLevel, true, out var log_level_enum); + log_level_enum = log_level_parsed ? log_level_enum : Duplicati.Library.Logging.LogMessageType.Information; + + using var console_sink = new Duplicati.CommandLine.ConsoleOutput(Console.Out, global_options); + using var multi_sink = new Duplicati.Library.Main.ControllerMultiLogTarget(console_sink, log_level_enum, null); + + // Parse the log file + // The log file sink doesn't have to be disposed, as the multi_sink will take care of it + Duplicati.Library.Logging.StreamLogDestination? log_file_sink = null; + if (!string.IsNullOrEmpty(config.LogFile)) + { + string log_file_dir = SystemIO.IO_OS.PathGetDirectoryName(config.LogFile); + if (!string.IsNullOrEmpty(log_file_dir) && !SystemIO.IO_OS.DirectoryExists(log_file_dir)) + SystemIO.IO_OS.DirectoryCreate(log_file_dir); + log_file_sink = new Duplicati.Library.Logging.StreamLogDestination(config.LogFile); + } + multi_sink.AddTarget(log_file_sink, log_level_enum, null); + + // Start the logging scope + using var _ = Duplicati.Library.Logging.Log.StartScope(multi_sink, log_level_enum); + + Dictionary src_opts = config.SrcOptions + .Select(x => x.Split("=")) + .ToDictionary(x => x[0], x => x[1]); + + Dictionary dst_opts = config.DstOptions + .Select(x => x.Split("=")) + .ToDictionary(x => x[0], x => x[1]); + + // Merge the global options into the source and destination options. The global options will be overridden by the source and destination options. + foreach (var x in global_options) + { + if (!src_opts.ContainsKey(x.Key)) + src_opts[x.Key] = x.Value; + if (!dst_opts.ContainsKey(x.Key)) + dst_opts[x.Key] = x.Value; + } + + // Check if we only had to parse the arguments + if (config.ParseArgumentsOnly) + { + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Arguments parsed successfully; exiting."); + return 0; + } + + // Load the backends + static Exception throw_message(string target) + { + var message = $"The {target} backend does not support streaming operations."; + var ex = new Exception(message); + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", ex, message); + return ex; + } + + using var b1 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(config.Src, src_opts); + var b1s = b1 as IStreamingBackend ?? throw throw_message("source"); + + using var b2 = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(config.Dst, dst_opts); + var b2s = b2 as IStreamingBackend ?? throw throw_message("destination"); + + // Prepare the operations + var (to_copy, to_delete, to_verify) = PrepareFileLists(b1s, b2s, config); + + // Verify the files if requested. If the files are not verified, they will be deleted and copied again. + long verified = 0, failed_verify = 0; + if (config.VerifyContents) + { + // As this is a potentially slow operation, ask for confirmation of the verification) + if (!config.Confirm) + { + Console.WriteLine($"This will verify {to_verify.Count()} files before copying them. Do you want to continue? [y/N]"); + var response = Console.ReadLine(); + if (!response?.Equals("y", StringComparison.CurrentCultureIgnoreCase) ?? true) + { + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Aborted"); + return -1; + } + } + + var not_verified = await VerifyAsync(b1s, b2s, to_verify, config); + failed_verify = not_verified.Count(); + verified = to_verify.Count() - failed_verify; + + if (not_verified.Any()) + { + Duplicati.Library.Logging.Log.WriteWarningMessage(LOGTAG, "rsync", null, + "{0} files failed verification. They will be deleted and copied again.", + not_verified.Count()); + to_delete = to_delete.Concat(not_verified); + to_copy = to_copy.Concat(not_verified); + } + } + + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "The remote synchronization plan is to {0} {1} files from {2}, then copy {3} files from {4} to {2}.", + config.Retention ? "rename" : "delete", + to_delete.Count(), b2s.DisplayName, to_copy.Count(), b1s.DisplayName); + + // As this is a potentially destructive operation, ask for confirmation + if (!config.Confirm) + { + var delete_rename = config.Retention ? "Rename" : "Delete"; + Console.WriteLine($"This will perform the following actions (in order):"); + Console.WriteLine($" - {delete_rename} {to_delete.Count()} files from {config.Dst}"); + Console.WriteLine($" - Copy {to_copy.Count()} files from {config.Src} to {config.Dst}"); + Console.WriteLine(); + Console.WriteLine("Do you want to continue? [y/N]"); + + var response = Console.ReadLine(); + if (!response?.Equals("y", StringComparison.CurrentCultureIgnoreCase) ?? true) + { + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", "Aborted"); + return -1; + } + } + + // Delete or rename the files that are not needed + long renamed = 0, deleted = 0; + if (config.Retention) + { + renamed = await RenameAsync(b2, to_delete, config); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Renamed {0} files in {1}", renamed, b2s.DisplayName); + } + else + { + deleted = await DeleteAsync(b2s, to_delete, config); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Deleted {0} files from {1}", deleted, b2s.DisplayName); + } + + // Copy the files + var (copied, copy_errors) = await CopyAsync(b1s, b2s, to_copy, config); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Copied {0} files from {1} to {2}", copied, b1s, b2s); + + // If there are still errors, retry a few times + if (copy_errors.Any()) + { + Duplicati.Library.Logging.Log.WriteWarningMessage(LOGTAG, "rsync", null, + "Could not copy {0} files.", copy_errors.Count()); + if (config.Retry > 0) + { + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Retrying {0} more times to copy the {1} files that failed", + config.Retry, copy_errors.Count()); + for (int i = 0; i < config.Retry; i++) + { + await Task.Delay(5000); // Wait 5 seconds before retrying + (copied, copy_errors) = await CopyAsync(b1s, b2s, copy_errors, config); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Copied {0} files from {1} to {2}", copied, b1s, b2s); + if (!copy_errors.Any()) + break; + } + } + + if (copy_errors.Any()) + { + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", null, + "Could not copy {0} files. Not retrying any more.", copy_errors.Count()); + return copy_errors.Count(); + } + } + + // Results reporting + if (verified > 0) + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Verified {0} files in {1} that didn't need to be copied", + verified, b2s.DisplayName); + if (failed_verify > 0) + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Failed to verify {0} files in {1}, which were then attempted to be copied", + failed_verify, b2s.DisplayName); + if (copied > 0) + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Copied {0} files from {1} to {2}", copied, b1s.DisplayName, b2s.DisplayName); + if (deleted > 0) + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Deleted {0} files from {1}", deleted, b2s.DisplayName); + if (renamed > 0) + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Renamed {0} files in {1}", renamed, b2s.DisplayName); + + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "rsync", + "Remote synchronization completed successfully"); + + return 0; + } + + // TODO have concurrency parameters: uploaders, downloaders + // TODO low memory mode, where things aren't kept in memory. Maybe utilize SQLite? + // TODO For convenience, have the option to launch a "duplicati test" on the destination backend after the synchronization + // TODO Save hash to minimize redownload + // TODO Duplicati Results + + /// + /// Copies the files from one backend to another. + /// The files are copied one by one, and each file is verified after uploading if the verify flag is set. + /// + /// The source backend. + /// The destination backend. + /// The files that will be copied. + /// The parsed configuration for the tool. + /// A tuple holding the number of succesful copies and a List of the files that failed. + private static async Task<(long, IEnumerable)> CopyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files, Config config) + { + long successful_copies = 0; + List errors = []; + using var s_src = new MemoryStream(); + using var s_dst = new MemoryStream(); + long i = 0, n = files.Count(); + var sw_get_src = new System.Diagnostics.Stopwatch(); + var sw_put_dst = new System.Diagnostics.Stopwatch(); + var sw_get_dst = new System.Diagnostics.Stopwatch(); + var sw_get_cmp = new System.Diagnostics.Stopwatch(); + + foreach (var f in files) + { + if (config.Progress) + Console.Write($"\rCopying: {i}/{n}"); + + Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", + "Copying {0} from {1} to {2}", f.Name, b_src.DisplayName, b_dst.DisplayName); + + try + { + sw_get_src.Start(); + await b_src.GetAsync(f.Name, s_src, CancellationToken.None); + sw_get_src.Stop(); + if (config.DryRun) + { + Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", + "Would write {0} bytes of {1} to {2}", + Duplicati.Library.Utility.Utility.FormatSizeString(s_src.Length), + f.Name, b_dst.DisplayName); + } + else + { + sw_put_dst.Start(); + await b_dst.PutAsync(f.Name, s_src, CancellationToken.None); + sw_put_dst.Stop(); + if (config.VerifyGetAfterPut) + { + sw_get_dst.Start(); + await b_dst.GetAsync(f.Name, s_dst, CancellationToken.None); + sw_get_dst.Stop(); + + sw_get_cmp.Start(); + string? err_string = null; + if (s_src.Length != s_dst.Length) + { + err_string = $"The sizes of the files do not match: {s_src.Length} != {s_dst.Length}."; + } + + if (!s_src.ToArray().SequenceEqual(s_dst.ToArray())) + { + err_string = (err_string is null ? "" : err_string + " ") + "The contents of the files do not match."; + } + sw_get_cmp.Stop(); + + if (err_string is not null) + { + throw new Exception(err_string); + } + } + } + + successful_copies++; + } + catch (Exception e) + { + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, + "Error copying {0}: {1}", f.Name, e.Message); + errors.Add(f); + } + finally + { + s_src.SetLength(0); + s_dst.SetLength(0); + i++; + + // Stop any running timers + sw_get_src.Stop(); + sw_put_dst.Stop(); + sw_get_dst.Stop(); + sw_get_cmp.Stop(); + } + } + + if (config.Progress) + Console.WriteLine($"\rCopying: {n}/{n}"); + + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", + "Copy | Get source: {0} ms, Put destination: {1} ms, Get destination: {2} ms, Get compare: {3} ms", + TimeSpan.FromMilliseconds(sw_get_src.ElapsedMilliseconds), + TimeSpan.FromMilliseconds(sw_put_dst.ElapsedMilliseconds), + TimeSpan.FromMilliseconds(sw_get_dst.ElapsedMilliseconds), + TimeSpan.FromMilliseconds(sw_get_cmp.ElapsedMilliseconds)); + + return (successful_copies, errors); + } + + /// + /// Deletes the files from a backend. + /// + /// The backend to delete the files from. + /// The files to delete. + /// The parsed configuration for the tool. + /// The number of successful deletions. + private static async Task DeleteAsync(IStreamingBackend b, IEnumerable files, Config config) + { + long successful_deletes = 0; + long i = 0, n = files.Count(); + using var timer = new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Delete operation"); + + foreach (var f in files) + { + if (n > 1 && config.Progress) + { + Console.Write($"\rDeleting: {i}/{n}"); + } + + Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", + "Deleting {0} from {1}", f.Name, b.DisplayName); + + try + { + if (config.DryRun) + { + Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", + "Would delete {0} from {1}", f.Name, b.DisplayName); + } + else + { + await b.DeleteAsync(f.Name, CancellationToken.None); + } + successful_deletes++; + } + catch (Exception e) + { + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, + "Error deleting {0}: {1}", f.Name, e.Message); + } + + i++; + } + + if (config.Progress) + Console.WriteLine($"\rDeleting: {n}/{n}"); + return successful_deletes; + } + + /// + /// Creates an option that allows multiple tokens and multiple arguments per token. + /// + /// The aliases for the option. + /// The description for the option. + /// The created option. + private static Option> OptionWithMultipleTokens(string[] aliases, string description, Func> getDefaultValue) + { + return new Option>(aliases: aliases, description: description, getDefaultValue: getDefaultValue) + { + Arity = ArgumentArity.OneOrMore, + AllowMultipleArgumentsPerToken = true + }; + } + + /// + /// Prepares the lists of files to copy, delete and verify. + /// The files to copy are the files that are not in the destination, have a different size or have a more recent modification date. + /// The files to delete are the files that are found in the destination but not found in the source. + /// The files to verify are the files that are found in both the source and the destination, and that have the same size and modification date. + /// + /// The source backend. + /// The destination backend. + /// The parsed configuration for the tool. + /// A tuple of Lists each holding the files to copy, delete and verify. + private static (IEnumerable, IEnumerable, IEnumerable) PrepareFileLists(IStreamingBackend b_src, IStreamingBackend b_dst, Config config) + { + IEnumerable files_src, files_dst; + + using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | List source")) + files_src = b_src.List(); + + using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | List destination")) + files_dst = b_dst.List(); + + // Shortcut for force + if (config.Force) + { + return (files_src, files_dst, []); + } + + // Shortcut for empty destination + if (!files_dst.Any()) + { + return (files_src, [], []); + } + + Dictionary lookup_src, lookup_dst; + using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | Build lookup for source and destination")) + { + lookup_src = files_src.ToDictionary(x => x.Name); + lookup_dst = files_dst.ToDictionary(x => x.Name); + } + + var to_copy = new List(); + var to_delete = new HashSet(); + var to_verify = new List(); + + // Find all of the files in src that are not in dst, have a different size or have a more recent modification date + using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | Check the files that are present in source against destination")) + foreach (var f_src in files_src) + { + if (lookup_dst.TryGetValue(f_src.Name, out var f_dst)) + { + if (f_src.Size != f_dst.Size || f_src.LastModification > f_dst.LastModification) + { + // The file is different, so we need to copy it + to_copy.Add(f_src); + to_delete.Add(f_dst.Name); + } + else + { + // The file seems to be the same, so we need to verify it if the user wants to + to_verify.Add(f_src); + } + } + else + { + // The file is not in the destination, so we need to copy it + to_copy.Add(f_src); + } + } + + // Find all of the files in dst that are not in src + using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | Check the files that are present in destination against source")) + foreach (var f_dst in files_dst) + { + if (to_delete.Contains(f_dst.Name)) + continue; + + if (!lookup_src.ContainsKey(f_dst.Name)) + { + to_delete.Add(f_dst.Name); + } + } + + List to_delete_lookedup; + using (new Duplicati.Library.Logging.Timer(LOGTAG, "rsync", "Prepare | Lookup the files to delete")) + to_delete_lookedup = [.. to_delete.Select(x => lookup_dst[x])]; + + return (to_copy, to_delete_lookedup, to_verify); + } + + /// + /// Renames the files in a backend. + /// The renaming is done by deleting the file and re-uploading it with a new name. + /// + /// The backend to rename the files in. + /// The files to rename. + /// The parsed configuration for the tool. + /// The number of successful renames. + private static async Task RenameAsync(IBackend b, IEnumerable files, Config config) + { + long successful_renames = 0; + string prefix = $"{System.DateTime.UtcNow:yyyyMMddHHmmss}.old"; + using var downloaded = new MemoryStream(); + long i = 0, n = files.Count(); + + switch (b) + { + case IStreamingBackend sb: + { + var sw_get_src = new System.Diagnostics.Stopwatch(); + var sw_put_dst = new System.Diagnostics.Stopwatch(); + var sw_del_src = new System.Diagnostics.Stopwatch(); + + foreach (var f in files) + { + if (config.Progress) + Console.Write($"\rRenaming: {i}/{n}"); + + Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", + "Renaming {0} to {1}.{0} by deleting and re-uploading {2} bytes to {3}", + f.Name, prefix, + Duplicati.Library.Utility.Utility.FormatSizeString(downloaded.Length), + sb.DisplayName); + + try + { + sw_get_src.Start(); + await sb.GetAsync(f.Name, downloaded, CancellationToken.None); + sw_get_src.Stop(); + + if (config.DryRun) + { + Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", + "Would rename {0} to {1}.{0} by deleting and re-uploading {2} bytes to {3}", + f.Name, prefix, + Duplicati.Library.Utility.Utility.FormatSizeString(downloaded.Length), + sb.DisplayName); + } + else + { + sw_put_dst.Start(); + await sb.PutAsync($"{prefix}.{f.Name}", downloaded, CancellationToken.None); + sw_put_dst.Stop(); + sw_del_src.Start(); + await sb.DeleteAsync(f.Name, CancellationToken.None); + sw_del_src.Stop(); + } + successful_renames++; + } + catch (Exception e) + { + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, + "Error renaming {0}: {1}", f.Name, e.Message); + } + finally + { + // Reset the stream + downloaded.SetLength(0); + + // Stop any running timers + sw_get_src.Stop(); + sw_put_dst.Stop(); + sw_del_src.Stop(); + } + + i++; + } + + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", + "Rename | Get source: {0} ms, Put destination: {1} ms, Delete source: {2} ms", + TimeSpan.FromMilliseconds(sw_get_src.ElapsedMilliseconds), + TimeSpan.FromMilliseconds(sw_put_dst.ElapsedMilliseconds), + TimeSpan.FromMilliseconds(sw_del_src.ElapsedMilliseconds)); + + if (config.Progress) + Console.WriteLine($"\rRenaming: {n}/{n}"); + + return successful_renames; + } + case IRenameEnabledBackend rb: + { + var sw = new System.Diagnostics.Stopwatch(); + + foreach (var f in files) + { + if (config.Progress) + Console.Write($"\rRenaming: {i}/{n}"); + + Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", + "Renaming {0} to {1}.{0} by calling Rename on {2}", + f.Name, prefix, rb.DisplayName); + + try + { + if (config.DryRun) + { + Duplicati.Library.Logging.Log.WriteDryrunMessage(LOGTAG, "rsync", + "Would rename {0} to {1}.{0} by calling Rename on {2}", + f.Name, prefix, rb.DisplayName); + } + else + { + sw.Start(); + await rb.RenameAsync(f.Name, $"{prefix}.{f.Name}", CancellationToken.None); + sw.Stop(); + } + successful_renames++; + } + catch (Exception e) + { + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, + "Error renaming {0}: {1}", f.Name, e.Message); + } + finally + { + // Ensure the timer is stopped + sw.Stop(); + } + + i++; + } + + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", + "Rename: {0} ms", + TimeSpan.FromMilliseconds(sw.ElapsedMilliseconds)); + + if (config.Progress) + Console.WriteLine($"\rRenaming: {n}/{n}"); + + return successful_renames; + } + default: + throw new NotSupportedException("The backend does not support renaming"); + } + } + + /// + /// Verifies the files in the destination backend. + /// The verification is done by downloading the files from the destination backend and comparing them to the source files. + /// + /// The source backend. + /// The destination backend. + /// The files to verify. + /// The parsed configuration for the tool. + /// A list of the files that failed verification. + private static async Task> VerifyAsync(IStreamingBackend b_src, IStreamingBackend b_dst, IEnumerable files, Config config) + { + var errors = new List(); + using var s_src = new MemoryStream(); + using var s_dst = new MemoryStream(); + long i = 0, n = files.Count(); + var sw_get = new System.Diagnostics.Stopwatch(); + var sw_cmp = new System.Diagnostics.Stopwatch(); + + foreach (var f in files) + { + if (config.Progress) + Console.Write($"\rVerifying: {i}/{n}"); + + Duplicati.Library.Logging.Log.WriteVerboseMessage(LOGTAG, "rsync", + "Verifying {0} by downloading and comparing {1} bytes from {2} and {3}", + f.Name, + Duplicati.Library.Utility.Utility.FormatSizeString(s_src.Length), + b_dst.DisplayName, b_src.DisplayName); + + try + { + // Get both files + sw_get.Start(); + var fs = b_src.GetAsync(f.Name, s_src, CancellationToken.None); + var ds = b_dst.GetAsync(f.Name, s_dst, CancellationToken.None); + await Task.WhenAll(fs, ds); + sw_get.Stop(); + + // Compare the contents + sw_cmp.Start(); + if (s_src.Length != s_dst.Length || !s_src.ToArray().SequenceEqual(s_dst.ToArray())) + { + errors.Add(f); + } + sw_cmp.Stop(); + } + catch (Exception e) + { + errors.Add(f); + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", e, + "Error during verification of {0}: {1}", f.Name, e.Message); + } + finally + { + // Reset the streams + s_src.SetLength(0); + s_dst.SetLength(0); + + // Stop any running timers + sw_get.Stop(); + sw_cmp.Stop(); + } + + i++; + } + + Duplicati.Library.Logging.Log.WriteProfilingMessage(LOGTAG, "rsync", + "Verify | Get: {0} ms, Compare: {1} ms", + TimeSpan.FromMilliseconds(sw_get.ElapsedMilliseconds), + TimeSpan.FromMilliseconds(sw_cmp.ElapsedMilliseconds)); + + if (config.Progress) + Console.WriteLine($"\rVerifying: {n}/{n}"); + + return errors; + } + + } +} diff --git a/Tools/RemoteSynchronization/RemoteSynchronization.csproj b/Tools/RemoteSynchronization/RemoteSynchronization.csproj new file mode 100644 index 000000000..6cd7b2704 --- /dev/null +++ b/Tools/RemoteSynchronization/RemoteSynchronization.csproj @@ -0,0 +1,16 @@ + + + + Exe + net8.0 + disable + enable + + + + + + + + +