From d98e1096e3e17d863f983aab9bb03b2ade2114eb Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 8 Jul 2025 07:50:39 +0200 Subject: [PATCH 1/7] Fixed case where the arguments passed to the remote synchronization tool might emit everything past the second equals sign --- Tools/RemoteSynchronization/Program.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 9c553a5c4..cbc988f1f 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -171,11 +171,11 @@ destination will be verified before being overwritten (if they seemingly match). Dictionary src_opts = config.SrcOptions .Select(x => x.Split("=")) - .ToDictionary(x => x[0], x => x[1]); + .ToDictionary(x => x[0], x => string.Join("=", x.Skip(1))); Dictionary dst_opts = config.DstOptions .Select(x => x.Split("=")) - .ToDictionary(x => x[0], x => x[1]); + .ToDictionary(x => x[0], x => string.Join("=", x.Skip(1))); // Merge the global options into the source and destination options. The global options will be overridden by the source and destination options. foreach (var x in global_options) From bb2847eb552886cffdca1e4d5fadf4330ef5c136 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 8 Jul 2025 07:51:43 +0200 Subject: [PATCH 2/7] Specified that the ToolTests relates to the remote synchronization tool --- Duplicati/UnitTest/ToolTests.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Duplicati/UnitTest/ToolTests.cs b/Duplicati/UnitTest/ToolTests.cs index 8d8a8935c..6bff53d4b 100644 --- a/Duplicati/UnitTest/ToolTests.cs +++ b/Duplicati/UnitTest/ToolTests.cs @@ -42,7 +42,7 @@ namespace Duplicati.UnitTest /// Tests that the remote synchronization tool doesn't do anything when the dry run option is used. /// [Test] - [Category("Tools")] + [Category("Tools/RemoteSynchronization")] public void TestDryRun() { var l1 = Path.Combine(TARGETFOLDER, "l1"); @@ -67,7 +67,7 @@ namespace Duplicati.UnitTest /// Tests that the remote synchronization tool works with an empty source to an empty destination. /// [Test] - [Category("Tools")] + [Category("Tools/RemoteSynchronization")] public void TestEmptySourceAndDestination() { var l1 = Path.Combine(TARGETFOLDER, "empty_src"); @@ -89,7 +89,7 @@ namespace Duplicati.UnitTest /// Test that remote synchronizing an empty source to a non-empty destination deletes the destination files. /// [Test] - [Category("Tools")] + [Category("Tools/RemoteSynchronization")] public void TestEmptySourceDeletesDestination() { var l1 = Path.Combine(TARGETFOLDER, "empty_src"); @@ -113,7 +113,7 @@ namespace Duplicati.UnitTest /// Test that remote synchronizing an empty source to a non-empty destination renames the destination files when the `--retention` option is used. /// [Test] - [Category("Tools")] + [Category("Tools/RemoteSynchronization")] public void TestEmptySourceRenamesDestination() { var l1 = Path.Combine(TARGETFOLDER, "empty_src"); @@ -147,6 +147,7 @@ namespace Duplicati.UnitTest /// Tests passing all arguments to the main method of the remote synchronization tool. /// [Test] + [Category("Tools/RemoteSynchronization")] public void TestMainMethodParsesArgumentsCorrectly() { string[][] testCases = @@ -192,7 +193,7 @@ namespace Duplicati.UnitTest /// Tests the original inded use of the remote synchronization tool on an empty destination. /// [Test] - [Category("Tools")] + [Category("Tools/RemoteSynchronization")] public void TestRemoteSynchronization() { var l1 = Path.Combine(TARGETFOLDER, "l1"); @@ -413,7 +414,7 @@ namespace Duplicati.UnitTest /// Tests that the remote synchronization tool verifies the contents of the files. /// [Test] - [Category("Tools")] + [Category("Tools/RemoteSynchronization")] public void TestVerifies() { var l1 = Path.Combine(TARGETFOLDER, "l1"); From 0475d896cc259fd54bc7925dc4158761fbd635bf Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 8 Jul 2025 07:52:22 +0200 Subject: [PATCH 3/7] Reduce test data size in remote synchronization tool test to speed it up --- Duplicati/UnitTest/ToolTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/UnitTest/ToolTests.cs b/Duplicati/UnitTest/ToolTests.cs index 6bff53d4b..277da9956 100644 --- a/Duplicati/UnitTest/ToolTests.cs +++ b/Duplicati/UnitTest/ToolTests.cs @@ -204,7 +204,7 @@ namespace Duplicati.UnitTest var options = TestOptions; var now = DateTime.Now; - GenerateTestData(DATAFOLDER, 10, 3, 3, 1024 * 1024).Wait(); + GenerateTestData(DATAFOLDER, 5, 2, 2, 1024).Wait(); Console.WriteLine($"Generated test data in {DATAFOLDER} in {DateTime.Now - now}"); // Create the directories if they do not exist @@ -310,7 +310,7 @@ namespace Duplicati.UnitTest Assert.IsTrue(DirectoriesAndContentsAreEqual(DATAFOLDER, l2r), "Restored second level files is not equal to original files"); // Add some more files to the source - GenerateTestData(Path.Combine(DATAFOLDER, "brand_new_files"), 5, 2, 2, 1024 * 1024).Wait(); + GenerateTestData(Path.Combine(DATAFOLDER, "brand_new_files"), 5, 2, 2, 1024).Wait(); // Backup the new files to l1 using (var c = new Controller($"file://{l1}", options, null)) From 4adfb4681db0ad7fd863ece893088cf13766063e Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 8 Jul 2025 07:54:03 +0200 Subject: [PATCH 4/7] Removed restore argument to make the calls to restore consistent --- Duplicati/UnitTest/ToolTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/UnitTest/ToolTests.cs b/Duplicati/UnitTest/ToolTests.cs index 277da9956..11f3c4f0e 100644 --- a/Duplicati/UnitTest/ToolTests.cs +++ b/Duplicati/UnitTest/ToolTests.cs @@ -248,7 +248,7 @@ namespace Duplicati.UnitTest using (var c = new Controller($"file://{l1}", options, null)) { now = DateTime.Now; - var results = c.Restore([Path.Combine(DATAFOLDER, "*")]); + var results = c.Restore([]); Assert.AreEqual(0, results.Errors.Count()); Assert.AreEqual(0, results.Warnings.Count()); Console.WriteLine($"Restored {results.RestoredFiles} files to {options["restore-path"]} in {DateTime.Now - now}"); From fbbea78aeb23501e620d001544620e2e2b8c49fa Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 8 Jul 2025 08:29:09 +0200 Subject: [PATCH 5/7] Moved options parsing to its own method, which now includes double checking by trying to reconstruct the parsed option. --- Tools/RemoteSynchronization/Program.cs | 41 +++++++++++++++++++------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index cbc988f1f..5ff4bae32 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -143,9 +143,7 @@ destination will be verified before being overwritten (if they seemingly match). private static async Task Run(Config config) { // Unpack and parse the multi token options - Dictionary global_options = config.GlobalOptions - .Select(x => x.Split("=")) - .ToDictionary(x => x[0], x => x[1]); + Dictionary global_options = ParseOptions(config.GlobalOptions); // Parse the log level var log_level_parsed = Enum.TryParse(config.LogLevel, true, out var log_level_enum); @@ -169,13 +167,8 @@ destination will be verified before being overwritten (if they seemingly match). // Start the logging scope using var _ = Duplicati.Library.Logging.Log.StartScope(multi_sink, log_level_enum); - Dictionary src_opts = config.SrcOptions - .Select(x => x.Split("=")) - .ToDictionary(x => x[0], x => string.Join("=", x.Skip(1))); - - Dictionary dst_opts = config.DstOptions - .Select(x => x.Split("=")) - .ToDictionary(x => x[0], x => string.Join("=", x.Skip(1))); + Dictionary src_opts = ParseOptions(config.SrcOptions); + Dictionary dst_opts = ParseOptions(config.DstOptions); // Merge the global options into the source and destination options. The global options will be overridden by the source and destination options. foreach (var x in global_options) @@ -529,6 +522,34 @@ destination will be verified before being overwritten (if they seemingly match). }; } + /// + /// Parses the options from a list of strings. + /// Each option should be in the format "key=value". If the value contains spaces, + /// it should be enclosed in quotes, e.g. "key=\"value with spaces\"". + /// + /// The list of string options to parse + /// A dictionary with the parsed options, where the key is the option name and the value is the option value. + /// If an option was not parsed correctly. + private static Dictionary ParseOptions(IEnumerable options) + { + var result = options + .Select(x => x.Split('=')) + .ToDictionary(x => x[0], x => string.Join("=", x.Skip(1))); + + // Double check that the options are valid by reconstructing them from the dictionary + foreach (var opt in result.Select(x => $"{x.Key}={x.Value}")) + { + if (!options.Contains(opt)) + { + Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", null, + "The source option '{0}' is not valid. Please check the syntax.", opt); + throw new ArgumentException($"The source option '{opt}' has not been parsed correctly."); + } + } + + return result; + } + /// /// Prepares the lists of files to copy, delete and verify. /// The files to copy are the files that are not in the destination, have a different size or have a more recent modification date. From 59582cdb787d33335f52c759d9cd4eff79423553 Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 8 Jul 2025 08:29:27 +0200 Subject: [PATCH 6/7] Added additional test cases for argument parsing in the remote synchronization tests --- Duplicati/UnitTest/ToolTests.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Duplicati/UnitTest/ToolTests.cs b/Duplicati/UnitTest/ToolTests.cs index 11f3c4f0e..b83929495 100644 --- a/Duplicati/UnitTest/ToolTests.cs +++ b/Duplicati/UnitTest/ToolTests.cs @@ -177,6 +177,18 @@ namespace Duplicati.UnitTest "--src-options", "somesrckey=somesrcvalue", "anothersrckey=anothersrcvalue", "--dst-options", "somedstkey=somedstvalue", "anotherdstkey=anotherdstvalue" ], + [ + "source", "destination", "--parse-arguments-only", + "--global-options", "somekey=somevalue=with=extra=equals", "anotherkey=anothervalue", + "--src-options", "somekey=somevalue=with=extra=equals", "anotherkey=anothervalue", + "--dst-options", "somekey=somevalue=with=extra=equals", "anotherkey=anothervalue" + ], + [ + "source", "destination", "--parse-arguments-only", + "--global-options", "somekey=\"some value with spaces\"", "anotherkey=anothervalue", + "--src-options", "somekey=\"some value with spaces\"", "anotherkey=anothervalue", + "--dst-options", "somekey=\"some value with spaces\"", "anotherkey=anothervalue" + ] }; foreach (var args in testCases) From a5d730e06b414d41fb7cbd05ea9a8279aa323e9a Mon Sep 17 00:00:00 2001 From: Carl Johnsen Date: Tue, 8 Jul 2025 15:27:36 +0200 Subject: [PATCH 7/7] Changed from explicit type to var --- Tools/RemoteSynchronization/Program.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 5ff4bae32..c713e1443 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -143,7 +143,7 @@ destination will be verified before being overwritten (if they seemingly match). private static async Task Run(Config config) { // Unpack and parse the multi token options - Dictionary global_options = ParseOptions(config.GlobalOptions); + var global_options = ParseOptions(config.GlobalOptions); // Parse the log level var log_level_parsed = Enum.TryParse(config.LogLevel, true, out var log_level_enum); @@ -167,8 +167,8 @@ destination will be verified before being overwritten (if they seemingly match). // Start the logging scope using var _ = Duplicati.Library.Logging.Log.StartScope(multi_sink, log_level_enum); - Dictionary src_opts = ParseOptions(config.SrcOptions); - Dictionary dst_opts = ParseOptions(config.DstOptions); + var src_opts = ParseOptions(config.SrcOptions); + var dst_opts = ParseOptions(config.DstOptions); // Merge the global options into the source and destination options. The global options will be overridden by the source and destination options. foreach (var x in global_options)