From 62814c1c6ef498e707f4b82516dc8005d3337c1f Mon Sep 17 00:00:00 2001 From: Kenneth Hsu Date: Sat, 8 Aug 2020 12:35:51 -0700 Subject: [PATCH 01/17] Add new test categories to local test runner script. We forgot to add these when introducing the new test categories. --- pipeline/start.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipeline/start.sh b/pipeline/start.sh index 9267229be..b6aa81f89 100755 --- a/pipeline/start.sh +++ b/pipeline/start.sh @@ -6,6 +6,6 @@ SCRIPT_DIR="$( cd "$(dirname "$0")" ; pwd -P )" ${ROOT_DIR}/pipeline/jobs/build_job.sh ${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories BulkNormal ${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories BulkNoSize -${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories SVNDataLong,SVNData,RecoveryTool +${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories SVNDataLong,SVNData,RecoveryTool,ProblematicPath ${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories Border -${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories Filter,Targeted,Purge,Serialization,WebApi,Utility,UriUtility,IO,ImportExport,Disruption,RestoreHandler,RepairHandler,DeleteHandler +${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories Filter,Targeted,Purge,Serialization,WebApi,Utility,UriUtility,IO,ImportExport,Disruption,RestoreHandler,RepairHandler,DeleteHandler,Controller From 5f94cc19a74a5c372dece9e1b656404bf43ab57d Mon Sep 17 00:00:00 2001 From: Dean Ferreyra Date: Sun, 30 Aug 2020 15:52:53 -0700 Subject: [PATCH 02/17] Fix restore GUI handling of folders with * or ? Add new filter syntax, prefixing with an "at sign"; i.e., `@...`, that disables glob expansion, treating `*` and `?` wildcard characters as literal characters. In the GUI, change the restore file picker to prefix paths with `@` when expanding a node to avoid mistaking literal `*` or `?` characters for wildcards characters and triggering the following error: ``` Filter for list-folder-contents must be a path prefix with no wildcards Parameter name: filter ``` This fixes #4300. --- Duplicati/Library/Utility/FilterExpression.cs | 11 +++++++++++ Duplicati/Library/Utility/Utility.cs | 9 +++++++++ .../ngax/scripts/directives/restoreFilePicker.js | 5 ++++- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Utility/FilterExpression.cs b/Duplicati/Library/Utility/FilterExpression.cs index 0c70f7ef7..576d3bfdb 100644 --- a/Duplicati/Library/Utility/FilterExpression.cs +++ b/Duplicati/Library/Utility/FilterExpression.cs @@ -119,6 +119,13 @@ namespace Duplicati.Library.Utility this.Filter = filter.Substring(1, filter.Length - 2); this.Regexp = GetFilterGroupRegex(this.Filter); } + else if (filter.StartsWith("@", StringComparison.Ordinal)) + { + // Take filter literally; i.e., don't treat wildcard characters as glogging characters + this.Type = FilterType.Simple; + this.Filter = filter.Substring(1); + this.Regexp = new Regex(Utility.ConvertLiteralToRegExp(this.Filter), REGEXP_OPTIONS); + } else { this.Type = (filter.Contains(MULTIPLE_WILDCARD) || filter.Contains(SINGLE_WILDCARD)) ? FilterType.Wildcard : FilterType.Simple; @@ -150,6 +157,10 @@ namespace Duplicati.Library.Utility { return filterString.Substring(1, filterString.Length - 2); } + else if (filterString.StartsWith("@", StringComparison.Ordinal)) + { + return Utility.ConvertLiteralToRegExp(filterString.Substring(1)); + } else { return Utility.ConvertGlobbingToRegExp(filterString); diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 703b6ea29..e8280231f 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -164,6 +164,15 @@ namespace Duplicati.Library.Utility return globexp; } + /// + /// Convert literal path to the equivalent regular expression. + /// + public static string ConvertLiteralToRegExp(string literalPath) + { + // Escape all special characters + return Regex.Escape(literalPath); + } + /// /// Returns a list of all files found in the given folder. /// The search is recursive. diff --git a/Duplicati/Server/webroot/ngax/scripts/directives/restoreFilePicker.js b/Duplicati/Server/webroot/ngax/scripts/directives/restoreFilePicker.js index ef142f6bc..3e71e8648 100644 --- a/Duplicati/Server/webroot/ngax/scripts/directives/restoreFilePicker.js +++ b/Duplicati/Server/webroot/ngax/scripts/directives/restoreFilePicker.js @@ -35,7 +35,10 @@ backupApp.directive('restoreFilePicker', function() { if (!node.children && !node.loading) { node.loading = true; - AppService.get('/backup/' + $scope.ngBackupId + '/files/' + encodeURIComponent(node.id) + '?prefix-only=false&folder-contents=true&time=' + encodeURIComponent($scope.ngTimestamp) + '&filter=' + encodeURIComponent(node.id)).then(function(data) { + // Prefix filter with "@" to prevent Duplicati from + // mistaking literal '*' and '?' characters in paths + // for glob wildcard characters + AppService.get('/backup/' + $scope.ngBackupId + '/files/' + encodeURIComponent(node.id) + '?prefix-only=false&folder-contents=true&time=' + encodeURIComponent($scope.ngTimestamp) + '&filter=' + encodeURIComponent('@' + node.id)).then(function(data) { var children = [] for(var n in data.data.Files) From aa50b1dfbf34cdf2732d994bd97ec001817c082d Mon Sep 17 00:00:00 2001 From: Dean Ferreyra Date: Sun, 30 Aug 2020 20:02:19 -0700 Subject: [PATCH 03/17] Fix typo in comment --- Duplicati/Library/Utility/FilterExpression.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Utility/FilterExpression.cs b/Duplicati/Library/Utility/FilterExpression.cs index 576d3bfdb..a0850309e 100644 --- a/Duplicati/Library/Utility/FilterExpression.cs +++ b/Duplicati/Library/Utility/FilterExpression.cs @@ -121,7 +121,8 @@ namespace Duplicati.Library.Utility } else if (filter.StartsWith("@", StringComparison.Ordinal)) { - // Take filter literally; i.e., don't treat wildcard characters as glogging characters + // Take filter literally; i.e., don't treat wildcard + // characters as globbing characters this.Type = FilterType.Simple; this.Filter = filter.Substring(1); this.Regexp = new Regex(Utility.ConvertLiteralToRegExp(this.Filter), REGEXP_OPTIONS); From fd76fb32ef84a91a8374d516a18c8e10a04a2867 Mon Sep 17 00:00:00 2001 From: Dean Ferreyra Date: Mon, 31 Aug 2020 21:25:18 -0700 Subject: [PATCH 04/17] Fix FilterEntry for @ entries to be of type Regexp --- Duplicati/Library/Utility/FilterExpression.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Utility/FilterExpression.cs b/Duplicati/Library/Utility/FilterExpression.cs index a0850309e..bb896e168 100644 --- a/Duplicati/Library/Utility/FilterExpression.cs +++ b/Duplicati/Library/Utility/FilterExpression.cs @@ -123,7 +123,7 @@ namespace Duplicati.Library.Utility { // Take filter literally; i.e., don't treat wildcard // characters as globbing characters - this.Type = FilterType.Simple; + this.Type = FilterType.Regexp; this.Filter = filter.Substring(1); this.Regexp = new Regex(Utility.ConvertLiteralToRegExp(this.Filter), REGEXP_OPTIONS); } From 5f972c2437195a195cc376bb7f73b51d9d6c4475 Mon Sep 17 00:00:00 2001 From: Dean Ferreyra Date: Tue, 1 Sep 2020 21:45:27 -0700 Subject: [PATCH 05/17] Make FilterEntry of type Simple for @ again Switch back to using `FilterType.Simple` for `@` filters to fix the restore picker again. In LocalDatabase.cs, replace code that was explicitly looking for the presence of wildcard characters to instead trigger off of `FilterType.Wildcard`. This fixes the "find" command. Change FilterEntry.ToString() to prefix Simple filters with `@` so that if simple filters with wildcard characters make the round-trip through a string, their `FilterType.Simple` type is preserved. --- Duplicati/Library/Main/Database/LocalDatabase.cs | 2 +- Duplicati/Library/Utility/FilterExpression.cs | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index f46694ef3..fd1048d20 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -1190,7 +1190,7 @@ ORDER BY var args = new List(); foreach (var f in ((Library.Utility.FilterExpression)filter).GetSimpleList()) { - if (f.Contains('*') || f.Contains('?')) + if (type == FilterType.Wildcard) { sb.Append(@"""Path"" LIKE ? OR "); args.Add(f.Replace('*', '%').Replace('?', '_')); diff --git a/Duplicati/Library/Utility/FilterExpression.cs b/Duplicati/Library/Utility/FilterExpression.cs index bb896e168..fc4eb1745 100644 --- a/Duplicati/Library/Utility/FilterExpression.cs +++ b/Duplicati/Library/Utility/FilterExpression.cs @@ -123,7 +123,7 @@ namespace Duplicati.Library.Utility { // Take filter literally; i.e., don't treat wildcard // characters as globbing characters - this.Type = FilterType.Regexp; + this.Type = FilterType.Simple; this.Filter = filter.Substring(1); this.Regexp = new Regex(Utility.ConvertLiteralToRegExp(this.Filter), REGEXP_OPTIONS); } @@ -296,6 +296,8 @@ namespace Duplicati.Library.Utility return "[" + this.Filter + "]"; case FilterType.Group: return "{" + this.Filter + "}"; + case FilterType.Simple: + return "@" + this.Filter; default: return this.Filter; } From b8501dfd77f20400df827b20381d94f5e6c205bc Mon Sep 17 00:00:00 2001 From: Dean Ferreyra Date: Tue, 1 Sep 2020 23:48:06 -0700 Subject: [PATCH 06/17] Add additional handling for @ filters Change "find" and "restore" commands to not prefix `@` file filters with `*` and to not suffix `@` folder filters with `*` since wildcards cannot be used with `@` filters. --- Duplicati/CommandLine/Commands.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Duplicati/CommandLine/Commands.cs b/Duplicati/CommandLine/Commands.cs index 0e60ffc71..13fc11e6c 100644 --- a/Duplicati/CommandLine/Commands.cs +++ b/Duplicati/CommandLine/Commands.cs @@ -239,7 +239,8 @@ namespace Duplicati.CommandLine // Prefix all filenames with "*/" so we search all folders var specialArgChars = new[] { '*', '?', Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }; return argList.Select(arg => arg.IndexOfAny(specialArgChars) < 0 - && !arg.StartsWith("[", StringComparison.Ordinal) ? "*" + Util.DirectorySeparatorString + arg : arg).ToList(); + && !arg.StartsWith("[", StringComparison.Ordinal) + && !arg.StartsWith("@", StringComparison.Ordinal) ? "*" + Util.DirectorySeparatorString + arg : arg).ToList(); } public static int List(TextWriter outwriter, Action setup, List args, Dictionary options, Library.Utility.IFilter filter) @@ -267,7 +268,7 @@ namespace Duplicati.CommandLine options["version"] = v.ToString(); } } - else if (args[0].IndexOfAny(new char[] { '*', '?' }) < 0 && !args[0].StartsWith("[", StringComparison.Ordinal)) + else if (args[0].IndexOfAny(new char[] { '*', '?' }) < 0 && !args[0].StartsWith("[", StringComparison.Ordinal) && !args[0].StartsWith("@", StringComparison.Ordinal)) { try { @@ -452,7 +453,7 @@ namespace Duplicati.CommandLine // suffix all folders with "*" so we restore all contents in the folder for (var ix = 0; ix < args.Count; ix++) - if (args[ix].IndexOfAny(new char[] { '*', '?' }) < 0 && !args[ix].StartsWith("[", StringComparison.Ordinal) && args[ix].EndsWith(Util.DirectorySeparatorString, StringComparison.Ordinal)) + if (args[ix].IndexOfAny(new char[] { '*', '?' }) < 0 && !args[ix].StartsWith("[", StringComparison.Ordinal) && args[ix].EndsWith(Util.DirectorySeparatorString, StringComparison.Ordinal) && !args[ix].StartsWith("@", StringComparison.Ordinal)) args[ix] += "*"; using(var output = new ConsoleOutput(outwriter, options)) From ff91c47f68622f0efe44328eacf7afae9ff6c615 Mon Sep 17 00:00:00 2001 From: Dean Ferreyra Date: Thu, 3 Sep 2020 22:59:24 -0700 Subject: [PATCH 07/17] Fix restore GUI for filters with wildcard characters Change RestoreController.js to handle the case where the selected restore paths contain literal wildcard characters. For files, prefix with `@`. For directories, convert to regular expression filters so that we can preserve the literal wildcard characters and also support the equivalent of a globbing `*` suffix. --- .../scripts/controllers/RestoreController.js | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/Duplicati/Server/webroot/ngax/scripts/controllers/RestoreController.js b/Duplicati/Server/webroot/ngax/scripts/controllers/RestoreController.js index 0ff98a378..650d414fa 100644 --- a/Duplicati/Server/webroot/ngax/scripts/controllers/RestoreController.js +++ b/Duplicati/Server/webroot/ngax/scripts/controllers/RestoreController.js @@ -346,10 +346,27 @@ backupApp.controller('RestoreController', function ($rootScope, $scope, $routePa var paths = []; for(var n in $scope.Selected) { var item = $scope.Selected[n]; - if (item.substr(item.length - 1) == dirsep) - paths.push(item + '*'); - else - paths.push(item); + if (item.indexOf('*') >= 0 || item.indexOf('?')) { + // Handle paths with literal wildcard characters + // specially + if (item.substr(item.length - 1) == dirsep) { + // Switch to a regular expression filter so we can + // preserve the literal wildcard characters and + // also support the equivalent of a globbing '*' + // suffix. + + // Escape regular expression metacharacters + var itemRegex = item.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + paths.push('[' + itemRegex + '.*]'); + } else { + paths.push('@' + item); + } + } else { + if (item.substr(item.length - 1) == dirsep) + paths.push(item + '*'); + else + paths.push(item); + } } if (paths.length > 0) From 071f7ed7b011b783a400bca43b054430cb13e221 Mon Sep 17 00:00:00 2001 From: Dean Ferreyra Date: Thu, 3 Sep 2020 23:55:27 -0700 Subject: [PATCH 08/17] Fix restore GUI for filters with wildcard characters Fix bug in line to detect literal wildcard characters. --- .../webroot/ngax/scripts/controllers/RestoreController.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Server/webroot/ngax/scripts/controllers/RestoreController.js b/Duplicati/Server/webroot/ngax/scripts/controllers/RestoreController.js index 650d414fa..549b5451b 100644 --- a/Duplicati/Server/webroot/ngax/scripts/controllers/RestoreController.js +++ b/Duplicati/Server/webroot/ngax/scripts/controllers/RestoreController.js @@ -346,7 +346,7 @@ backupApp.controller('RestoreController', function ($rootScope, $scope, $routePa var paths = []; for(var n in $scope.Selected) { var item = $scope.Selected[n]; - if (item.indexOf('*') >= 0 || item.indexOf('?')) { + if (item.indexOf('*') >= 0 || item.indexOf('?') >= 0) { // Handle paths with literal wildcard characters // specially if (item.substr(item.length - 1) == dirsep) { From f6e79da78d09257340dddcb7e04264ae3b1901ff Mon Sep 17 00:00:00 2001 From: Dean Ferreyra Date: Fri, 4 Sep 2020 01:01:56 -0700 Subject: [PATCH 09/17] Support globbing for "@" filters in "find" and "restore" For `@` filters, support normal behavior of "find" and "restore" where they automatically prefix or suffix some file name arguments with `*`. --- Duplicati/CommandLine/Commands.cs | 93 ++++++++++++++++++++++++++----- 1 file changed, 80 insertions(+), 13 deletions(-) diff --git a/Duplicati/CommandLine/Commands.cs b/Duplicati/CommandLine/Commands.cs index 13fc11e6c..dde5aa9e1 100644 --- a/Duplicati/CommandLine/Commands.cs +++ b/Duplicati/CommandLine/Commands.cs @@ -19,6 +19,7 @@ using System; using System.Linq; using System.Collections.Generic; using System.IO; +using System.Text.RegularExpressions; using Duplicati.Library.Common.IO; namespace Duplicati.CommandLine @@ -234,15 +235,86 @@ namespace Duplicati.CommandLine } } - public static List PrefixArgsWithAsterisk(IList argList) + /// + /// For bare file names with no wildcards, replace with the + /// equivalent of prefixing with "*/" so we search all folders. + /// + public static IEnumerable PrefixArgsWithAsterisk(IEnumerable argList) => + argList.Select(PrefixArgWithAsterisk).ToList(); + + /// + /// For bare file names with no wildcards, replace with the + /// equivalent of prefixing with "*/" so we search all folders. + /// + private static string PrefixArgWithAsterisk(string arg) { - // Prefix all filenames with "*/" so we search all folders - var specialArgChars = new[] { '*', '?', Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }; - return argList.Select(arg => arg.IndexOfAny(specialArgChars) < 0 - && !arg.StartsWith("[", StringComparison.Ordinal) - && !arg.StartsWith("@", StringComparison.Ordinal) ? "*" + Util.DirectorySeparatorString + arg : arg).ToList(); + var containsSeparators = ContainsDirectorySeparators(arg); + var containsWildcards = ContainsWildcards(arg); + + if (!containsSeparators && arg.StartsWith("@", StringComparison.Ordinal)) + { + // Convert to Regexp filter and prefix with ".*/" + return $"[.*{Regex.Escape(Util.DirectorySeparatorString + arg.Substring(1))}]"; + } + else if (!containsSeparators && !containsWildcards && !arg.StartsWith("[", StringComparison.Ordinal)) + { + // Prefix files with "*/" + return "*" + Util.DirectorySeparatorString + arg; + } + else + { + return arg; + } } + /// + /// For folders, replace with the equivalent of suffixing with "/*" + /// so we restore contents in the folder. + /// + public static IEnumerable SuffixArgsWithAsterisk(IEnumerable argList) => + argList.Select(SuffixArgWithAsterisk).ToList(); + + /// + /// For folders, replace with the equivalent of suffixing with "/*" + /// so we restore contents in the folder. + /// + private static string SuffixArgWithAsterisk(string arg) + { + var containsWildcards = ContainsWildcards(arg); + var endsWithSeparator = arg.EndsWith(Util.DirectorySeparatorString); + + if (endsWithSeparator && arg.StartsWith("@", StringComparison.Ordinal)) + { + // Convert to Regexp filter and suffix with "/.*" + return $"[{Regex.Escape(arg.Substring(1))}.*]"; + } + else if (endsWithSeparator && !containsWildcards && !arg.StartsWith("[", StringComparison.Ordinal)) + { + // Suffix with "/*" + return arg + "*"; + } + else + { + return arg; + } + } + + private static readonly char[] pathSeparatorsCharacters = new[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }; + + /// + /// Returns true if contains directory separator characters. + /// + public static bool ContainsDirectorySeparators(string s) => + s.IndexOfAny(pathSeparatorsCharacters) >= 0; + + private static readonly char[] wildcardCharacters = new[] { '*', '?' }; + + /// + /// Returns true if contains wildcard characters. + /// + public static bool ContainsWildcards(string s) => + s.IndexOfAny(wildcardCharacters) >= 0; + public static int List(TextWriter outwriter, Action setup, List args, Dictionary options, Library.Utility.IFilter filter) { filter = filter ?? new Duplicati.Library.Utility.FilterExpression(); @@ -283,7 +355,7 @@ namespace Duplicati.CommandLine } } - args = PrefixArgsWithAsterisk(args); + args = PrefixArgsWithAsterisk(args).ToList(); // Support for not adding the --auth-username if possible string dbpath; @@ -449,12 +521,7 @@ namespace Duplicati.CommandLine bool controlFiles = Library.Utility.Utility.ParseBoolOption(options, "control-files"); options.Remove("control-files"); - args = PrefixArgsWithAsterisk(args); - - // suffix all folders with "*" so we restore all contents in the folder - for (var ix = 0; ix < args.Count; ix++) - if (args[ix].IndexOfAny(new char[] { '*', '?' }) < 0 && !args[ix].StartsWith("[", StringComparison.Ordinal) && args[ix].EndsWith(Util.DirectorySeparatorString, StringComparison.Ordinal) && !args[ix].StartsWith("@", StringComparison.Ordinal)) - args[ix] += "*"; + args = SuffixArgsWithAsterisk(PrefixArgsWithAsterisk(args)).ToList(); using(var output = new ConsoleOutput(outwriter, options)) using (var i = new Library.Main.Controller(backend, options, output)) From 3a66668e6ad6f8ed218f9226409fc64513670a3c Mon Sep 17 00:00:00 2001 From: Dean Ferreyra Date: Fri, 4 Sep 2020 07:25:31 -0700 Subject: [PATCH 10/17] Support globbing for "@" filters in "find" and "restore" Remove unneeded calls to `ToList()`. --- Duplicati/CommandLine/Commands.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/CommandLine/Commands.cs b/Duplicati/CommandLine/Commands.cs index dde5aa9e1..d83de888f 100644 --- a/Duplicati/CommandLine/Commands.cs +++ b/Duplicati/CommandLine/Commands.cs @@ -240,7 +240,7 @@ namespace Duplicati.CommandLine /// equivalent of prefixing with "*/" so we search all folders. /// public static IEnumerable PrefixArgsWithAsterisk(IEnumerable argList) => - argList.Select(PrefixArgWithAsterisk).ToList(); + argList.Select(PrefixArgWithAsterisk); /// /// For bare file names with no wildcards, replace with the @@ -272,7 +272,7 @@ namespace Duplicati.CommandLine /// so we restore contents in the folder. /// public static IEnumerable SuffixArgsWithAsterisk(IEnumerable argList) => - argList.Select(SuffixArgWithAsterisk).ToList(); + argList.Select(SuffixArgWithAsterisk); /// /// For folders, replace with the equivalent of suffixing with "/*" From 5c2fb96f5f45990a3552ddfc3a4c248aecbeabec Mon Sep 17 00:00:00 2001 From: Dean Ferreyra Date: Fri, 4 Sep 2020 09:54:33 -0700 Subject: [PATCH 11/17] Support globbing for "@" filters in "find" and "restore" Change some comments. --- Duplicati/CommandLine/Commands.cs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Duplicati/CommandLine/Commands.cs b/Duplicati/CommandLine/Commands.cs index d83de888f..37fbf5a5d 100644 --- a/Duplicati/CommandLine/Commands.cs +++ b/Duplicati/CommandLine/Commands.cs @@ -236,15 +236,15 @@ namespace Duplicati.CommandLine } /// - /// For bare file names with no wildcards, replace with the - /// equivalent of prefixing with "*/" so we search all folders. + /// For bare file names with no wildcards, replace argument with + /// the equivalent of prefixing with "*/" so we search all folders. /// public static IEnumerable PrefixArgsWithAsterisk(IEnumerable argList) => argList.Select(PrefixArgWithAsterisk); /// - /// For bare file names with no wildcards, replace with the - /// equivalent of prefixing with "*/" so we search all folders. + /// For bare file names with no wildcards, return argument with + /// the equivalent of prefixing with "*/" so we search all folders. /// private static string PrefixArgWithAsterisk(string arg) { @@ -268,15 +268,15 @@ namespace Duplicati.CommandLine } /// - /// For folders, replace with the equivalent of suffixing with "/*" - /// so we restore contents in the folder. + /// For folders, replace argument with the equivalent of suffixing + /// with "*" so we restore contents in the folder. /// public static IEnumerable SuffixArgsWithAsterisk(IEnumerable argList) => argList.Select(SuffixArgWithAsterisk); /// - /// For folders, replace with the equivalent of suffixing with "/*" - /// so we restore contents in the folder. + /// For folders, return argument with the equivalent of suffixing + /// with "*" so we restore contents in the folder. /// private static string SuffixArgWithAsterisk(string arg) { @@ -285,12 +285,12 @@ namespace Duplicati.CommandLine if (endsWithSeparator && arg.StartsWith("@", StringComparison.Ordinal)) { - // Convert to Regexp filter and suffix with "/.*" + // Convert to Regexp filter and suffix with ".*" return $"[{Regex.Escape(arg.Substring(1))}.*]"; } else if (endsWithSeparator && !containsWildcards && !arg.StartsWith("[", StringComparison.Ordinal)) { - // Suffix with "/*" + // Suffix with "*" return arg + "*"; } else From 5436d16c20c969ae95a3006ec93a2e69837c6d36 Mon Sep 17 00:00:00 2001 From: Dean Ferreyra Date: Fri, 4 Sep 2020 16:54:49 -0700 Subject: [PATCH 12/17] Fix restore GUI for filters with wildcard characters In RestoreController.js, remove the branch on whether the path contains wildcard characters and instead use code that works for all cases. --- .../scripts/controllers/RestoreController.js | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/Duplicati/Server/webroot/ngax/scripts/controllers/RestoreController.js b/Duplicati/Server/webroot/ngax/scripts/controllers/RestoreController.js index 549b5451b..346c64fdf 100644 --- a/Duplicati/Server/webroot/ngax/scripts/controllers/RestoreController.js +++ b/Duplicati/Server/webroot/ngax/scripts/controllers/RestoreController.js @@ -346,26 +346,21 @@ backupApp.controller('RestoreController', function ($rootScope, $scope, $routePa var paths = []; for(var n in $scope.Selected) { var item = $scope.Selected[n]; - if (item.indexOf('*') >= 0 || item.indexOf('?') >= 0) { - // Handle paths with literal wildcard characters - // specially - if (item.substr(item.length - 1) == dirsep) { - // Switch to a regular expression filter so we can - // preserve the literal wildcard characters and - // also support the equivalent of a globbing '*' - // suffix. + if (item.substr(item.length - 1) == dirsep) { + // To support the possibility of encountering paths + // with literal wildcard characters, but also being + // able to add the globbing "*" suffix, use a regular + // expression filter - // Escape regular expression metacharacters - var itemRegex = item.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - paths.push('[' + itemRegex + '.*]'); - } else { - paths.push('@' + item); - } + // Escape regular expression metacharacters + var itemRegex = item.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + // Add "globbing" suffix + paths.push('[' + itemRegex + '.*]'); } else { - if (item.substr(item.length - 1) == dirsep) - paths.push(item + '*'); - else - paths.push(item); + // To support the possibility of encountering paths + // with literal wildcard characters, create a literal + // filter + paths.push('@' + item); } } From 7d13d9ed81d3d6bb375ba2d08a2d5690b175e83d Mon Sep 17 00:00:00 2001 From: Kenneth Hsu Date: Sat, 5 Sep 2020 10:25:03 -0700 Subject: [PATCH 13/17] Add tests for directory names with wildcards. --- Duplicati/UnitTest/ProblematicPathTests.cs | 124 +++++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/Duplicati/UnitTest/ProblematicPathTests.cs b/Duplicati/UnitTest/ProblematicPathTests.cs index 7d1f05a3a..4009748f4 100644 --- a/Duplicati/UnitTest/ProblematicPathTests.cs +++ b/Duplicati/UnitTest/ProblematicPathTests.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using Duplicati.Library.Common; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Main; @@ -21,6 +22,129 @@ namespace Duplicati.UnitTest } } + [Test] + [Category("ProblematicPath")] + public void DirectoriesWithWildcards() + { + if (Platform.IsClientWindows) + { + return; + } + + const string file = "file"; + List directories = new List(); + + const string asterisk = "*"; + string dirWithAsterisk = Path.Combine(this.DATAFOLDER, asterisk); + SystemIO.IO_OS.DirectoryCreate(dirWithAsterisk); + WriteFile(SystemIO.IO_OS.PathCombine(dirWithAsterisk, file), new byte[] {0}); + directories.Add(dirWithAsterisk); + + const string questionMark = "?"; + string dirWithQuestionMark = Path.Combine(this.DATAFOLDER, questionMark); + SystemIO.IO_OS.DirectoryCreate(dirWithQuestionMark); + WriteFile(SystemIO.IO_OS.PathCombine(dirWithQuestionMark, file), new byte[] {1}); + directories.Add(dirWithQuestionMark); + + const string dir = "dir"; + string normalDir = Path.Combine(this.DATAFOLDER, dir); + SystemIO.IO_OS.DirectoryCreate(normalDir); + WriteFile(SystemIO.IO_OS.PathCombine(normalDir, file), new byte[] {2}); + directories.Add(normalDir); + + // Backup all files. + Dictionary options = new Dictionary(this.TestOptions); + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + IBackupResults backupResults = c.Backup(new[] {this.DATAFOLDER}); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + } + + // Restore all files. + Dictionary restoreOptions = new Dictionary(options) {["restore-path"] = this.RESTOREFOLDER}; + using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null)) + { + IRestoreResults restoreResults = c.Restore(null); + Assert.AreEqual(0, restoreResults.Errors.Count()); + Assert.AreEqual(0, restoreResults.Warnings.Count()); + + foreach (string directory in directories) + { + string directoryName = SystemIO.IO_OS.PathGetFileName(directory); + foreach (string expectedFilePath in SystemIO.IO_OS.EnumerateFiles(directory)) + { + string fileName = SystemIO.IO_OS.PathGetFileName(expectedFilePath); + string restoredFilePath = SystemIO.IO_OS.PathCombine(this.RESTOREFOLDER, directoryName, fileName); + Assert.IsTrue(TestUtils.CompareFiles(expectedFilePath, restoredFilePath, expectedFilePath, false)); + } + } + + // List results using * should return a match for each directory. + IListResults listResults = c.List(SystemIO.IO_OS.PathCombine(dirWithAsterisk, file)); + Assert.AreEqual(0, listResults.Errors.Count()); + Assert.AreEqual(0, listResults.Warnings.Count()); + Assert.AreEqual(directories.Count, listResults.Files.Count()); + + // List results using ? should return 2 matches, one for the directory with '*' and one for the directory with '?'. + listResults = c.List(SystemIO.IO_OS.PathCombine(dirWithQuestionMark, file)); + Assert.AreEqual(0, listResults.Errors.Count()); + Assert.AreEqual(0, listResults.Warnings.Count()); + Assert.AreEqual(2, listResults.Files.Count()); + } + + SystemIO.IO_OS.DirectoryDelete(this.RESTOREFOLDER, true); + + // Restore one file at a time using the verbatim identifier. + foreach (string directory in directories) + { + foreach (string expectedFilePath in SystemIO.IO_OS.EnumerateFiles(directory)) + { + using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null)) + { + string verbatimFilePath = "@" + expectedFilePath; + + // Verify that list result using verbatim identifier contains only one file. + IListResults listResults = c.List(verbatimFilePath); + Assert.AreEqual(0, listResults.Errors.Count()); + Assert.AreEqual(0, listResults.Warnings.Count()); + Assert.AreEqual(1, listResults.Files.Count()); + Assert.AreEqual(expectedFilePath, listResults.Files.Single().Path); + + IRestoreResults restoreResults = c.Restore(new[] {verbatimFilePath}); + Assert.AreEqual(0, restoreResults.Errors.Count()); + Assert.AreEqual(0, restoreResults.Warnings.Count()); + + string fileName = SystemIO.IO_OS.PathGetFileName(expectedFilePath); + string restoredFilePath = SystemIO.IO_OS.PathCombine(this.RESTOREFOLDER, fileName); + Assert.IsTrue(TestUtils.CompareFiles(expectedFilePath, restoredFilePath, expectedFilePath, false)); + + SystemIO.IO_OS.FileDelete(restoredFilePath); + } + } + } + + // Backup with asterisk in include filter should include all directories. + FilterExpression filter = new FilterExpression(dirWithAsterisk); + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + IBackupResults backupResults = c.Backup(new[] {this.DATAFOLDER}, filter); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + Assert.AreEqual(directories.Count, backupResults.ExaminedFiles); + } + + // Backup with verbatim asterisk in include filter should include one directory. + filter = new FilterExpression("@" + SystemIO.IO_OS.PathCombine(dirWithAsterisk, file)); + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + IBackupResults backupResults = c.Backup(new[] {this.DATAFOLDER}, filter); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + Assert.AreEqual(1, backupResults.ExaminedFiles); + } + } + [Test] [Category("ProblematicPath")] public void ExcludeProblematicPaths() From 27da5a4dc7f4b02aef68ad26cdbc187b4fc1dc65 Mon Sep 17 00:00:00 2001 From: Dean Ferreyra Date: Sun, 6 Sep 2020 14:55:39 -0700 Subject: [PATCH 14/17] Include Windows in tests for directory names with wildcards --- Duplicati/UnitTest/ProblematicPathTests.cs | 56 +++++++++++++++------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/Duplicati/UnitTest/ProblematicPathTests.cs b/Duplicati/UnitTest/ProblematicPathTests.cs index 4009748f4..56cc8b954 100644 --- a/Duplicati/UnitTest/ProblematicPathTests.cs +++ b/Duplicati/UnitTest/ProblematicPathTests.cs @@ -26,25 +26,45 @@ namespace Duplicati.UnitTest [Category("ProblematicPath")] public void DirectoriesWithWildcards() { - if (Platform.IsClientWindows) - { - return; - } - const string file = "file"; List directories = new List(); + // Keep expected match counts since they'll differ between + // Linux and Windows. + var questionMarkWildcardShouldMatchCount = 0; + var verbatimAsteriskShouldMatchCount = 0; + const string asterisk = "*"; string dirWithAsterisk = Path.Combine(this.DATAFOLDER, asterisk); - SystemIO.IO_OS.DirectoryCreate(dirWithAsterisk); - WriteFile(SystemIO.IO_OS.PathCombine(dirWithAsterisk, file), new byte[] {0}); - directories.Add(dirWithAsterisk); + // Windows does not support literal asterisks in paths. + if (!Platform.IsClientWindows) + { + SystemIO.IO_OS.DirectoryCreate(dirWithAsterisk); + WriteFile(SystemIO.IO_OS.PathCombine(dirWithAsterisk, file), new byte[] {0}); + directories.Add(dirWithAsterisk); + questionMarkWildcardShouldMatchCount++; + verbatimAsteriskShouldMatchCount++; + } const string questionMark = "?"; string dirWithQuestionMark = Path.Combine(this.DATAFOLDER, questionMark); - SystemIO.IO_OS.DirectoryCreate(dirWithQuestionMark); - WriteFile(SystemIO.IO_OS.PathCombine(dirWithQuestionMark, file), new byte[] {1}); - directories.Add(dirWithQuestionMark); + // Windows does not support literal question marks in paths. + if (!Platform.IsClientWindows) + { + SystemIO.IO_OS.DirectoryCreate(dirWithQuestionMark); + WriteFile(SystemIO.IO_OS.PathCombine(dirWithQuestionMark, file), new byte[] { 1 }); + directories.Add(dirWithQuestionMark); + questionMarkWildcardShouldMatchCount++; + } + + // Include at least one single character directory in Windows + // for a '?' wildcard can match on + const string singleCharacterDir = "X"; + string dirWithSingleCharacter = Path.Combine(this.DATAFOLDER, singleCharacterDir); + SystemIO.IO_OS.DirectoryCreate(dirWithSingleCharacter); + WriteFile(SystemIO.IO_OS.PathCombine(dirWithSingleCharacter, file), new byte[] { 1 }); + directories.Add(dirWithSingleCharacter); + questionMarkWildcardShouldMatchCount++; const string dir = "dir"; string normalDir = Path.Combine(this.DATAFOLDER, dir); @@ -86,11 +106,14 @@ namespace Duplicati.UnitTest Assert.AreEqual(0, listResults.Warnings.Count()); Assert.AreEqual(directories.Count, listResults.Files.Count()); - // List results using ? should return 2 matches, one for the directory with '*' and one for the directory with '?'. listResults = c.List(SystemIO.IO_OS.PathCombine(dirWithQuestionMark, file)); Assert.AreEqual(0, listResults.Errors.Count()); Assert.AreEqual(0, listResults.Warnings.Count()); - Assert.AreEqual(2, listResults.Files.Count()); + // List results using ? should return 3 matches in Linux, + // one for the directory with '*' and one for the directory + // with '?', plus one for directory 'X'; but should return + // 1 matches in Windows just for directory 'X'. + Assert.AreEqual(questionMarkWildcardShouldMatchCount, listResults.Files.Count()); } SystemIO.IO_OS.DirectoryDelete(this.RESTOREFOLDER, true); @@ -134,16 +157,17 @@ namespace Duplicati.UnitTest Assert.AreEqual(directories.Count, backupResults.ExaminedFiles); } - // Backup with verbatim asterisk in include filter should include one directory. + // Backup with verbatim asterisk in include filter should include + // one directory in Linux and zero directories in Windows. filter = new FilterExpression("@" + SystemIO.IO_OS.PathCombine(dirWithAsterisk, file)); using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { IBackupResults backupResults = c.Backup(new[] {this.DATAFOLDER}, filter); Assert.AreEqual(0, backupResults.Errors.Count()); Assert.AreEqual(0, backupResults.Warnings.Count()); - Assert.AreEqual(1, backupResults.ExaminedFiles); + Assert.AreEqual(verbatimAsteriskShouldMatchCount, backupResults.ExaminedFiles); } - } + } [Test] [Category("ProblematicPath")] From 536ce69668658ad21b0f86a3e55a7897d17acfbe Mon Sep 17 00:00:00 2001 From: Dean Ferreyra Date: Sun, 6 Sep 2020 21:30:46 -0700 Subject: [PATCH 15/17] Address review comments from @warwickmm --- Duplicati/CommandLine/Commands.cs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Duplicati/CommandLine/Commands.cs b/Duplicati/CommandLine/Commands.cs index 37fbf5a5d..1f5b0b1d7 100644 --- a/Duplicati/CommandLine/Commands.cs +++ b/Duplicati/CommandLine/Commands.cs @@ -21,6 +21,7 @@ using System.Collections.Generic; using System.IO; using System.Text.RegularExpressions; using Duplicati.Library.Common.IO; +using Duplicati.Library.Utility; namespace Duplicati.CommandLine { @@ -254,7 +255,7 @@ namespace Duplicati.CommandLine if (!containsSeparators && arg.StartsWith("@", StringComparison.Ordinal)) { // Convert to Regexp filter and prefix with ".*/" - return $"[.*{Regex.Escape(Util.DirectorySeparatorString + arg.Substring(1))}]"; + return $"[.*{Utility.ConvertLiteralToRegExp(Util.DirectorySeparatorString + arg.Substring(1))}]"; } else if (!containsSeparators && !containsWildcards && !arg.StartsWith("[", StringComparison.Ordinal)) { @@ -281,12 +282,12 @@ namespace Duplicati.CommandLine private static string SuffixArgWithAsterisk(string arg) { var containsWildcards = ContainsWildcards(arg); - var endsWithSeparator = arg.EndsWith(Util.DirectorySeparatorString); + var endsWithSeparator = arg.EndsWith(Util.DirectorySeparatorString, StringComparison.Ordinal); if (endsWithSeparator && arg.StartsWith("@", StringComparison.Ordinal)) { // Convert to Regexp filter and suffix with ".*" - return $"[{Regex.Escape(arg.Substring(1))}.*]"; + return $"[{Utility.ConvertLiteralToRegExp(arg.Substring(1))}.*]"; } else if (endsWithSeparator && !containsWildcards && !arg.StartsWith("[", StringComparison.Ordinal)) { @@ -304,16 +305,14 @@ namespace Duplicati.CommandLine /// /// Returns true if contains directory separator characters. /// - public static bool ContainsDirectorySeparators(string s) => - s.IndexOfAny(pathSeparatorsCharacters) >= 0; + public static bool ContainsDirectorySeparators(string s) => s.IndexOfAny(pathSeparatorsCharacters) >= 0; private static readonly char[] wildcardCharacters = new[] { '*', '?' }; /// /// Returns true if contains wildcard characters. /// - public static bool ContainsWildcards(string s) => - s.IndexOfAny(wildcardCharacters) >= 0; + public static bool ContainsWildcards(string s) => s.IndexOfAny(wildcardCharacters) >= 0; public static int List(TextWriter outwriter, Action setup, List args, Dictionary options, Library.Utility.IFilter filter) { From 78555241c1a2a0764acd0d022c101ce846f0246f Mon Sep 17 00:00:00 2001 From: Dean Ferreyra Date: Mon, 7 Sep 2020 14:45:08 -0700 Subject: [PATCH 16/17] Address review comments from @warwickmm --- Duplicati/CommandLine/Commands.cs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Duplicati/CommandLine/Commands.cs b/Duplicati/CommandLine/Commands.cs index 1f5b0b1d7..74523762c 100644 --- a/Duplicati/CommandLine/Commands.cs +++ b/Duplicati/CommandLine/Commands.cs @@ -240,8 +240,7 @@ namespace Duplicati.CommandLine /// For bare file names with no wildcards, replace argument with /// the equivalent of prefixing with "*/" so we search all folders. /// - public static IEnumerable PrefixArgsWithAsterisk(IEnumerable argList) => - argList.Select(PrefixArgWithAsterisk); + public static IEnumerable PrefixArgsWithAsterisk(IEnumerable argList) => argList.Select(PrefixArgWithAsterisk); /// /// For bare file names with no wildcards, return argument with @@ -272,8 +271,7 @@ namespace Duplicati.CommandLine /// For folders, replace argument with the equivalent of suffixing /// with "*" so we restore contents in the folder. /// - public static IEnumerable SuffixArgsWithAsterisk(IEnumerable argList) => - argList.Select(SuffixArgWithAsterisk); + public static IEnumerable SuffixArgsWithAsterisk(IEnumerable argList) => argList.Select(SuffixArgWithAsterisk); /// /// For folders, return argument with the equivalent of suffixing @@ -339,7 +337,7 @@ namespace Duplicati.CommandLine options["version"] = v.ToString(); } } - else if (args[0].IndexOfAny(new char[] { '*', '?' }) < 0 && !args[0].StartsWith("[", StringComparison.Ordinal) && !args[0].StartsWith("@", StringComparison.Ordinal)) + else if (!ContainsWildcards(args[0]) && !args[0].StartsWith("[", StringComparison.Ordinal) && !args[0].StartsWith("@", StringComparison.Ordinal)) { try { From 93601d0a826fd4e48e9381a98ee9d2d6356df5ef Mon Sep 17 00:00:00 2001 From: Dean Ferreyra Date: Mon, 7 Sep 2020 18:56:59 -0700 Subject: [PATCH 17/17] Address review comments from @warwickmm --- Duplicati/UnitTest/ProblematicPathTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/UnitTest/ProblematicPathTests.cs b/Duplicati/UnitTest/ProblematicPathTests.cs index 56cc8b954..27164b960 100644 --- a/Duplicati/UnitTest/ProblematicPathTests.cs +++ b/Duplicati/UnitTest/ProblematicPathTests.cs @@ -62,14 +62,14 @@ namespace Duplicati.UnitTest const string singleCharacterDir = "X"; string dirWithSingleCharacter = Path.Combine(this.DATAFOLDER, singleCharacterDir); SystemIO.IO_OS.DirectoryCreate(dirWithSingleCharacter); - WriteFile(SystemIO.IO_OS.PathCombine(dirWithSingleCharacter, file), new byte[] { 1 }); + WriteFile(SystemIO.IO_OS.PathCombine(dirWithSingleCharacter, file), new byte[] { 2 }); directories.Add(dirWithSingleCharacter); questionMarkWildcardShouldMatchCount++; const string dir = "dir"; string normalDir = Path.Combine(this.DATAFOLDER, dir); SystemIO.IO_OS.DirectoryCreate(normalDir); - WriteFile(SystemIO.IO_OS.PathCombine(normalDir, file), new byte[] {2}); + WriteFile(SystemIO.IO_OS.PathCombine(normalDir, file), new byte[] {3}); directories.Add(normalDir); // Backup all files.