diff --git a/Duplicati/CommandLine/Commands.cs b/Duplicati/CommandLine/Commands.cs index 0e60ffc71..74523762c 100644 --- a/Duplicati/CommandLine/Commands.cs +++ b/Duplicati/CommandLine/Commands.cs @@ -19,7 +19,9 @@ using System; using System.Linq; using System.Collections.Generic; using System.IO; +using System.Text.RegularExpressions; using Duplicati.Library.Common.IO; +using Duplicati.Library.Utility; namespace Duplicati.CommandLine { @@ -234,14 +236,82 @@ namespace Duplicati.CommandLine } } - public static List PrefixArgsWithAsterisk(IList argList) + /// + /// For bare file names with no wildcards, replace argument with + /// the equivalent of prefixing with "*/" so we search all folders. + /// + public static IEnumerable PrefixArgsWithAsterisk(IEnumerable argList) => argList.Select(PrefixArgWithAsterisk); + + /// + /// For bare file names with no wildcards, return argument with + /// the equivalent of prefixing with "*/" so we search all folders. + /// + private static string PrefixArgWithAsterisk(string arg) { - // Prefix all filenames with "*/" so we search all folders - var specialArgChars = new[] { '*', '?', Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }; - return argList.Select(arg => arg.IndexOfAny(specialArgChars) < 0 - && !arg.StartsWith("[", StringComparison.Ordinal) ? "*" + Util.DirectorySeparatorString + arg : arg).ToList(); + var containsSeparators = ContainsDirectorySeparators(arg); + var containsWildcards = ContainsWildcards(arg); + + if (!containsSeparators && arg.StartsWith("@", StringComparison.Ordinal)) + { + // Convert to Regexp filter and prefix with ".*/" + return $"[.*{Utility.ConvertLiteralToRegExp(Util.DirectorySeparatorString + arg.Substring(1))}]"; + } + else if (!containsSeparators && !containsWildcards && !arg.StartsWith("[", StringComparison.Ordinal)) + { + // Prefix files with "*/" + return "*" + Util.DirectorySeparatorString + arg; + } + else + { + return arg; + } } + /// + /// For folders, replace argument with the equivalent of suffixing + /// with "*" so we restore contents in the folder. + /// + public static IEnumerable SuffixArgsWithAsterisk(IEnumerable argList) => argList.Select(SuffixArgWithAsterisk); + + /// + /// For folders, return argument with the equivalent of suffixing + /// with "*" so we restore contents in the folder. + /// + private static string SuffixArgWithAsterisk(string arg) + { + var containsWildcards = ContainsWildcards(arg); + var endsWithSeparator = arg.EndsWith(Util.DirectorySeparatorString, StringComparison.Ordinal); + + if (endsWithSeparator && arg.StartsWith("@", StringComparison.Ordinal)) + { + // Convert to Regexp filter and suffix with ".*" + return $"[{Utility.ConvertLiteralToRegExp(arg.Substring(1))}.*]"; + } + else if (endsWithSeparator && !containsWildcards && !arg.StartsWith("[", StringComparison.Ordinal)) + { + // Suffix with "*" + return arg + "*"; + } + else + { + return arg; + } + } + + private static readonly char[] pathSeparatorsCharacters = new[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }; + + /// + /// Returns true if contains directory separator characters. + /// + public static bool ContainsDirectorySeparators(string s) => s.IndexOfAny(pathSeparatorsCharacters) >= 0; + + private static readonly char[] wildcardCharacters = new[] { '*', '?' }; + + /// + /// Returns true if contains wildcard characters. + /// + public static bool ContainsWildcards(string s) => s.IndexOfAny(wildcardCharacters) >= 0; + public static int List(TextWriter outwriter, Action setup, List args, Dictionary options, Library.Utility.IFilter filter) { filter = filter ?? new Duplicati.Library.Utility.FilterExpression(); @@ -267,7 +337,7 @@ namespace Duplicati.CommandLine options["version"] = v.ToString(); } } - else if (args[0].IndexOfAny(new char[] { '*', '?' }) < 0 && !args[0].StartsWith("[", StringComparison.Ordinal)) + else if (!ContainsWildcards(args[0]) && !args[0].StartsWith("[", StringComparison.Ordinal) && !args[0].StartsWith("@", StringComparison.Ordinal)) { try { @@ -282,7 +352,7 @@ namespace Duplicati.CommandLine } } - args = PrefixArgsWithAsterisk(args); + args = PrefixArgsWithAsterisk(args).ToList(); // Support for not adding the --auth-username if possible string dbpath; @@ -448,12 +518,7 @@ namespace Duplicati.CommandLine bool controlFiles = Library.Utility.Utility.ParseBoolOption(options, "control-files"); options.Remove("control-files"); - args = PrefixArgsWithAsterisk(args); - - // suffix all folders with "*" so we restore all contents in the folder - for (var ix = 0; ix < args.Count; ix++) - if (args[ix].IndexOfAny(new char[] { '*', '?' }) < 0 && !args[ix].StartsWith("[", StringComparison.Ordinal) && args[ix].EndsWith(Util.DirectorySeparatorString, StringComparison.Ordinal)) - args[ix] += "*"; + args = SuffixArgsWithAsterisk(PrefixArgsWithAsterisk(args)).ToList(); using(var output = new ConsoleOutput(outwriter, options)) using (var i = new Library.Main.Controller(backend, options, output)) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index f46694ef3..fd1048d20 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -1190,7 +1190,7 @@ ORDER BY var args = new List(); foreach (var f in ((Library.Utility.FilterExpression)filter).GetSimpleList()) { - if (f.Contains('*') || f.Contains('?')) + if (type == FilterType.Wildcard) { sb.Append(@"""Path"" LIKE ? OR "); args.Add(f.Replace('*', '%').Replace('?', '_')); diff --git a/Duplicati/Library/Utility/FilterExpression.cs b/Duplicati/Library/Utility/FilterExpression.cs index 0c70f7ef7..fc4eb1745 100644 --- a/Duplicati/Library/Utility/FilterExpression.cs +++ b/Duplicati/Library/Utility/FilterExpression.cs @@ -119,6 +119,14 @@ namespace Duplicati.Library.Utility this.Filter = filter.Substring(1, filter.Length - 2); this.Regexp = GetFilterGroupRegex(this.Filter); } + else if (filter.StartsWith("@", StringComparison.Ordinal)) + { + // Take filter literally; i.e., don't treat wildcard + // characters as globbing characters + this.Type = FilterType.Simple; + this.Filter = filter.Substring(1); + this.Regexp = new Regex(Utility.ConvertLiteralToRegExp(this.Filter), REGEXP_OPTIONS); + } else { this.Type = (filter.Contains(MULTIPLE_WILDCARD) || filter.Contains(SINGLE_WILDCARD)) ? FilterType.Wildcard : FilterType.Simple; @@ -150,6 +158,10 @@ namespace Duplicati.Library.Utility { return filterString.Substring(1, filterString.Length - 2); } + else if (filterString.StartsWith("@", StringComparison.Ordinal)) + { + return Utility.ConvertLiteralToRegExp(filterString.Substring(1)); + } else { return Utility.ConvertGlobbingToRegExp(filterString); @@ -284,6 +296,8 @@ namespace Duplicati.Library.Utility return "[" + this.Filter + "]"; case FilterType.Group: return "{" + this.Filter + "}"; + case FilterType.Simple: + return "@" + this.Filter; default: return this.Filter; } diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 703b6ea29..e8280231f 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -164,6 +164,15 @@ namespace Duplicati.Library.Utility return globexp; } + /// + /// Convert literal path to the equivalent regular expression. + /// + public static string ConvertLiteralToRegExp(string literalPath) + { + // Escape all special characters + return Regex.Escape(literalPath); + } + /// /// Returns a list of all files found in the given folder. /// The search is recursive. diff --git a/Duplicati/Server/webroot/ngax/scripts/controllers/RestoreController.js b/Duplicati/Server/webroot/ngax/scripts/controllers/RestoreController.js index 0ff98a378..346c64fdf 100644 --- a/Duplicati/Server/webroot/ngax/scripts/controllers/RestoreController.js +++ b/Duplicati/Server/webroot/ngax/scripts/controllers/RestoreController.js @@ -346,10 +346,22 @@ backupApp.controller('RestoreController', function ($rootScope, $scope, $routePa var paths = []; for(var n in $scope.Selected) { var item = $scope.Selected[n]; - if (item.substr(item.length - 1) == dirsep) - paths.push(item + '*'); - else - paths.push(item); + if (item.substr(item.length - 1) == dirsep) { + // To support the possibility of encountering paths + // with literal wildcard characters, but also being + // able to add the globbing "*" suffix, use a regular + // expression filter + + // Escape regular expression metacharacters + var itemRegex = item.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + // Add "globbing" suffix + paths.push('[' + itemRegex + '.*]'); + } else { + // To support the possibility of encountering paths + // with literal wildcard characters, create a literal + // filter + paths.push('@' + item); + } } if (paths.length > 0) diff --git a/Duplicati/Server/webroot/ngax/scripts/directives/restoreFilePicker.js b/Duplicati/Server/webroot/ngax/scripts/directives/restoreFilePicker.js index ef142f6bc..3e71e8648 100644 --- a/Duplicati/Server/webroot/ngax/scripts/directives/restoreFilePicker.js +++ b/Duplicati/Server/webroot/ngax/scripts/directives/restoreFilePicker.js @@ -35,7 +35,10 @@ backupApp.directive('restoreFilePicker', function() { if (!node.children && !node.loading) { node.loading = true; - AppService.get('/backup/' + $scope.ngBackupId + '/files/' + encodeURIComponent(node.id) + '?prefix-only=false&folder-contents=true&time=' + encodeURIComponent($scope.ngTimestamp) + '&filter=' + encodeURIComponent(node.id)).then(function(data) { + // Prefix filter with "@" to prevent Duplicati from + // mistaking literal '*' and '?' characters in paths + // for glob wildcard characters + AppService.get('/backup/' + $scope.ngBackupId + '/files/' + encodeURIComponent(node.id) + '?prefix-only=false&folder-contents=true&time=' + encodeURIComponent($scope.ngTimestamp) + '&filter=' + encodeURIComponent('@' + node.id)).then(function(data) { var children = [] for(var n in data.data.Files) diff --git a/Duplicati/UnitTest/ProblematicPathTests.cs b/Duplicati/UnitTest/ProblematicPathTests.cs index 3e458a779..3a3285cf1 100644 --- a/Duplicati/UnitTest/ProblematicPathTests.cs +++ b/Duplicati/UnitTest/ProblematicPathTests.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using Duplicati.Library.Common; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Main; @@ -13,6 +14,153 @@ namespace Duplicati.UnitTest [TestFixture] public class ProblematicPathTests : BasicSetupHelper { + [Test] + [Category("ProblematicPath")] + public void DirectoriesWithWildcards() + { + const string file = "file"; + List directories = new List(); + + // Keep expected match counts since they'll differ between + // Linux and Windows. + var questionMarkWildcardShouldMatchCount = 0; + var verbatimAsteriskShouldMatchCount = 0; + + const string asterisk = "*"; + string dirWithAsterisk = Path.Combine(this.DATAFOLDER, asterisk); + // Windows does not support literal asterisks in paths. + if (!Platform.IsClientWindows) + { + SystemIO.IO_OS.DirectoryCreate(dirWithAsterisk); + WriteFile(SystemIO.IO_OS.PathCombine(dirWithAsterisk, file), new byte[] {0}); + directories.Add(dirWithAsterisk); + questionMarkWildcardShouldMatchCount++; + verbatimAsteriskShouldMatchCount++; + } + + const string questionMark = "?"; + string dirWithQuestionMark = Path.Combine(this.DATAFOLDER, questionMark); + // Windows does not support literal question marks in paths. + if (!Platform.IsClientWindows) + { + SystemIO.IO_OS.DirectoryCreate(dirWithQuestionMark); + WriteFile(SystemIO.IO_OS.PathCombine(dirWithQuestionMark, file), new byte[] { 1 }); + directories.Add(dirWithQuestionMark); + questionMarkWildcardShouldMatchCount++; + } + + // Include at least one single character directory in Windows + // for a '?' wildcard can match on + const string singleCharacterDir = "X"; + string dirWithSingleCharacter = Path.Combine(this.DATAFOLDER, singleCharacterDir); + SystemIO.IO_OS.DirectoryCreate(dirWithSingleCharacter); + WriteFile(SystemIO.IO_OS.PathCombine(dirWithSingleCharacter, file), new byte[] { 2 }); + directories.Add(dirWithSingleCharacter); + questionMarkWildcardShouldMatchCount++; + + const string dir = "dir"; + string normalDir = Path.Combine(this.DATAFOLDER, dir); + SystemIO.IO_OS.DirectoryCreate(normalDir); + WriteFile(SystemIO.IO_OS.PathCombine(normalDir, file), new byte[] {3}); + directories.Add(normalDir); + + // Backup all files. + Dictionary options = new Dictionary(this.TestOptions); + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + IBackupResults backupResults = c.Backup(new[] {this.DATAFOLDER}); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + } + + // Restore all files. + Dictionary restoreOptions = new Dictionary(options) {["restore-path"] = this.RESTOREFOLDER}; + using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null)) + { + IRestoreResults restoreResults = c.Restore(null); + Assert.AreEqual(0, restoreResults.Errors.Count()); + Assert.AreEqual(0, restoreResults.Warnings.Count()); + + foreach (string directory in directories) + { + string directoryName = SystemIO.IO_OS.PathGetFileName(directory); + foreach (string expectedFilePath in SystemIO.IO_OS.EnumerateFiles(directory)) + { + string fileName = SystemIO.IO_OS.PathGetFileName(expectedFilePath); + string restoredFilePath = SystemIO.IO_OS.PathCombine(this.RESTOREFOLDER, directoryName, fileName); + Assert.IsTrue(TestUtils.CompareFiles(expectedFilePath, restoredFilePath, expectedFilePath, false)); + } + } + + // List results using * should return a match for each directory. + IListResults listResults = c.List(SystemIO.IO_OS.PathCombine(dirWithAsterisk, file)); + Assert.AreEqual(0, listResults.Errors.Count()); + Assert.AreEqual(0, listResults.Warnings.Count()); + Assert.AreEqual(directories.Count, listResults.Files.Count()); + + listResults = c.List(SystemIO.IO_OS.PathCombine(dirWithQuestionMark, file)); + Assert.AreEqual(0, listResults.Errors.Count()); + Assert.AreEqual(0, listResults.Warnings.Count()); + // List results using ? should return 3 matches in Linux, + // one for the directory with '*' and one for the directory + // with '?', plus one for directory 'X'; but should return + // 1 matches in Windows just for directory 'X'. + Assert.AreEqual(questionMarkWildcardShouldMatchCount, listResults.Files.Count()); + } + + SystemIO.IO_OS.DirectoryDelete(this.RESTOREFOLDER, true); + + // Restore one file at a time using the verbatim identifier. + foreach (string directory in directories) + { + foreach (string expectedFilePath in SystemIO.IO_OS.EnumerateFiles(directory)) + { + using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null)) + { + string verbatimFilePath = "@" + expectedFilePath; + + // Verify that list result using verbatim identifier contains only one file. + IListResults listResults = c.List(verbatimFilePath); + Assert.AreEqual(0, listResults.Errors.Count()); + Assert.AreEqual(0, listResults.Warnings.Count()); + Assert.AreEqual(1, listResults.Files.Count()); + Assert.AreEqual(expectedFilePath, listResults.Files.Single().Path); + + IRestoreResults restoreResults = c.Restore(new[] {verbatimFilePath}); + Assert.AreEqual(0, restoreResults.Errors.Count()); + Assert.AreEqual(0, restoreResults.Warnings.Count()); + + string fileName = SystemIO.IO_OS.PathGetFileName(expectedFilePath); + string restoredFilePath = SystemIO.IO_OS.PathCombine(this.RESTOREFOLDER, fileName); + Assert.IsTrue(TestUtils.CompareFiles(expectedFilePath, restoredFilePath, expectedFilePath, false)); + + SystemIO.IO_OS.FileDelete(restoredFilePath); + } + } + } + + // Backup with asterisk in include filter should include all directories. + FilterExpression filter = new FilterExpression(dirWithAsterisk); + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + IBackupResults backupResults = c.Backup(new[] {this.DATAFOLDER}, filter); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + Assert.AreEqual(directories.Count, backupResults.ExaminedFiles); + } + + // Backup with verbatim asterisk in include filter should include + // one directory in Linux and zero directories in Windows. + filter = new FilterExpression("@" + SystemIO.IO_OS.PathCombine(dirWithAsterisk, file)); + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + IBackupResults backupResults = c.Backup(new[] {this.DATAFOLDER}, filter); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + Assert.AreEqual(verbatimAsteriskShouldMatchCount, backupResults.ExaminedFiles); + } + } + [Test] [Category("ProblematicPath")] public void ExcludeProblematicPaths() diff --git a/pipeline/start.sh b/pipeline/start.sh index 9267229be..b6aa81f89 100755 --- a/pipeline/start.sh +++ b/pipeline/start.sh @@ -6,6 +6,6 @@ SCRIPT_DIR="$( cd "$(dirname "$0")" ; pwd -P )" ${ROOT_DIR}/pipeline/jobs/build_job.sh ${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories BulkNormal ${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories BulkNoSize -${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories SVNDataLong,SVNData,RecoveryTool +${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories SVNDataLong,SVNData,RecoveryTool,ProblematicPath ${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories Border -${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories Filter,Targeted,Purge,Serialization,WebApi,Utility,UriUtility,IO,ImportExport,Disruption,RestoreHandler,RepairHandler,DeleteHandler +${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories Filter,Targeted,Purge,Serialization,WebApi,Utility,UriUtility,IO,ImportExport,Disruption,RestoreHandler,RepairHandler,DeleteHandler,Controller