From baeb7e6a7ca24e97fb40ab8da087521d7c8b85df Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Wed, 21 Mar 2018 14:53:45 -0600 Subject: [PATCH] Added a new FilterType which is used by FilterEntry to represent a filter that should affect all files in one of the built-in filter groups. This allows filter groups to be intermingled with other filters using the '{groupName}' syntax proposed by kenkendk. It also means that when the filter that caused a file to be ignored is reported, it can report the name of the group, rather than a large and unreadable list of wildcard and regex patterns. This is still missing several things (better documentation, removing the old --default-filters parameter, updating the UI), but should be another step in the right direction. There might also might easily be other side effects that I haven't considered here that will need to be teased out and fixed. --- .../Library/Main/Database/LocalDatabase.cs | 2 +- Duplicati/Library/Utility/FilterExpression.cs | 128 +++++++++++++++--- Duplicati/Library/Utility/FilterGroups.cs | 2 +- 3 files changed, 112 insertions(+), 20 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index c7c28b273..83f887c43 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -1147,7 +1147,7 @@ ORDER BY if (type != Library.Utility.FilterType.Regexp && !Library.Utility.Utility.IsFSCaseSensitive && filter.ToString().Any(x => x > 127)) type = Library.Utility.FilterType.Regexp; - if (type == Library.Utility.FilterType.Regexp) + if (type == Library.Utility.FilterType.Regexp || type == Library.Utility.FilterType.Group) { using(var cmd = m_connection.CreateCommand()) { diff --git a/Duplicati/Library/Utility/FilterExpression.cs b/Duplicati/Library/Utility/FilterExpression.cs index e03139b93..f1e5fe32c 100644 --- a/Duplicati/Library/Utility/FilterExpression.cs +++ b/Duplicati/Library/Utility/FilterExpression.cs @@ -18,6 +18,7 @@ using System; using System.Linq; using System.Collections.Generic; +using System.Text.RegularExpressions; namespace Duplicati.Library.Utility { @@ -41,7 +42,11 @@ namespace Duplicati.Library.Utility /// /// A list of files described with regular expressions /// - Regexp + Regexp, + /// + /// A built in set of filters + /// + Group, } /// @@ -65,7 +70,7 @@ namespace Duplicati.Library.Utility /// /// The regular expression version of the filter /// - public readonly System.Text.RegularExpressions.Regex Regexp; + public readonly Regex Regexp; /// /// The single wildcard character (DOS style) @@ -79,10 +84,15 @@ namespace Duplicati.Library.Utility /// /// The regular expression flags /// - private static readonly System.Text.RegularExpressions.RegexOptions REGEXP_OPTIONS = - System.Text.RegularExpressions.RegexOptions.Compiled | - System.Text.RegularExpressions.RegexOptions.ExplicitCapture | - (Library.Utility.Utility.IsFSCaseSensitive ? System.Text.RegularExpressions.RegexOptions.None : System.Text.RegularExpressions.RegexOptions.IgnoreCase); + private static readonly RegexOptions REGEXP_OPTIONS = + RegexOptions.Compiled | + RegexOptions.ExplicitCapture | + (Utility.IsFSCaseSensitive ? RegexOptions.None : RegexOptions.IgnoreCase); + + // Since we might need to get the regex for a particular filter group multiple times + // (e.g., when combining multiple FilterExpressions together, which discards the existing FilterEntries and recreates them from the Filter representations), + // and since they don't change (and compiling them isn't a super cheap operation), we keep a cache of the ones we've built for re-use. + private static Dictionary filterGroupRegexCache = new Dictionary(); /// /// Initializes a new instance of the struct. @@ -100,13 +110,75 @@ namespace Duplicati.Library.Utility { this.Type = FilterType.Regexp; this.Filter = filter.Substring(1, filter.Length - 2); - this.Regexp = new System.Text.RegularExpressions.Regex(this.Filter, REGEXP_OPTIONS); + this.Regexp = new Regex(this.Filter, REGEXP_OPTIONS); + } + else if (filter.StartsWith("{", StringComparison.Ordinal) && filter.EndsWith("}", StringComparison.Ordinal)) + { + this.Type = FilterType.Group; + this.Filter = filter.Substring(1, filter.Length - 2); + this.Regexp = GetFilterGroupRegex(this.Filter); } else { this.Type = (filter.Contains(MULTIPLE_WILDCARD) || filter.Contains(SINGLE_WILDCARD)) ? FilterType.Wildcard : FilterType.Simple; this.Filter = (!Utility.IsFSCaseSensitive && this.Type == FilterType.Wildcard) ? filter.ToUpper() : filter; - this.Regexp = new System.Text.RegularExpressions.Regex(Library.Utility.Utility.ConvertGlobbingToRegExp(filter), REGEXP_OPTIONS); + this.Regexp = new Regex(Utility.ConvertGlobbingToRegExp(filter), REGEXP_OPTIONS); + } + } + + /// + /// Gets the regex that represents the given filter group + /// + /// Filter group name + /// Group regex + private static Regex GetFilterGroupRegex(string filterGroupName) + { + FilterGroup filterGroup; + if (Enum.TryParse(filterGroupName, true /* ignoreCase */, out filterGroup)) + { + Regex result; + if (FilterEntry.filterGroupRegexCache.TryGetValue(filterGroup, out result)) + { + return result; + } + else + { + // Get the filter strings for this filter group, and convert them to their regex forms + List regexStrings = FilterGroups.GetFilterStrings(filterGroup) + .Select(filterString => + { + if (filterString.StartsWith("[", StringComparison.Ordinal) && filterString.EndsWith("]", StringComparison.Ordinal)) + { + return filterString.Substring(1, filterString.Length - 2); + } + else + { + return Utility.ConvertGlobbingToRegExp(filterString); + } + }) + .ToList(); + + string regexString; + if (regexStrings.Count == 1) + { + regexString = regexStrings.Single(); + } + else + { + // If there are multiple regex strings, then they need to be merged by wrapping each in parenthesis and ORing them together + regexString = "(" + string.Join(")|(", regexStrings) + ")"; + } + + result = new Regex(regexString, REGEXP_OPTIONS); + + FilterEntry.filterGroupRegexCache[filterGroup] = result; + + return result; + } + } + else + { + throw new ArgumentException(string.Format("Unknown filter group {0}", filterGroupName)); } } @@ -203,12 +275,26 @@ namespace Duplicati.Library.Utility case FilterType.Wildcard: return IsWildcardMatch(!Utility.IsFSCaseSensitive ? path.ToUpper() : path, this.Filter); case FilterType.Regexp: + case FilterType.Group: var m = this.Regexp.Match(path); return m.Success && m.Length == path.Length; default: return false; } } + + public override string ToString() + { + switch (this.Type) + { + case FilterType.Regexp: + return "[" + this.Filter + "]"; + case FilterType.Group: + return "{" + this.Filter + "}"; + default: + return this.Filter; + } + } } /// @@ -318,11 +404,17 @@ namespace Duplicati.Library.Utility { if (string.IsNullOrWhiteSpace(filter)) return null; - - if (filter.Length < 2 || (filter.StartsWith("[", StringComparison.Ordinal) && filter.EndsWith("]", StringComparison.Ordinal))) + + if (filter.Length < 2 || + (filter.StartsWith("[", StringComparison.Ordinal) && filter.EndsWith("]", StringComparison.Ordinal)) || + (filter.StartsWith("{", StringComparison.Ordinal) && filter.EndsWith("}", StringComparison.Ordinal))) + { return new string[] { filter }; + } else + { return filter.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries); + } } private static List Compact(IEnumerable items) @@ -333,7 +425,9 @@ namespace Duplicati.Library.Utility foreach(var f in items) if (combined == null) { - if (f.Type == FilterType.Simple || f.Type == FilterType.Wildcard) + // Note that even though group filters may include regexes, we don't want to merge them together and compact them, + // since that would make their names much more difficult to interpret on the command line. + if (f.Type == FilterType.Simple || f.Type == FilterType.Wildcard || f.Type == FilterType.Group) r.Add(f); else if (f.Type != FilterType.Empty) { @@ -343,7 +437,7 @@ namespace Duplicati.Library.Utility } else { - if (f.Type == FilterType.Simple || f.Type == FilterType.Wildcard) + if (f.Type == FilterType.Simple || f.Type == FilterType.Wildcard || f.Type == FilterType.Group) { r.Add(new FilterEntry("[" + combined + "]")); r.Add(f); @@ -496,7 +590,7 @@ namespace Duplicati.Library.Utility if (first.Result != second.Result) throw new ArgumentException("Both filters must have the same result property"); - return new FilterExpression(first.m_filters.Union(second.m_filters).Select(x => x.Type == FilterType.Regexp ? ("[" + x.Filter + "]") : x.Filter), first.Result); + return new FilterExpression(first.m_filters.Union(second.m_filters).Select(x => x.ToString()), first.Result); } /// @@ -530,7 +624,7 @@ namespace Duplicati.Library.Utility "(" + string.Join(") || (", (from n in m_filters - select n.Type == FilterType.Regexp ? "[" + n.Filter + "]" : n.Filter) + select n.ToString()) ) + ")"; } @@ -547,11 +641,9 @@ namespace Duplicati.Library.Utility return (from n in m_filters select string.Format( - "{0}{1}{2}{3}", + "{0}{1}", this.Result ? "+" : "-", - n.Type == FilterType.Regexp ? "[" : "", - n.Filter, - n.Type == FilterType.Regexp ? "]" : "" + n.ToString() ) ).ToArray(); } diff --git a/Duplicati/Library/Utility/FilterGroups.cs b/Duplicati/Library/Utility/FilterGroups.cs index c432600d9..1df23018f 100644 --- a/Duplicati/Library/Utility/FilterGroups.cs +++ b/Duplicati/Library/Utility/FilterGroups.cs @@ -168,7 +168,7 @@ namespace Duplicati.Library.Utility /// /// The filters from the group. /// The group to use. - private static IEnumerable GetFilterStrings(FilterGroup group) + public static IEnumerable GetFilterStrings(FilterGroup group) { IEnumerable osFilters;