haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added a new FilterType which is used by FilterEntry to represent a filter that should affect all files in one of the built-in filter groups.

Browse Source 
This allows filter groups to be intermingled with other filters using the '{groupName}' syntax proposed by kenkendk.
It also means that when the filter that caused a file to be ignored is reported, it can report the name of the group, rather than a large and unreadable list of wildcard and regex patterns.
This is still missing several things (better documentation, removing the old --default-filters parameter, updating the UI), but should be another step in the right direction.
There might also might easily be other side effects that I haven't considered here that will need to be teased out and fixed.
This commit is contained in:
Tyler Gill
2018-03-21 14:53:45 -06:00
parent 03df606a80
commit baeb7e6a7c
3 changed files with 112 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Main/Database/LocalDatabase.cs
+1 -1
View File
@@ -1147,7 +1147,7 @@ ORDER BY
if (type != Library.Utility.FilterType.Regexp && !Library.Utility.Utility.IsFSCaseSensitive && filter.ToString().Any(x => x > 127))
type = Library.Utility.FilterType.Regexp;
if (type == Library.Utility.FilterType.Regexp)
if (type == Library.Utility.FilterType.Regexp || type == Library.Utility.FilterType.Group)
{
using(var cmd = m_connection.CreateCommand())
{
Duplicati/Library/Utility/FilterExpression.cs
+110 -18
View File
@@ -18,6 +18,7 @@
using System;
using System.Linq;
using System.Collections.Generic;
using System.Text.RegularExpressions;
namespace Duplicati.Library.Utility
{
@@ -41,7 +42,11 @@ namespace Duplicati.Library.Utility
/// <summary>
/// A list of files described with regular expressions
/// </summary>
Regexp
Regexp,
/// <summary>
/// A built in set of filters
/// </summary>
Group,
}
/// <summary>
@@ -65,7 +70,7 @@ namespace Duplicati.Library.Utility
/// <summary>
/// The regular expression version of the filter
/// </summary>
public readonly System.Text.RegularExpressions.Regex Regexp;
public readonly Regex Regexp;
/// <summary>
/// The single wildcard character (DOS style)
@@ -79,10 +84,15 @@ namespace Duplicati.Library.Utility
/// <summary>
/// The regular expression flags
/// </summary>
private static readonly System.Text.RegularExpressions.RegexOptions REGEXP_OPTIONS =
System.Text.RegularExpressions.RegexOptions.Compiled |
System.Text.RegularExpressions.RegexOptions.ExplicitCapture |
(Library.Utility.Utility.IsFSCaseSensitive ? System.Text.RegularExpressions.RegexOptions.None : System.Text.RegularExpressions.RegexOptions.IgnoreCase);
private static readonly RegexOptions REGEXP_OPTIONS =
RegexOptions.Compiled |
RegexOptions.ExplicitCapture |
(Utility.IsFSCaseSensitive ? RegexOptions.None : RegexOptions.IgnoreCase);
// Since we might need to get the regex for a particular filter group multiple times
// (e.g., when combining multiple FilterExpressions together, which discards the existing FilterEntries and recreates them from the Filter representations),
// and since they don't change (and compiling them isn't a super cheap operation), we keep a cache of the ones we've built for re-use.
private static Dictionary<FilterGroup, Regex> filterGroupRegexCache = new Dictionary<FilterGroup, Regex>();
/// <summary>
/// Initializes a new instance of the <see cref="T:Duplicati.Library.Utility.FilterExpression.FilterEntry"/> struct.
@@ -100,13 +110,75 @@ namespace Duplicati.Library.Utility
{
this.Type = FilterType.Regexp;
this.Filter = filter.Substring(1, filter.Length - 2);
this.Regexp = new System.Text.RegularExpressions.Regex(this.Filter, REGEXP_OPTIONS);
this.Regexp = new Regex(this.Filter, REGEXP_OPTIONS);
}
else if (filter.StartsWith("{", StringComparison.Ordinal) && filter.EndsWith("}", StringComparison.Ordinal))
{
this.Type = FilterType.Group;
this.Filter = filter.Substring(1, filter.Length - 2);
this.Regexp = GetFilterGroupRegex(this.Filter);
}
else
{
this.Type = (filter.Contains(MULTIPLE_WILDCARD) || filter.Contains(SINGLE_WILDCARD)) ? FilterType.Wildcard : FilterType.Simple;
this.Filter = (!Utility.IsFSCaseSensitive && this.Type == FilterType.Wildcard) ? filter.ToUpper() : filter;
this.Regexp = new System.Text.RegularExpressions.Regex(Library.Utility.Utility.ConvertGlobbingToRegExp(filter), REGEXP_OPTIONS);
this.Regexp = new Regex(Utility.ConvertGlobbingToRegExp(filter), REGEXP_OPTIONS);
}
}
/// <summary>
/// Gets the regex that represents the given filter group
/// </summary>
/// <param name="filterGroupName">Filter group name</param>
/// <returns>Group regex</returns>
private static Regex GetFilterGroupRegex(string filterGroupName)
{
FilterGroup filterGroup;
if (Enum.TryParse(filterGroupName, true /* ignoreCase */, out filterGroup))
{
Regex result;
if (FilterEntry.filterGroupRegexCache.TryGetValue(filterGroup, out result))
{
return result;
}
else
{
// Get the filter strings for this filter group, and convert them to their regex forms
List<string> regexStrings = FilterGroups.GetFilterStrings(filterGroup)
.Select(filterString =>
{
if (filterString.StartsWith("[", StringComparison.Ordinal) && filterString.EndsWith("]", StringComparison.Ordinal))
{
return filterString.Substring(1, filterString.Length - 2);
}
else
{
return Utility.ConvertGlobbingToRegExp(filterString);
}
})
.ToList();
string regexString;
if (regexStrings.Count == 1)
{
regexString = regexStrings.Single();
}
else
{
// If there are multiple regex strings, then they need to be merged by wrapping each in parenthesis and ORing them together
regexString = "(" + string.Join(")|(", regexStrings) + ")";
}
result = new Regex(regexString, REGEXP_OPTIONS);
FilterEntry.filterGroupRegexCache[filterGroup] = result;
return result;
}
}
else
{
throw new ArgumentException(string.Format("Unknown filter group {0}", filterGroupName));
}
}
@@ -203,12 +275,26 @@ namespace Duplicati.Library.Utility
case FilterType.Wildcard:
return IsWildcardMatch(!Utility.IsFSCaseSensitive ? path.ToUpper() : path, this.Filter);
case FilterType.Regexp:
case FilterType.Group:
var m = this.Regexp.Match(path);
return m.Success && m.Length == path.Length;
default:
return false;
}
}
public override string ToString()
{
switch (this.Type)
{
case FilterType.Regexp:
return "[" + this.Filter + "]";
case FilterType.Group:
return "{" + this.Filter + "}";
default:
return this.Filter;
}
}
}
/// <summary>
@@ -318,11 +404,17 @@ namespace Duplicati.Library.Utility
{
if (string.IsNullOrWhiteSpace(filter))
return null;
if (filter.Length < 2 || (filter.StartsWith("[", StringComparison.Ordinal) && filter.EndsWith("]", StringComparison.Ordinal)))
if (filter.Length < 2 ||
(filter.StartsWith("[", StringComparison.Ordinal) && filter.EndsWith("]", StringComparison.Ordinal)) ||
(filter.StartsWith("{", StringComparison.Ordinal) && filter.EndsWith("}", StringComparison.Ordinal)))
{
return new string[] { filter };
}
else
{
return filter.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries);
}
}
private static List<FilterEntry> Compact(IEnumerable<FilterEntry> items)
@@ -333,7 +425,9 @@ namespace Duplicati.Library.Utility
foreach(var f in items)
if (combined == null)
{
if (f.Type == FilterType.Simple || f.Type == FilterType.Wildcard)
// Note that even though group filters may include regexes, we don't want to merge them together and compact them,
// since that would make their names much more difficult to interpret on the command line.
if (f.Type == FilterType.Simple || f.Type == FilterType.Wildcard || f.Type == FilterType.Group)
r.Add(f);
else if (f.Type != FilterType.Empty)
{
@@ -343,7 +437,7 @@ namespace Duplicati.Library.Utility
}
else
{
if (f.Type == FilterType.Simple || f.Type == FilterType.Wildcard)
if (f.Type == FilterType.Simple || f.Type == FilterType.Wildcard || f.Type == FilterType.Group)
{
r.Add(new FilterEntry("[" + combined + "]"));
r.Add(f);
@@ -496,7 +590,7 @@ namespace Duplicati.Library.Utility
if (first.Result != second.Result)
throw new ArgumentException("Both filters must have the same result property");
return new FilterExpression(first.m_filters.Union(second.m_filters).Select(x => x.Type == FilterType.Regexp ? ("[" + x.Filter + "]") : x.Filter), first.Result);
return new FilterExpression(first.m_filters.Union(second.m_filters).Select(x => x.ToString()), first.Result);
}
/// <summary>
@@ -530,7 +624,7 @@ namespace Duplicati.Library.Utility
"(" +
string.Join(") || (",
(from n in m_filters
select n.Type == FilterType.Regexp ? "[" + n.Filter + "]" : n.Filter)
select n.ToString())
) +
")";
}
@@ -547,11 +641,9 @@ namespace Duplicati.Library.Utility
return
(from n in m_filters
select string.Format(
"{0}{1}{2}{3}",
"{0}{1}",
this.Result ? "+" : "-",
n.Type == FilterType.Regexp ? "[" : "",
n.Filter,
n.Type == FilterType.Regexp ? "]" : ""
n.ToString()
)
).ToArray();
}
Duplicati/Library/Utility/FilterGroups.cs
+1 -1
View File
@@ -168,7 +168,7 @@ namespace Duplicati.Library.Utility
/// </summary>
/// <returns>The filters from the group.</returns>
/// <param name="group">The group to use.</param>
private static IEnumerable<string> GetFilterStrings(FilterGroup group)
public static IEnumerable<string> GetFilterStrings(FilterGroup group)
{
IEnumerable<string> osFilters;