haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #5053 from Jojo-1000/fix-regexp-combine

Browse Source 
Fix incorrect filter behavior when combining multiple regexp
This commit is contained in:
Kenneth Skovhede
2025-05-01 14:42:43 +02:00
committed by GitHub
parent ec9eed335d a15c3262be
commit ea16becc04
3 changed files with 82 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Utility/FilterExpression.cs
+42 -35
View File
@@ -76,7 +76,7 @@ namespace Duplicati.Library.Utility
/// The regular expression version of the filter
/// </summary>
public readonly Regex Regexp;
/// <summary>
/// The single wildcard character (DOS style)
/// </summary>
@@ -116,7 +116,14 @@ namespace Duplicati.Library.Utility
{
this.Type = FilterType.Regexp;
this.Filter = filter.Substring(1, filter.Length - 2);
this.Regexp = new Regex(this.Filter, REGEXP_OPTIONS);
if (Filter.StartsWith("^") && Filter.EndsWith("$"))
{
this.Regexp = new Regex(this.Filter, REGEXP_OPTIONS);
}
else
{
this.Regexp = new Regex("^(" + this.Filter + ")$", REGEXP_OPTIONS);
}
}
else if (filter.StartsWith("{", StringComparison.Ordinal) && filter.EndsWith("}", StringComparison.Ordinal))
{
@@ -185,14 +192,14 @@ namespace Duplicati.Library.Utility
regexString = "(" + string.Join(")|(", regexStrings) + ")";
}
result = new Regex(regexString, REGEXP_OPTIONS);
result = new Regex("^(" + regexString + ")$", REGEXP_OPTIONS);
FilterEntry.filterGroupRegexCache[filterGroup] = result;
return result;
}
}
/// <summary>
/// Tests whether specified string can be matched against provided pattern string. Pattern may contain single- and multiple-replacing
/// wildcard characters.
@@ -215,7 +222,7 @@ namespace Duplicati.Library.Utility
inputPos++;
patternPos++;
}
// Push this position to stack if it points to end of pattern or to a general wildcard
if (patternPos == pattern.Length || pattern[patternPos] == MULTIPLE_WILDCARD)
{
@@ -236,10 +243,10 @@ namespace Duplicati.Library.Utility
if (inputPos == input.Length && (patternPos == pattern.Length || (patternPos == pattern.Length - 1 && pattern[patternPos] == MULTIPLE_WILDCARD)))
matched = true; // Reached end of both pattern and input string, hence matching is successful
else
{
{
// First character in next pattern block is guaranteed to be multiple wildcard
// So skip it and search for all matches in value string until next multiple wildcard character is reached in pattern
for(int curInputStart = inputPos; curInputStart < input.Length; curInputStart++)
for (int curInputStart = inputPos; curInputStart < input.Length; curInputStart++)
{
int curInputPos = curInputStart;
int curPatternPos = patternPos + 1;
@@ -259,7 +266,7 @@ namespace Duplicati.Library.Utility
// If we have reached next multiple wildcard character in pattern without breaking the matching sequence, then we have another candidate for full match
// This candidate should be pushed to stack for further processing
// At the same time, pair (input position, pattern position) will be marked as tested, so that it will not be pushed to stack later again
if (((curPatternPos == pattern.Length && curInputPos == input.Length) || (curPatternPos < pattern.Length && pattern[curPatternPos] == MULTIPLE_WILDCARD))
if (((curPatternPos == pattern.Length && curInputPos == input.Length) || (curPatternPos < pattern.Length && pattern[curPatternPos] == MULTIPLE_WILDCARD))
&& !pointTested[curInputPos, curPatternPos])
{
pointTested[curInputPos, curPatternPos] = true;
@@ -271,7 +278,7 @@ namespace Duplicati.Library.Utility
}
return matched;
}
/// <summary>
/// Gets a value indicating if the filter matches the path
/// </summary>
@@ -289,7 +296,7 @@ namespace Duplicati.Library.Utility
var m = this.Regexp.Match(path);
return m.Success && m.Length == path.Length;
default:
return false;
return false;
}
}
@@ -308,12 +315,12 @@ namespace Duplicati.Library.Utility
}
}
}
/// <summary>
/// The internal list of expressions
/// </summary>
private readonly List<FilterEntry> m_filters;
/// <summary>
/// Gets the type of the filter
/// </summary>
@@ -341,7 +348,7 @@ namespace Duplicati.Library.Utility
else
throw new InvalidOperationException($"Cannot extract simple list when the type is: {this.Type}");
}
/// <summary>
/// Gets a value indicating if the filter matches the path
/// </summary>
@@ -355,14 +362,14 @@ namespace Duplicati.Library.Utility
match = null;
return false;
}
if (m_filters.Any(x => x.Matches(path)))
{
match = this;
result = this.Result;
return true;
}
match = null;
return false;
}
@@ -371,7 +378,7 @@ namespace Duplicati.Library.Utility
public string GetFilterHash()
{
var hash = MD5HashHelper.GetHash(m_filters?.Select(x => x.Filter));
return Utility.ByteArrayAsHexString(hash);
return Utility.ByteArrayAsHexString(hash);
}
/// <summary>
@@ -399,26 +406,26 @@ namespace Duplicati.Library.Utility
public FilterExpression(IEnumerable<string> filter, bool result = true)
{
this.Result = result;
if (filter == null)
{
this.Type = FilterType.Empty;
return;
}
m_filters = Compact(
(from n in filter
let nx = new FilterEntry(n)
where nx.Type != FilterType.Empty
select nx)
let nx = new FilterEntry(n)
where nx.Type != FilterType.Empty
select nx)
);
if (m_filters.Count == 0)
this.Type = FilterType.Empty;
else
this.Type = m_filters.Max((a) => a.Type);
}
private static IEnumerable<string> Expand(string filter)
{
if (string.IsNullOrWhiteSpace(filter))
@@ -438,7 +445,7 @@ namespace Duplicati.Library.Utility
return filter.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries);
}
private static List<FilterEntry> Compact(IEnumerable<FilterEntry> items)
{
var r = new List<FilterEntry>();
@@ -482,7 +489,7 @@ namespace Duplicati.Library.Utility
if (combined.Length > 0)
r.Add(new FilterEntry("[" + combined.Append("]")));
return r;
return r;
}
/// <summary>
@@ -520,7 +527,7 @@ namespace Duplicati.Library.Utility
// Check for cached results
if (filter != null)
lock(_matchLock)
lock (_matchLock)
if (_matchFallbackLookup.TryGetValue(filter, out cacheLookup))
{
includes = cacheLookup.Item1;
@@ -553,7 +560,7 @@ namespace Duplicati.Library.Utility
}
// Populate the cache
lock(_matchLock)
lock (_matchLock)
{
if (_matchFallbackLookup.Count > 10)
_matchFallbackLookup.Remove(_matchFallbackLookup.Keys.Skip(new Random().Next(0, _matchFallbackLookup.Count)).First());
@@ -575,7 +582,7 @@ namespace Duplicati.Library.Utility
match = null;
return true;
}
bool result;
if (filter.Matches(path, out result, out match))
return result;
@@ -598,7 +605,7 @@ namespace Duplicati.Library.Utility
}
}
/// <summary>
/// Combine the specified filter expressions.
/// </summary>
@@ -642,12 +649,12 @@ namespace Duplicati.Library.Utility
{
if (this.Empty)
return "";
return
return
"(" +
string.Join(") || (",
(from n in m_filters
select n.ToString())
select n.ToString())
) +
")";
}
@@ -663,7 +670,7 @@ namespace Duplicati.Library.Utility
return
(from n in m_filters
select $"{(this.Result ? "+" : "-")}{n.ToString()}"
select $"{(this.Result ? "+" : "-")}{n.ToString()}"
).ToArray();
}
@@ -675,7 +682,7 @@ namespace Duplicati.Library.Utility
{
if (filter == null || filter.Empty)
return new string[0];
IEnumerable<string> res = new string[0];
var work = new Stack<IFilter>();
work.Push(filter);
@@ -709,7 +716,7 @@ namespace Duplicati.Library.Utility
return null;
IFilter res = null;
foreach(var n in filters)
foreach (var n in filters)
{
bool include;
if (n.StartsWith("+", StringComparison.Ordinal))
Duplicati/Library/Utility/Utility.cs
+1 -1
View File
@@ -209,7 +209,7 @@ namespace Duplicati.Library.Utility
//Replace the globbing expressions with the corresponding regular expressions
globexp = globexp.Replace('?', '.').Replace("*", ".*");
return globexp;
return "^" + globexp + "$";
}
/// <summary>
Duplicati/UnitTest/FilterTest.cs
+39 -5
View File
@@ -67,7 +67,7 @@ namespace Duplicati.UnitTest
// Create a fileset with all data present
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new string[] {DATAFOLDER});
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
@@ -92,7 +92,7 @@ namespace Duplicati.UnitTest
testopts["ignore-filenames"] = "exclude.me";
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new string[] {DATAFOLDER});
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
@@ -117,7 +117,7 @@ namespace Duplicati.UnitTest
testopts["exclude-empty-folders"] = "true";
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new string[] {DATAFOLDER});
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
@@ -142,7 +142,7 @@ namespace Duplicati.UnitTest
var excludefilter = new Library.Utility.FilterExpression($"*{System.IO.Path.DirectorySeparatorChar}myfile.txt", false);
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new string[] {DATAFOLDER}, excludefilter);
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }, excludefilter);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
@@ -167,7 +167,7 @@ namespace Duplicati.UnitTest
File.Delete(Path.Combine(source, "toplevel", "normal", "standard.txt"));
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new string[] {DATAFOLDER}, excludefilter);
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }, excludefilter);
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
@@ -219,5 +219,39 @@ namespace Duplicati.UnitTest
Assert.IsFalse(filter.Matches(entry.Value, out _, out _));
}
}
[Test]
[Category("Filter")]
public static void CombineRegexp()
{
FilterExpression f1 = new FilterExpression(@"[/(a|b)/]");
FilterExpression f2 = new FilterExpression(@"[/a/c/]");
FilterExpression f3 = new FilterExpression(@"/b/c/");
FilterExpression f4 = new FilterExpression(@"[/b/c/d/]");
FilterExpression combined = FilterExpression.Combine(f1, FilterExpression.Combine(f2, FilterExpression.Combine(f3, f4)));
List<string> shouldMatch = new List<string>()
{
"/a/",
"/b/",
"/a/c/",
"/b/c/",
"/b/c/d/"
};
List<string> shouldNotMatch = new List<string>()
{
"/b/d/",
"/b/d/e",
"/a/d/",
};
foreach (string s in shouldMatch)
{
Assert.IsTrue(combined.Matches(s, out _, out _));
}
foreach (string s in shouldNotMatch)
{
Assert.IsFalse(combined.Matches(s, out _, out _));
}
}
}
}