diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs
index 3c5eabd2c..3a16c1f6e 100644
--- a/Duplicati/Library/Main/Operation/BackupHandler.cs
+++ b/Duplicati/Library/Main/Operation/BackupHandler.cs
@@ -444,7 +444,7 @@ namespace Duplicati.Library.Main.Operation
// the modified sources may contain non-existent files / folders, which need to
// be removed before trying to enumerate them
var existingModifiedSources = modifiedSources.Files.Where(snapshot.FileExists)
- .Concat(modifiedSources.Folders.Where(snapshot.FolderExists));
+ .Concat(modifiedSources.Folders.Where(snapshot.DirectoryExists));
foreach (var path in filterhandler.EnumerateFilesAndFolders(existingModifiedSources))
{
diff --git a/Duplicati/Library/Snapshots/ISnapshotService.cs b/Duplicati/Library/Snapshots/ISnapshotService.cs
index 7aceef496..15e554790 100644
--- a/Duplicati/Library/Snapshots/ISnapshotService.cs
+++ b/Duplicati/Library/Snapshots/ISnapshotService.cs
@@ -127,7 +127,7 @@ namespace Duplicati.Library.Snapshots
///
/// The local path
/// True if folder exists, false otherwise
- bool FolderExists(string localFolderPath);
+ bool DirectoryExists(string localFolderPath);
///
/// Tests if a file or folder exists in the snapshot
diff --git a/Duplicati/Library/Snapshots/SnapshotBase.cs b/Duplicati/Library/Snapshots/SnapshotBase.cs
index 130222cb9..d32d090d4 100644
--- a/Duplicati/Library/Snapshots/SnapshotBase.cs
+++ b/Duplicati/Library/Snapshots/SnapshotBase.cs
@@ -50,7 +50,7 @@ namespace Duplicati.Library.Snapshots
public IEnumerable EnumerateFilesAndFolders(IEnumerable sources, Utility.Utility.EnumerationFilterDelegate callback, Utility.Utility.ReportAccessError errorCallback)
{
// Add trailing slashes to folders
- var sanitizedSources = sources.Select(x => Directory.Exists(x) ? Utility.Utility.AppendDirSeparator(x) : x).ToList();
+ var sanitizedSources = sources.Select(x => DirectoryExists(x) ? Utility.Utility.AppendDirSeparator(x) : x).ToList();
return sanitizedSources.SelectMany(
s => Utility.Utility.EnumerateFileSystemEntries(s, callback, ListFolders, ListFiles, GetAttributes, errorCallback)
@@ -152,19 +152,19 @@ namespace Duplicati.Library.Snapshots
///
public bool FileExists(string localFilePath)
{
- return new FileInfo(ConvertToSnapshotPath(localFilePath)).Exists;
+ return File.Exists(ConvertToSnapshotPath(localFilePath));
}
///
- public bool FolderExists(string localFolderPath)
+ public bool DirectoryExists(string localFolderPath)
{
- return new DirectoryInfo(ConvertToSnapshotPath(localFolderPath)).Exists;
+ return Directory.Exists(ConvertToSnapshotPath(localFolderPath));
}
///
public bool PathExists(string localFileOrFolderPath)
{
- return FileExists(localFileOrFolderPath) || FolderExists(localFileOrFolderPath);
+ return FileExists(localFileOrFolderPath) || DirectoryExists(localFileOrFolderPath);
}
///
diff --git a/Duplicati/Library/Snapshots/USNJournal.cs b/Duplicati/Library/Snapshots/USNJournal.cs
index c34ddac3e..02d7b0492 100644
--- a/Duplicati/Library/Snapshots/USNJournal.cs
+++ b/Duplicati/Library/Snapshots/USNJournal.cs
@@ -133,30 +133,31 @@ namespace Duplicati.Library.Snapshots
///
/// Returns a list of files or folders that have changed since the recorded USN
///
- /// The folder to find entries for
+ /// The file or folder to find entries for
/// Minimum USN of entry
/// A list of tuples with changed files and folders and their type
- public IEnumerable> GetChangedFileSystemEntries(string sourceFolder, long minUsn)
+ public IEnumerable> GetChangedFileSystemEntries(string sourceFileOrFolder, long minUsn)
{
- return GetChangedFileSystemEntries(sourceFolder, minUsn, ChangeReason.Any);
+ return GetChangedFileSystemEntries(sourceFileOrFolder, minUsn, ChangeReason.Any);
}
///
/// Returns a list of files or folders that have changed since the recorded USN
///
- /// The folder to find entries for
+ /// The file or folder to find entries for
/// Minimum USN of entry
/// Filter expression for change reason
/// A list of tuples with changed files and folders and their type
- public IEnumerable> GetChangedFileSystemEntries(string sourceFolder, long minUsn, ChangeReason reason)
+ public IEnumerable> GetChangedFileSystemEntries(string sourceFileOrFolder, long minUsn, ChangeReason reason)
{
+ var isFolder = sourceFileOrFolder.EndsWith(Utility.Utility.DirectorySeparatorString, StringComparison.Ordinal);
+
foreach (var r in GetRecords(minUsn))
{
if (r.UsnRecord.Usn >= minUsn
&& (reason == ChangeReason.Any || (MapChangeReason(r.UsnRecord.Reason) & reason) != 0)
- && (string.IsNullOrEmpty(sourceFolder) || r.FullPath.Equals(sourceFolder,
- Utility.Utility.ClientFilenameStringComparision)
- || Utility.Utility.IsPathBelowFolder(r.FullPath, sourceFolder)))
+ && (r.FullPath.Equals(sourceFileOrFolder, Utility.Utility.ClientFilenameStringComparision)
+ || isFolder && Utility.Utility.IsPathBelowFolder(r.FullPath, sourceFileOrFolder)))
{
yield return Tuple.Create(r.FullPath,
r.UsnRecord.FileAttributes.HasFlag(Win32USN.FileAttributes.Directory)
diff --git a/Duplicati/Library/Snapshots/UsnJournalService.cs b/Duplicati/Library/Snapshots/UsnJournalService.cs
index 38bb53b9a..b5f0c5f69 100644
--- a/Duplicati/Library/Snapshots/UsnJournalService.cs
+++ b/Duplicati/Library/Snapshots/UsnJournalService.cs
@@ -75,6 +75,7 @@ namespace Duplicati.Library.Snapshots
foreach (var sourcesPerVolume in SortByVolume(sources))
{
var volume = sourcesPerVolume.Key;
+ var volumeSources = sourcesPerVolume.Value;
try
{
@@ -107,7 +108,7 @@ namespace Duplicati.Library.Snapshots
var changedFolders = new HashSet(Utility.Utility.ClientFilenameStringComparer);
// obtain changed files and folders, per volume
- foreach (var source in sourcesPerVolume.Value)
+ foreach (var source in volumeSources)
{
foreach (var entry in journal.GetChangedFileSystemEntries(source, data.NextUsn))
{
@@ -122,6 +123,11 @@ namespace Duplicati.Library.Snapshots
}
}
+ // prepare cache for includes (value = true) and excludes (value = false, will be populated
+ // on-demand)
+ var cache = new Dictionary();
+ volumeSources.ForEach(p => cache[p] = true);
+
// At this point we have:
// - a list of folders (changedFolders) that were possibly modified
// - a list of files (changedFiles) that were possibly modified
@@ -137,8 +143,9 @@ namespace Duplicati.Library.Snapshots
// specific "C:\A\B\" in our list, even though it's meant to be excluded.
// The reason why the filter doesn't exclude it is because during a regular (non-USN) full scan,
// FilterHandler.EnumerateFilesAndFolders() works top-down, and won't even enumerate child
- // folders.
- var cache = new HashSet(); // cache to speed up processing
+ // folders.
+ // The sources are needed to stop evaluating parent folders above the specified source folders
+
result.Folders.AddRange(FilterExcludedFolders(simplifiedFolders, filter, cache));
// 3. Our list of files may contain entries inside one of the simplified folders (from step 1., above).
@@ -178,10 +185,10 @@ namespace Duplicati.Library.Snapshots
///
/// Files to filter
/// Exclusion filter
- /// Cache of excluded folders (optional)
+ /// Cache of included and exculded files / folders
/// Filtered files
private IEnumerable FilterExcludedFiles(IEnumerable files,
- Utility.Utility.EnumerationFilterDelegate filter, ISet cache = null)
+ Utility.Utility.EnumerationFilterDelegate filter, IDictionary cache)
{
var result = new List();
@@ -221,7 +228,7 @@ namespace Duplicati.Library.Snapshots
/// Cache of excluded folders (optional)
/// Filtered folders
private IEnumerable FilterExcludedFolders(IEnumerable folders,
- Utility.Utility.EnumerationFilterDelegate filter, ISet cache = null)
+ Utility.Utility.EnumerationFilterDelegate filter, IDictionary cache)
{
var result = new List();
@@ -255,26 +262,29 @@ namespace Duplicati.Library.Snapshots
/// Filter
/// Cache of excluded folders (optional)
/// True if excluded, false otherwise
- private bool IsFolderOrAncestorsExcluded(string folder, Utility.Utility.EnumerationFilterDelegate filter, ISet cache = null)
+ private bool IsFolderOrAncestorsExcluded(string folder, Utility.Utility.EnumerationFilterDelegate filter, IDictionary cache)
{
List parents = null;
while (folder != null)
{
// first check cache
- if (cache != null)
+ if (cache.TryGetValue(folder, out var include))
{
- if (cache.Contains(folder))
- break; // hit!
+ if (include)
+ return false;
- // remember folder for cache
- if (parents == null)
- {
- parents = new List(); // create on-demand
- }
- parents.Add(folder);
+ break; // hit!
}
- var attr = m_snapshot.FolderExists(folder) ? m_snapshot.GetAttributes(folder) : FileAttributes.Directory;
+ // remember folder for cache
+ if (parents == null)
+ {
+ parents = new List(); // create on-demand
+ }
+ parents.Add(folder);
+
+
+ var attr = m_snapshot.DirectoryExists(folder) ? m_snapshot.GetAttributes(folder) : FileAttributes.Directory;
if (!filter(folder, folder, attr))
break; // excluded
@@ -282,10 +292,10 @@ namespace Duplicati.Library.Snapshots
folder = Utility.Utility.GetParent(folder, true);
}
- if (folder != null && parents != null)
+ if (folder != null)
{
// update cache
- cache.UnionWith(parents);
+ parents?.ForEach(p => cache[p] = false);
}
return folder != null;
diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs
index a677de231..84e1c5e55 100644
--- a/Duplicati/Library/Utility/Utility.cs
+++ b/Duplicati/Library/Utility/Utility.cs
@@ -254,7 +254,7 @@ namespace Duplicati.Library.Utility
/// A list of the full filenames
public static IEnumerable EnumerateFileSystemEntries(string rootpath, EnumerationFilterDelegate callback, FileSystemInteraction folderList, FileSystemInteraction fileList, ExtractFileAttributes attributeReader, ReportAccessError errorCallback = null)
{
- Stack lst = new Stack();
+ var lst = new Stack();
if (IsFolder(rootpath, attributeReader))
{
@@ -262,7 +262,7 @@ namespace Duplicati.Library.Utility
try
{
- FileAttributes attr = attributeReader == null ? FileAttributes.Directory : attributeReader(rootpath);
+ var attr = attributeReader?.Invoke(rootpath) ?? FileAttributes.Directory;
if (callback(rootpath, rootpath, attr))
lst.Push(rootpath);
}
@@ -272,25 +272,24 @@ namespace Duplicati.Library.Utility
}
catch (Exception ex)
{
- if (errorCallback != null)
- errorCallback(rootpath, rootpath, ex);
+ errorCallback?.Invoke(rootpath, rootpath, ex);
callback(rootpath, rootpath, FileAttributes.Directory | ATTRIBUTE_ERROR);
}
while (lst.Count > 0)
{
- string f = AppendDirSeparator(lst.Pop());
+ var f = AppendDirSeparator(lst.Pop());
yield return f;
try
{
- foreach (string s in folderList(f))
+ foreach (var s in folderList(f))
{
var sf = AppendDirSeparator(s);
try
{
- FileAttributes attr = attributeReader == null ? FileAttributes.Directory : attributeReader(sf);
+ var attr = attributeReader?.Invoke(sf) ?? FileAttributes.Directory;
if (callback(rootpath, sf, attr))
lst.Push(sf);
}
@@ -300,8 +299,7 @@ namespace Duplicati.Library.Utility
}
catch (Exception ex)
{
- if (errorCallback != null)
- errorCallback(rootpath, sf, ex);
+ errorCallback?.Invoke(rootpath, sf, ex);
callback(rootpath, sf, FileAttributes.Directory | ATTRIBUTE_ERROR);
}
}
@@ -312,13 +310,13 @@ namespace Duplicati.Library.Utility
}
catch (Exception ex)
{
- if (errorCallback != null)
- errorCallback(rootpath, f, ex);
+ errorCallback?.Invoke(rootpath, f, ex);
callback(rootpath, f, FileAttributes.Directory | ATTRIBUTE_ERROR);
}
string[] files = null;
if (fileList != null)
+ {
try
{
files = fileList(f);
@@ -329,17 +327,18 @@ namespace Duplicati.Library.Utility
}
catch (Exception ex)
{
- if (errorCallback != null)
- errorCallback(rootpath, f, ex);
+ errorCallback?.Invoke(rootpath, f, ex);
callback(rootpath, f, FileAttributes.Directory | ATTRIBUTE_ERROR);
}
+ }
if (files != null)
+ {
foreach (var s in files)
{
try
{
- FileAttributes attr = attributeReader == null ? FileAttributes.Normal : attributeReader(s);
+ var attr = attributeReader?.Invoke(s) ?? FileAttributes.Normal;
if (!callback(rootpath, s, attr))
continue;
}
@@ -349,20 +348,20 @@ namespace Duplicati.Library.Utility
}
catch (Exception ex)
{
- if (errorCallback != null)
- errorCallback(rootpath, s, ex);
+ errorCallback?.Invoke(rootpath, s, ex);
callback(rootpath, s, ATTRIBUTE_ERROR);
continue;
}
yield return s;
}
+ }
}
}
else
{
try
{
- FileAttributes attr = attributeReader == null ? FileAttributes.Normal : attributeReader(rootpath);
+ var attr = attributeReader?.Invoke(rootpath) ?? FileAttributes.Normal;
if (!callback(rootpath, rootpath, attr))
yield break;
}
@@ -372,8 +371,7 @@ namespace Duplicati.Library.Utility
}
catch (Exception ex)
{
- if (errorCallback != null)
- errorCallback(rootpath, rootpath, ex);
+ errorCallback?.Invoke(rootpath, rootpath, ex);
callback(rootpath, rootpath, ATTRIBUTE_ERROR);
yield break;
}
@@ -531,9 +529,7 @@ namespace Duplicati.Library.Utility
/// The path with the directory separator appended
public static string AppendDirSeparator(string path)
{
- return !path.EndsWith(DirectorySeparatorString, StringComparison.Ordinal)
- ? path + DirectorySeparatorString
- : path;
+ return AppendDirSeparator(path, DirectorySeparatorString);
}
///
@@ -545,7 +541,9 @@ namespace Duplicati.Library.Utility
/// The path with the directory separator appended
public static string AppendDirSeparator(string path, string separator)
{
- return AppendDirSeparator(separator);
+ return !path.EndsWith(DirectorySeparatorString, StringComparison.Ordinal)
+ ? path + separator
+ : path;
}
///