From a7d60d88d7cb12e6ddf73a4b52d5059bce8bdfea Mon Sep 17 00:00:00 2001 From: Daniel Gehriger Date: Sun, 22 Apr 2018 23:03:15 +0200 Subject: [PATCH] Fixed post-processing of modified file list obtain from USN --- .../Library/Main/Operation/BackupHandler.cs | 2 +- .../Library/Snapshots/ISnapshotService.cs | 2 +- Duplicati/Library/Snapshots/SnapshotBase.cs | 10 ++-- Duplicati/Library/Snapshots/USNJournal.cs | 17 +++---- .../Library/Snapshots/UsnJournalService.cs | 48 +++++++++++-------- Duplicati/Library/Utility/Utility.cs | 44 ++++++++--------- 6 files changed, 66 insertions(+), 57 deletions(-) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 3c5eabd2c..3a16c1f6e 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -444,7 +444,7 @@ namespace Duplicati.Library.Main.Operation // the modified sources may contain non-existent files / folders, which need to // be removed before trying to enumerate them var existingModifiedSources = modifiedSources.Files.Where(snapshot.FileExists) - .Concat(modifiedSources.Folders.Where(snapshot.FolderExists)); + .Concat(modifiedSources.Folders.Where(snapshot.DirectoryExists)); foreach (var path in filterhandler.EnumerateFilesAndFolders(existingModifiedSources)) { diff --git a/Duplicati/Library/Snapshots/ISnapshotService.cs b/Duplicati/Library/Snapshots/ISnapshotService.cs index 7aceef496..15e554790 100644 --- a/Duplicati/Library/Snapshots/ISnapshotService.cs +++ b/Duplicati/Library/Snapshots/ISnapshotService.cs @@ -127,7 +127,7 @@ namespace Duplicati.Library.Snapshots /// /// The local path /// True if folder exists, false otherwise - bool FolderExists(string localFolderPath); + bool DirectoryExists(string localFolderPath); /// /// Tests if a file or folder exists in the snapshot diff --git a/Duplicati/Library/Snapshots/SnapshotBase.cs b/Duplicati/Library/Snapshots/SnapshotBase.cs index 130222cb9..d32d090d4 100644 --- a/Duplicati/Library/Snapshots/SnapshotBase.cs +++ b/Duplicati/Library/Snapshots/SnapshotBase.cs @@ -50,7 +50,7 @@ namespace Duplicati.Library.Snapshots public IEnumerable EnumerateFilesAndFolders(IEnumerable sources, Utility.Utility.EnumerationFilterDelegate callback, Utility.Utility.ReportAccessError errorCallback) { // Add trailing slashes to folders - var sanitizedSources = sources.Select(x => Directory.Exists(x) ? Utility.Utility.AppendDirSeparator(x) : x).ToList(); + var sanitizedSources = sources.Select(x => DirectoryExists(x) ? Utility.Utility.AppendDirSeparator(x) : x).ToList(); return sanitizedSources.SelectMany( s => Utility.Utility.EnumerateFileSystemEntries(s, callback, ListFolders, ListFiles, GetAttributes, errorCallback) @@ -152,19 +152,19 @@ namespace Duplicati.Library.Snapshots /// public bool FileExists(string localFilePath) { - return new FileInfo(ConvertToSnapshotPath(localFilePath)).Exists; + return File.Exists(ConvertToSnapshotPath(localFilePath)); } /// - public bool FolderExists(string localFolderPath) + public bool DirectoryExists(string localFolderPath) { - return new DirectoryInfo(ConvertToSnapshotPath(localFolderPath)).Exists; + return Directory.Exists(ConvertToSnapshotPath(localFolderPath)); } /// public bool PathExists(string localFileOrFolderPath) { - return FileExists(localFileOrFolderPath) || FolderExists(localFileOrFolderPath); + return FileExists(localFileOrFolderPath) || DirectoryExists(localFileOrFolderPath); } /// diff --git a/Duplicati/Library/Snapshots/USNJournal.cs b/Duplicati/Library/Snapshots/USNJournal.cs index c34ddac3e..02d7b0492 100644 --- a/Duplicati/Library/Snapshots/USNJournal.cs +++ b/Duplicati/Library/Snapshots/USNJournal.cs @@ -133,30 +133,31 @@ namespace Duplicati.Library.Snapshots /// /// Returns a list of files or folders that have changed since the recorded USN /// - /// The folder to find entries for + /// The file or folder to find entries for /// Minimum USN of entry /// A list of tuples with changed files and folders and their type - public IEnumerable> GetChangedFileSystemEntries(string sourceFolder, long minUsn) + public IEnumerable> GetChangedFileSystemEntries(string sourceFileOrFolder, long minUsn) { - return GetChangedFileSystemEntries(sourceFolder, minUsn, ChangeReason.Any); + return GetChangedFileSystemEntries(sourceFileOrFolder, minUsn, ChangeReason.Any); } /// /// Returns a list of files or folders that have changed since the recorded USN /// - /// The folder to find entries for + /// The file or folder to find entries for /// Minimum USN of entry /// Filter expression for change reason /// A list of tuples with changed files and folders and their type - public IEnumerable> GetChangedFileSystemEntries(string sourceFolder, long minUsn, ChangeReason reason) + public IEnumerable> GetChangedFileSystemEntries(string sourceFileOrFolder, long minUsn, ChangeReason reason) { + var isFolder = sourceFileOrFolder.EndsWith(Utility.Utility.DirectorySeparatorString, StringComparison.Ordinal); + foreach (var r in GetRecords(minUsn)) { if (r.UsnRecord.Usn >= minUsn && (reason == ChangeReason.Any || (MapChangeReason(r.UsnRecord.Reason) & reason) != 0) - && (string.IsNullOrEmpty(sourceFolder) || r.FullPath.Equals(sourceFolder, - Utility.Utility.ClientFilenameStringComparision) - || Utility.Utility.IsPathBelowFolder(r.FullPath, sourceFolder))) + && (r.FullPath.Equals(sourceFileOrFolder, Utility.Utility.ClientFilenameStringComparision) + || isFolder && Utility.Utility.IsPathBelowFolder(r.FullPath, sourceFileOrFolder))) { yield return Tuple.Create(r.FullPath, r.UsnRecord.FileAttributes.HasFlag(Win32USN.FileAttributes.Directory) diff --git a/Duplicati/Library/Snapshots/UsnJournalService.cs b/Duplicati/Library/Snapshots/UsnJournalService.cs index 38bb53b9a..b5f0c5f69 100644 --- a/Duplicati/Library/Snapshots/UsnJournalService.cs +++ b/Duplicati/Library/Snapshots/UsnJournalService.cs @@ -75,6 +75,7 @@ namespace Duplicati.Library.Snapshots foreach (var sourcesPerVolume in SortByVolume(sources)) { var volume = sourcesPerVolume.Key; + var volumeSources = sourcesPerVolume.Value; try { @@ -107,7 +108,7 @@ namespace Duplicati.Library.Snapshots var changedFolders = new HashSet(Utility.Utility.ClientFilenameStringComparer); // obtain changed files and folders, per volume - foreach (var source in sourcesPerVolume.Value) + foreach (var source in volumeSources) { foreach (var entry in journal.GetChangedFileSystemEntries(source, data.NextUsn)) { @@ -122,6 +123,11 @@ namespace Duplicati.Library.Snapshots } } + // prepare cache for includes (value = true) and excludes (value = false, will be populated + // on-demand) + var cache = new Dictionary(); + volumeSources.ForEach(p => cache[p] = true); + // At this point we have: // - a list of folders (changedFolders) that were possibly modified // - a list of files (changedFiles) that were possibly modified @@ -137,8 +143,9 @@ namespace Duplicati.Library.Snapshots // specific "C:\A\B\" in our list, even though it's meant to be excluded. // The reason why the filter doesn't exclude it is because during a regular (non-USN) full scan, // FilterHandler.EnumerateFilesAndFolders() works top-down, and won't even enumerate child - // folders. - var cache = new HashSet(); // cache to speed up processing + // folders. + // The sources are needed to stop evaluating parent folders above the specified source folders + result.Folders.AddRange(FilterExcludedFolders(simplifiedFolders, filter, cache)); // 3. Our list of files may contain entries inside one of the simplified folders (from step 1., above). @@ -178,10 +185,10 @@ namespace Duplicati.Library.Snapshots /// /// Files to filter /// Exclusion filter - /// Cache of excluded folders (optional) + /// Cache of included and exculded files / folders /// Filtered files private IEnumerable FilterExcludedFiles(IEnumerable files, - Utility.Utility.EnumerationFilterDelegate filter, ISet cache = null) + Utility.Utility.EnumerationFilterDelegate filter, IDictionary cache) { var result = new List(); @@ -221,7 +228,7 @@ namespace Duplicati.Library.Snapshots /// Cache of excluded folders (optional) /// Filtered folders private IEnumerable FilterExcludedFolders(IEnumerable folders, - Utility.Utility.EnumerationFilterDelegate filter, ISet cache = null) + Utility.Utility.EnumerationFilterDelegate filter, IDictionary cache) { var result = new List(); @@ -255,26 +262,29 @@ namespace Duplicati.Library.Snapshots /// Filter /// Cache of excluded folders (optional) /// True if excluded, false otherwise - private bool IsFolderOrAncestorsExcluded(string folder, Utility.Utility.EnumerationFilterDelegate filter, ISet cache = null) + private bool IsFolderOrAncestorsExcluded(string folder, Utility.Utility.EnumerationFilterDelegate filter, IDictionary cache) { List parents = null; while (folder != null) { // first check cache - if (cache != null) + if (cache.TryGetValue(folder, out var include)) { - if (cache.Contains(folder)) - break; // hit! + if (include) + return false; - // remember folder for cache - if (parents == null) - { - parents = new List(); // create on-demand - } - parents.Add(folder); + break; // hit! } - var attr = m_snapshot.FolderExists(folder) ? m_snapshot.GetAttributes(folder) : FileAttributes.Directory; + // remember folder for cache + if (parents == null) + { + parents = new List(); // create on-demand + } + parents.Add(folder); + + + var attr = m_snapshot.DirectoryExists(folder) ? m_snapshot.GetAttributes(folder) : FileAttributes.Directory; if (!filter(folder, folder, attr)) break; // excluded @@ -282,10 +292,10 @@ namespace Duplicati.Library.Snapshots folder = Utility.Utility.GetParent(folder, true); } - if (folder != null && parents != null) + if (folder != null) { // update cache - cache.UnionWith(parents); + parents?.ForEach(p => cache[p] = false); } return folder != null; diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index a677de231..84e1c5e55 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -254,7 +254,7 @@ namespace Duplicati.Library.Utility /// A list of the full filenames public static IEnumerable EnumerateFileSystemEntries(string rootpath, EnumerationFilterDelegate callback, FileSystemInteraction folderList, FileSystemInteraction fileList, ExtractFileAttributes attributeReader, ReportAccessError errorCallback = null) { - Stack lst = new Stack(); + var lst = new Stack(); if (IsFolder(rootpath, attributeReader)) { @@ -262,7 +262,7 @@ namespace Duplicati.Library.Utility try { - FileAttributes attr = attributeReader == null ? FileAttributes.Directory : attributeReader(rootpath); + var attr = attributeReader?.Invoke(rootpath) ?? FileAttributes.Directory; if (callback(rootpath, rootpath, attr)) lst.Push(rootpath); } @@ -272,25 +272,24 @@ namespace Duplicati.Library.Utility } catch (Exception ex) { - if (errorCallback != null) - errorCallback(rootpath, rootpath, ex); + errorCallback?.Invoke(rootpath, rootpath, ex); callback(rootpath, rootpath, FileAttributes.Directory | ATTRIBUTE_ERROR); } while (lst.Count > 0) { - string f = AppendDirSeparator(lst.Pop()); + var f = AppendDirSeparator(lst.Pop()); yield return f; try { - foreach (string s in folderList(f)) + foreach (var s in folderList(f)) { var sf = AppendDirSeparator(s); try { - FileAttributes attr = attributeReader == null ? FileAttributes.Directory : attributeReader(sf); + var attr = attributeReader?.Invoke(sf) ?? FileAttributes.Directory; if (callback(rootpath, sf, attr)) lst.Push(sf); } @@ -300,8 +299,7 @@ namespace Duplicati.Library.Utility } catch (Exception ex) { - if (errorCallback != null) - errorCallback(rootpath, sf, ex); + errorCallback?.Invoke(rootpath, sf, ex); callback(rootpath, sf, FileAttributes.Directory | ATTRIBUTE_ERROR); } } @@ -312,13 +310,13 @@ namespace Duplicati.Library.Utility } catch (Exception ex) { - if (errorCallback != null) - errorCallback(rootpath, f, ex); + errorCallback?.Invoke(rootpath, f, ex); callback(rootpath, f, FileAttributes.Directory | ATTRIBUTE_ERROR); } string[] files = null; if (fileList != null) + { try { files = fileList(f); @@ -329,17 +327,18 @@ namespace Duplicati.Library.Utility } catch (Exception ex) { - if (errorCallback != null) - errorCallback(rootpath, f, ex); + errorCallback?.Invoke(rootpath, f, ex); callback(rootpath, f, FileAttributes.Directory | ATTRIBUTE_ERROR); } + } if (files != null) + { foreach (var s in files) { try { - FileAttributes attr = attributeReader == null ? FileAttributes.Normal : attributeReader(s); + var attr = attributeReader?.Invoke(s) ?? FileAttributes.Normal; if (!callback(rootpath, s, attr)) continue; } @@ -349,20 +348,20 @@ namespace Duplicati.Library.Utility } catch (Exception ex) { - if (errorCallback != null) - errorCallback(rootpath, s, ex); + errorCallback?.Invoke(rootpath, s, ex); callback(rootpath, s, ATTRIBUTE_ERROR); continue; } yield return s; } + } } } else { try { - FileAttributes attr = attributeReader == null ? FileAttributes.Normal : attributeReader(rootpath); + var attr = attributeReader?.Invoke(rootpath) ?? FileAttributes.Normal; if (!callback(rootpath, rootpath, attr)) yield break; } @@ -372,8 +371,7 @@ namespace Duplicati.Library.Utility } catch (Exception ex) { - if (errorCallback != null) - errorCallback(rootpath, rootpath, ex); + errorCallback?.Invoke(rootpath, rootpath, ex); callback(rootpath, rootpath, ATTRIBUTE_ERROR); yield break; } @@ -531,9 +529,7 @@ namespace Duplicati.Library.Utility /// The path with the directory separator appended public static string AppendDirSeparator(string path) { - return !path.EndsWith(DirectorySeparatorString, StringComparison.Ordinal) - ? path + DirectorySeparatorString - : path; + return AppendDirSeparator(path, DirectorySeparatorString); } /// @@ -545,7 +541,9 @@ namespace Duplicati.Library.Utility /// The path with the directory separator appended public static string AppendDirSeparator(string path, string separator) { - return AppendDirSeparator(separator); + return !path.EndsWith(DirectorySeparatorString, StringComparison.Ordinal) + ? path + separator + : path; } ///