haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed post-processing of modified file list obtain from USN

Browse Source
This commit is contained in:
Daniel Gehriger
2018-04-22 23:03:15 +02:00
parent 59af7c8057
commit a7d60d88d7
6 changed files with 66 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Main/Operation/BackupHandler.cs
+1 -1
View File
@@ -444,7 +444,7 @@ namespace Duplicati.Library.Main.Operation
// the modified sources may contain non-existent files / folders, which need to
// be removed before trying to enumerate them
var existingModifiedSources = modifiedSources.Files.Where(snapshot.FileExists)
.Concat(modifiedSources.Folders.Where(snapshot.FolderExists));
.Concat(modifiedSources.Folders.Where(snapshot.DirectoryExists));
foreach (var path in filterhandler.EnumerateFilesAndFolders(existingModifiedSources))
{
Duplicati/Library/Snapshots/ISnapshotService.cs
+1 -1
View File
@@ -127,7 +127,7 @@ namespace Duplicati.Library.Snapshots
/// </summary>
/// <param name="localFolderPath">The local path</param>
/// <returns>True if folder exists, false otherwise</returns>
bool FolderExists(string localFolderPath);
bool DirectoryExists(string localFolderPath);
/// <summary>
/// Tests if a file or folder exists in the snapshot
Duplicati/Library/Snapshots/SnapshotBase.cs
+5 -5
View File
@@ -50,7 +50,7 @@ namespace Duplicati.Library.Snapshots
public IEnumerable<string> EnumerateFilesAndFolders(IEnumerable<string> sources, Utility.Utility.EnumerationFilterDelegate callback, Utility.Utility.ReportAccessError errorCallback)
{
// Add trailing slashes to folders
var sanitizedSources = sources.Select(x => Directory.Exists(x) ? Utility.Utility.AppendDirSeparator(x) : x).ToList();
var sanitizedSources = sources.Select(x => DirectoryExists(x) ? Utility.Utility.AppendDirSeparator(x) : x).ToList();
return sanitizedSources.SelectMany(
s => Utility.Utility.EnumerateFileSystemEntries(s, callback, ListFolders, ListFiles, GetAttributes, errorCallback)
@@ -152,19 +152,19 @@ namespace Duplicati.Library.Snapshots
/// <inheritdoc />
public bool FileExists(string localFilePath)
{
return new FileInfo(ConvertToSnapshotPath(localFilePath)).Exists;
return File.Exists(ConvertToSnapshotPath(localFilePath));
}
/// <inheritdoc />
public bool FolderExists(string localFolderPath)
public bool DirectoryExists(string localFolderPath)
{
return new DirectoryInfo(ConvertToSnapshotPath(localFolderPath)).Exists;
return Directory.Exists(ConvertToSnapshotPath(localFolderPath));
}
/// <inheritdoc />
public bool PathExists(string localFileOrFolderPath)
{
return FileExists(localFileOrFolderPath) || FolderExists(localFileOrFolderPath);
return FileExists(localFileOrFolderPath) || DirectoryExists(localFileOrFolderPath);
}
/// <inheritdoc />
Duplicati/Library/Snapshots/USNJournal.cs
+9 -8
View File
@@ -133,30 +133,31 @@ namespace Duplicati.Library.Snapshots
/// <summary>
/// Returns a list of files or folders that have changed since the recorded USN
/// </summary>
/// <param name="sourceFolder">The folder to find entries for</param>
/// <param name="sourceFileOrFolder">The file or folder to find entries for</param>
/// <param name="minUsn">Minimum USN of entry</param>
/// <returns>A list of tuples with changed files and folders and their type</returns>
public IEnumerable<Tuple<string, EntryType>> GetChangedFileSystemEntries(string sourceFolder, long minUsn)
public IEnumerable<Tuple<string, EntryType>> GetChangedFileSystemEntries(string sourceFileOrFolder, long minUsn)
{
return GetChangedFileSystemEntries(sourceFolder, minUsn, ChangeReason.Any);
return GetChangedFileSystemEntries(sourceFileOrFolder, minUsn, ChangeReason.Any);
}
/// <summary>
/// Returns a list of files or folders that have changed since the recorded USN
/// </summary>
/// <param name="sourceFolder">The folder to find entries for</param>
/// <param name="sourceFileOrFolder">The file or folder to find entries for</param>
/// <param name="minUsn">Minimum USN of entry</param>
/// <param name="reason">Filter expression for change reason</param>
/// <returns>A list of tuples with changed files and folders and their type</returns>
public IEnumerable<Tuple<string, EntryType>> GetChangedFileSystemEntries(string sourceFolder, long minUsn, ChangeReason reason)
public IEnumerable<Tuple<string, EntryType>> GetChangedFileSystemEntries(string sourceFileOrFolder, long minUsn, ChangeReason reason)
{
var isFolder = sourceFileOrFolder.EndsWith(Utility.Utility.DirectorySeparatorString, StringComparison.Ordinal);
foreach (var r in GetRecords(minUsn))
{
if (r.UsnRecord.Usn >= minUsn
&& (reason == ChangeReason.Any || (MapChangeReason(r.UsnRecord.Reason) & reason) != 0)
&& (string.IsNullOrEmpty(sourceFolder) || r.FullPath.Equals(sourceFolder,
Utility.Utility.ClientFilenameStringComparision)
|| Utility.Utility.IsPathBelowFolder(r.FullPath, sourceFolder)))
&& (r.FullPath.Equals(sourceFileOrFolder, Utility.Utility.ClientFilenameStringComparision)
|| isFolder && Utility.Utility.IsPathBelowFolder(r.FullPath, sourceFileOrFolder)))
{
yield return Tuple.Create(r.FullPath,
r.UsnRecord.FileAttributes.HasFlag(Win32USN.FileAttributes.Directory)
Duplicati/Library/Snapshots/UsnJournalService.cs
+29 -19
View File
@@ -75,6 +75,7 @@ namespace Duplicati.Library.Snapshots
foreach (var sourcesPerVolume in SortByVolume(sources))
{
var volume = sourcesPerVolume.Key;
var volumeSources = sourcesPerVolume.Value;
try
{
@@ -107,7 +108,7 @@ namespace Duplicati.Library.Snapshots
var changedFolders = new HashSet<string>(Utility.Utility.ClientFilenameStringComparer);
// obtain changed files and folders, per volume
foreach (var source in sourcesPerVolume.Value)
foreach (var source in volumeSources)
{
foreach (var entry in journal.GetChangedFileSystemEntries(source, data.NextUsn))
{
@@ -122,6 +123,11 @@ namespace Duplicati.Library.Snapshots
}
}
// prepare cache for includes (value = true) and excludes (value = false, will be populated
// on-demand)
var cache = new Dictionary<string, bool>();
volumeSources.ForEach(p => cache[p] = true);
// At this point we have:
// - a list of folders (changedFolders) that were possibly modified
// - a list of files (changedFiles) that were possibly modified
@@ -137,8 +143,9 @@ namespace Duplicati.Library.Snapshots
// specific "C:\A\B\" in our list, even though it's meant to be excluded.
// The reason why the filter doesn't exclude it is because during a regular (non-USN) full scan,
// FilterHandler.EnumerateFilesAndFolders() works top-down, and won't even enumerate child
// folders.
var cache = new HashSet<string>(); // cache to speed up processing
// folders.
// The sources are needed to stop evaluating parent folders above the specified source folders
result.Folders.AddRange(FilterExcludedFolders(simplifiedFolders, filter, cache));
// 3. Our list of files may contain entries inside one of the simplified folders (from step 1., above).
@@ -178,10 +185,10 @@ namespace Duplicati.Library.Snapshots
/// </summary>
/// <param name="files">Files to filter</param>
/// <param name="filter">Exclusion filter</param>
/// <param name="cache">Cache of excluded folders (optional)</param>
/// <param name="cache">Cache of included and exculded files / folders</param>
/// <returns>Filtered files</returns>
private IEnumerable<string> FilterExcludedFiles(IEnumerable<string> files,
Utility.Utility.EnumerationFilterDelegate filter, ISet<string> cache = null)
Utility.Utility.EnumerationFilterDelegate filter, IDictionary<string, bool> cache)
{
var result = new List<string>();
@@ -221,7 +228,7 @@ namespace Duplicati.Library.Snapshots
/// <param name="cache">Cache of excluded folders (optional)</param>
/// <returns>Filtered folders</returns>
private IEnumerable<string> FilterExcludedFolders(IEnumerable<string> folders,
Utility.Utility.EnumerationFilterDelegate filter, ISet<string> cache = null)
Utility.Utility.EnumerationFilterDelegate filter, IDictionary<string, bool> cache)
{
var result = new List<string>();
@@ -255,26 +262,29 @@ namespace Duplicati.Library.Snapshots
/// <param name="filter">Filter</param>
/// <param name="cache">Cache of excluded folders (optional)</param>
/// <returns>True if excluded, false otherwise</returns>
private bool IsFolderOrAncestorsExcluded(string folder, Utility.Utility.EnumerationFilterDelegate filter, ISet<string> cache = null)
private bool IsFolderOrAncestorsExcluded(string folder, Utility.Utility.EnumerationFilterDelegate filter, IDictionary<string, bool> cache)
{
List<string> parents = null;
while (folder != null)
{
// first check cache
if (cache != null)
if (cache.TryGetValue(folder, out var include))
{
if (cache.Contains(folder))
break; // hit!
if (include)
return false;
// remember folder for cache
if (parents == null)
{
parents = new List<string>(); // create on-demand
}
parents.Add(folder);
break; // hit!
}
var attr = m_snapshot.FolderExists(folder) ? m_snapshot.GetAttributes(folder) : FileAttributes.Directory;
// remember folder for cache
if (parents == null)
{
parents = new List<string>(); // create on-demand
}
parents.Add(folder);
var attr = m_snapshot.DirectoryExists(folder) ? m_snapshot.GetAttributes(folder) : FileAttributes.Directory;
if (!filter(folder, folder, attr))
break; // excluded
@@ -282,10 +292,10 @@ namespace Duplicati.Library.Snapshots
folder = Utility.Utility.GetParent(folder, true);
}
if (folder != null && parents != null)
if (folder != null)
{
// update cache
cache.UnionWith(parents);
parents?.ForEach(p => cache[p] = false);
}
return folder != null;
Duplicati/Library/Utility/Utility.cs
+21 -23
View File
@@ -254,7 +254,7 @@ namespace Duplicati.Library.Utility
/// <returns>A list of the full filenames</returns>
public static IEnumerable<string> EnumerateFileSystemEntries(string rootpath, EnumerationFilterDelegate callback, FileSystemInteraction folderList, FileSystemInteraction fileList, ExtractFileAttributes attributeReader, ReportAccessError errorCallback = null)
{
Stack<string> lst = new Stack<string>();
var lst = new Stack<string>();
if (IsFolder(rootpath, attributeReader))
{
@@ -262,7 +262,7 @@ namespace Duplicati.Library.Utility
try
{
FileAttributes attr = attributeReader == null ? FileAttributes.Directory : attributeReader(rootpath);
var attr = attributeReader?.Invoke(rootpath) ?? FileAttributes.Directory;
if (callback(rootpath, rootpath, attr))
lst.Push(rootpath);
}
@@ -272,25 +272,24 @@ namespace Duplicati.Library.Utility
}
catch (Exception ex)
{
if (errorCallback != null)
errorCallback(rootpath, rootpath, ex);
errorCallback?.Invoke(rootpath, rootpath, ex);
callback(rootpath, rootpath, FileAttributes.Directory | ATTRIBUTE_ERROR);
}
while (lst.Count > 0)
{
string f = AppendDirSeparator(lst.Pop());
var f = AppendDirSeparator(lst.Pop());
yield return f;
try
{
foreach (string s in folderList(f))
foreach (var s in folderList(f))
{
var sf = AppendDirSeparator(s);
try
{
FileAttributes attr = attributeReader == null ? FileAttributes.Directory : attributeReader(sf);
var attr = attributeReader?.Invoke(sf) ?? FileAttributes.Directory;
if (callback(rootpath, sf, attr))
lst.Push(sf);
}
@@ -300,8 +299,7 @@ namespace Duplicati.Library.Utility
}
catch (Exception ex)
{
if (errorCallback != null)
errorCallback(rootpath, sf, ex);
errorCallback?.Invoke(rootpath, sf, ex);
callback(rootpath, sf, FileAttributes.Directory | ATTRIBUTE_ERROR);
}
}
@@ -312,13 +310,13 @@ namespace Duplicati.Library.Utility
}
catch (Exception ex)
{
if (errorCallback != null)
errorCallback(rootpath, f, ex);
errorCallback?.Invoke(rootpath, f, ex);
callback(rootpath, f, FileAttributes.Directory | ATTRIBUTE_ERROR);
}
string[] files = null;
if (fileList != null)
{
try
{
files = fileList(f);
@@ -329,17 +327,18 @@ namespace Duplicati.Library.Utility
}
catch (Exception ex)
{
if (errorCallback != null)
errorCallback(rootpath, f, ex);
errorCallback?.Invoke(rootpath, f, ex);
callback(rootpath, f, FileAttributes.Directory | ATTRIBUTE_ERROR);
}
}
if (files != null)
{
foreach (var s in files)
{
try
{
FileAttributes attr = attributeReader == null ? FileAttributes.Normal : attributeReader(s);
var attr = attributeReader?.Invoke(s) ?? FileAttributes.Normal;
if (!callback(rootpath, s, attr))
continue;
}
@@ -349,20 +348,20 @@ namespace Duplicati.Library.Utility
}
catch (Exception ex)
{
if (errorCallback != null)
errorCallback(rootpath, s, ex);
errorCallback?.Invoke(rootpath, s, ex);
callback(rootpath, s, ATTRIBUTE_ERROR);
continue;
}
yield return s;
}
}
}
}
else
{
try
{
FileAttributes attr = attributeReader == null ? FileAttributes.Normal : attributeReader(rootpath);
var attr = attributeReader?.Invoke(rootpath) ?? FileAttributes.Normal;
if (!callback(rootpath, rootpath, attr))
yield break;
}
@@ -372,8 +371,7 @@ namespace Duplicati.Library.Utility
}
catch (Exception ex)
{
if (errorCallback != null)
errorCallback(rootpath, rootpath, ex);
errorCallback?.Invoke(rootpath, rootpath, ex);
callback(rootpath, rootpath, ATTRIBUTE_ERROR);
yield break;
}
@@ -531,9 +529,7 @@ namespace Duplicati.Library.Utility
/// <returns>The path with the directory separator appended</returns>
public static string AppendDirSeparator(string path)
{
return !path.EndsWith(DirectorySeparatorString, StringComparison.Ordinal)
? path + DirectorySeparatorString
: path;
return AppendDirSeparator(path, DirectorySeparatorString);
}
/// <summary>
@@ -545,7 +541,9 @@ namespace Duplicati.Library.Utility
/// <returns>The path with the directory separator appended</returns>
public static string AppendDirSeparator(string path, string separator)
{
return AppendDirSeparator(separator);
return !path.EndsWith(DirectorySeparatorString, StringComparison.Ordinal)
? path + separator
: path;
}
/// <summary>