haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Correctly filtering list of changed files / folders using exclusion filter

Browse Source
This commit is contained in:
Daniel Gehriger
2018-04-20 17:30:08 +02:00
parent 96d1390412
commit b97d4e0dbb
12 changed files with 288 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Interface/USNJournalDataEntry.cs
+4 -4
View File
@@ -28,21 +28,21 @@ namespace Duplicati.Library.Interface
/// <summary>
/// Volume of journal
/// </summary>
public string Volume;
public string Volume { get; set; }
/// <summary>
/// Journal ID, as recorded in the journal
/// </summary>
public long JournalId;
public long JournalId { get; set; }
/// <summary>
/// USN to start at for next fileset (backup)
/// </summary>
public long NextUsn;
public long NextUsn { get; set; }
/// <summary>
/// A hash value representing the active exclusion filter
/// </summary>
public string ConfigHash;
public string ConfigHash { get; set; }
}
}
Duplicati/Library/Main/Operation/BackupHandler.cs
+4 -2
View File
@@ -420,7 +420,7 @@ namespace Duplicati.Library.Main.Operation
{
try
{
var journalService = new UsnJournalService(UsnErrorHandler, () =>
var journalService = new UsnJournalService(snapshot, UsnErrorHandler, () =>
{
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
throw new OperationCanceledException();
@@ -436,7 +436,7 @@ namespace Duplicati.Library.Main.Operation
// filter sources using USN journal, to obtain a sub-set of files / folders that
// may have been modified
// ReSharper disable once PossibleMultipleEnumeration
var modifiedSources = journalService.FilterSources(sources, configHash, journalData);
var modifiedSources = journalService.FilterSources(sources, filterhandler.AttributeFilter, configHash, journalData);
// scan modified sources for modifications
if (modifiedSources.Files.Any() || modifiedSources.Folders.Any())
@@ -477,6 +477,8 @@ namespace Duplicati.Library.Main.Operation
else
{
// add full previous fileset, as nothing has changed
//TODO: possible optimization is to somehow skip this step if compacting is enabled,
// as all this informatino will be removed anyway.
m_database.AppendFilesFromPreviousSet(m_transaction);
}
Duplicati/Library/Main/Operation/SystemInfoHandler.cs
+1 -1
View File
@@ -30,7 +30,7 @@ namespace Duplicati.Library.Main.Operation
public static IEnumerable<string> GetSystemInfo()
{
yield return string.Format("Duplicati: {0} ({1})", Duplicati.Library.Utility.Utility.getEntryAssembly().FullName, System.Reflection.Assembly.GetExecutingAssembly().FullName);
yield return string.Format("Duplicati: {0} ({1})", Duplicati.Library.Utility.Utility.GetEntryAssembly().FullName, System.Reflection.Assembly.GetExecutingAssembly().FullName);
yield return string.Format("Autoupdate urls: {0}", string.Join(";", Duplicati.Library.AutoUpdater.AutoUpdateSettings.URLs));
yield return string.Format("Update folder: {0}", Duplicati.Library.AutoUpdater.UpdaterManager.INSTALLDIR);
Duplicati/Library/Main/Options.cs
+1 -1
View File
@@ -395,7 +395,7 @@ namespace Duplicati.Library.Main
{
get
{
return System.IO.Path.GetFileNameWithoutExtension(Library.Utility.Utility.getEntryAssembly().Location);
return System.IO.Path.GetFileNameWithoutExtension(Library.Utility.Utility.GetEntryAssembly().Location);
}
}
Duplicati/Library/Modules/Builtin/ReportHelper.cs
+2 -2
View File
@@ -314,7 +314,7 @@ namespace Duplicati.Library.Modules.Builtin
if (m_options.ContainsKey("backup-name"))
extra["backup-name"] = m_options["backup-name"];
else
extra["backup-name"] = System.IO.Path.GetFileNameWithoutExtension(Duplicati.Library.Utility.Utility.getEntryAssembly().Location);
extra["backup-name"] = System.IO.Path.GetFileNameWithoutExtension(Duplicati.Library.Utility.Utility.GetEntryAssembly().Location);
}
foreach (KeyValuePair<string, string> kv in m_options)
@@ -344,7 +344,7 @@ namespace Duplicati.Library.Modules.Builtin
input = Regex.Replace(input, "\\%" + kv.Key + "\\%", kv.Value ?? "", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
if (!m_options.ContainsKey("backup-name"))
input = Regex.Replace(input, "\\%backup-name\\%", System.IO.Path.GetFileNameWithoutExtension(Duplicati.Library.Utility.Utility.getEntryAssembly().Location) ?? "", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
input = Regex.Replace(input, "\\%backup-name\\%", System.IO.Path.GetFileNameWithoutExtension(Duplicati.Library.Utility.Utility.GetEntryAssembly().Location) ?? "", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
input = Regex.Replace(input, "\\%[^\\%]+\\%", "");
return input;
Duplicati/Library/Modules/Builtin/RunScript.cs
+1 -1
View File
@@ -210,7 +210,7 @@ namespace Duplicati.Library.Modules.Builtin
psi.EnvironmentVariables["DUPLICATI__" + kv.Key.Replace('-', '_')] = kv.Value;
if (!options.ContainsKey("backup-name"))
psi.EnvironmentVariables["DUPLICATI__backup_name"] = System.IO.Path.GetFileNameWithoutExtension(Duplicati.Library.Utility.Utility.getEntryAssembly().Location);
psi.EnvironmentVariables["DUPLICATI__backup_name"] = System.IO.Path.GetFileNameWithoutExtension(Duplicati.Library.Utility.Utility.GetEntryAssembly().Location);
psi.EnvironmentVariables["DUPLICATI__EVENTNAME"] = eventname;
psi.EnvironmentVariables["DUPLICATI__OPERATIONNAME"] = operationname;
Duplicati/Library/Snapshots/IChangeJournalService.cs
+2 -1
View File
@@ -32,10 +32,11 @@ namespace Duplicati.Library.Snapshots
/// change, as specified by <c>journalData</c>.
/// </summary>
/// <param name="sources">Sources to filter</param>
/// <param name="filter">Filter callback to exclude filtered items</param>
/// <param name="filterHash">A hash value representing current exclusion filter. A full scan is triggered if hash has changed.</param>
/// <param name="journalData">The journal data of the previous backup (fileset)</param>
/// <returns>Filtered sources</returns>
FilterData FilterSources(IEnumerable<string> sources, string filterHash, IEnumerable<USNJournalDataEntry> journalData);
FilterData FilterSources(IEnumerable<string> sources, Utility.Utility.EnumerationFilterDelegate filter, string filterHash, IEnumerable<USNJournalDataEntry> journalData);
}
/// <summary>
Duplicati/Library/Snapshots/SnapshotBase.cs
+5 -1
View File
@@ -205,6 +205,8 @@ namespace Duplicati.Library.Snapshots
return Directory.GetFiles(localFolderPath);
}
#region IDisposable interface
/// <inheritdoc />
public void Dispose()
{
@@ -213,7 +215,9 @@ namespace Duplicati.Library.Snapshots
// Now suppress finalization for this object, since we've already handled our resource cleanup tasks
GC.SuppressFinalize(this);
}
}
#endregion
/// <summary>
/// Releases unmanaged and - optionally - managed resources
Duplicati/Library/Snapshots/UsnJournalService.cs
+160 -7
View File
@@ -22,6 +22,7 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Duplicati.Library.Interface;
using Duplicati.Library.Logging;
@@ -35,16 +36,20 @@ namespace Duplicati.Library.Snapshots
/// </summary>
private static readonly string LOGTAG = Log.LogTagFromType<UsnJournalService>();
private readonly ISnapshotService m_snapshot;
private readonly Utility.Utility.ReportAccessError m_errorCallback;
private readonly Action m_cancelHandler;
/// <summary>
/// Constructor.
/// </summary>
/// <param name="snapshot"></param>
/// <param name="errorCallback">Callback for handling errors</param>
/// <param name="cancelHandler">Callback for aborting operation. Throw OperationCancelledException() to abort.</param>
public UsnJournalService(Utility.Utility.ReportAccessError errorCallback, Action cancelHandler)
public UsnJournalService(ISnapshotService snapshot, Utility.Utility.ReportAccessError errorCallback,
Action cancelHandler)
{
m_snapshot = snapshot;
m_errorCallback = errorCallback;
m_cancelHandler = cancelHandler;
}
@@ -52,7 +57,8 @@ namespace Duplicati.Library.Snapshots
#region IChangeJournalService Members
/// <inheritdoc />
public FilterData FilterSources(IEnumerable<string> sources, string filterHash, IEnumerable<USNJournalDataEntry> journalData)
public FilterData FilterSources(IEnumerable<string> sources, Utility.Utility.EnumerationFilterDelegate filter,
string filterHash, IEnumerable<USNJournalDataEntry> journalData)
{
// create lookup for journal data
var journalDataDict = journalData.ToDictionary(data => data.Volume);
@@ -100,7 +106,7 @@ namespace Duplicati.Library.Snapshots
var changedFiles = new HashSet<string>(Utility.Utility.ClientFilenameStringComparer);
var changedFolders = new HashSet<string>(Utility.Utility.ClientFilenameStringComparer);
// iterate over sources and retrieve *renamed* directories
// obtain changed files and folders, per volume
foreach (var source in sourcesPerVolume.Value)
{
foreach (var entry in journal.GetChangedFileSystemEntries(source, data.NextUsn))
@@ -116,10 +122,38 @@ namespace Duplicati.Library.Snapshots
}
}
var reducedFolderList = Utility.Utility.SimplifyFolderList(changedFolders).ToList();
var reducedFileList = Utility.Utility.GetFilesNotInFolders(changedFiles, reducedFolderList);
result.Folders.AddRange(reducedFolderList);
result.Files.UnionWith(reducedFileList);
// At this point we have:
// - a list of folders (changedFolders) that were possibly modified
// - a list of files (changedFiles) that were possibly modified
//
// With this, we need still need to do the following:
//
// 1. Simplify the folder list, such that it only contains the parent-most entries
// (eg. { "C:\A\B\", "C:\A\B\C\", "C:\A\B\D\E\" } => { "C:\A\B\" }
var simplifiedFolders = Utility.Utility.SimplifyFolderList(changedFolders).ToList();
// 2. Check the simplified folders, and their parent folders against the exclusion filter.
// This is needed because the filter may exclude "C:\A\", but this won't match the more
// specific "C:\A\B\" in our list, even though it's meant to be excluded.
// The reason why the filter doesn't exclude it is because during a regular (non-USN) full scan,
// FilterHandler.EnumerateFilesAndFolders() works top-down, and won't even enumerate child
// folders.
var cache = new HashSet<string>(); // cache to speed up processing
result.Folders.AddRange(FilterExcludedFolders(simplifiedFolders, filter, cache));
// 3. Our list of files may contain entries inside one of the simplified folders (from step 1., above).
// Since that folder is going to be fully scanned, those files can be removed.
// Note: it would be wrong to use the result from step 2. as the folder list! The entries removed
// between 1. and 2. are *excluded* folders, and files below them are to be *excluded*, too.
var simplifiedFiles = Utility.Utility.GetFilesNotInFolders(changedFiles, simplifiedFolders);
// 4. The simplified file list still needs to be checked against the exclusion filter, as it
// may contain entries excluded due to attributes, but also because they are below excluded
// folders, which themselves aren't in the folder list from step 1.
// Note that the simplified file list may contain entries that have been deleted! They need to
// be kept in the list (unless excluded by the filter) in order for the backup handler to record their
// deletion.
result.Files.UnionWith(FilterExcludedFiles(simplifiedFiles, filter, cache));
}
}
catch (UsnJournalSoftFailureException)
@@ -138,6 +172,125 @@ namespace Duplicati.Library.Snapshots
return result;
}
/// <summary>
/// Filter supplied <c>files</c>, removing any files which itself, or one
/// of its parent folders, is excluded by the <c>filter</c>.
/// </summary>
/// <param name="files">Files to filter</param>
/// <param name="filter">Exclusion filter</param>
/// <param name="cache">Cache of excluded folders (optional)</param>
/// <returns>Filtered files</returns>
private IEnumerable<string> FilterExcludedFiles(IEnumerable<string> files,
Utility.Utility.EnumerationFilterDelegate filter, ISet<string> cache = null)
{
var result = new List<string>();
foreach (var file in files)
{
var attr = m_snapshot.FileExists(file) ? m_snapshot.GetAttributes(file) : FileAttributes.Normal;
try
{
if (!filter(file, file, attr))
continue;
if (!IsFolderOrAncestorsExcluded(Utility.Utility.GetParent(file, true), filter, cache))
{
result.Add(file);
}
}
catch (System.Threading.ThreadAbortException)
{
throw;
}
catch (Exception ex)
{
m_errorCallback?.Invoke(file, file, ex);
filter(file, file, attr | Utility.Utility.ATTRIBUTE_ERROR);
}
}
return result;
}
/// <summary>
/// Filter supplied <c>folders</c>, removing any folder which itself, or one
/// of its ancestors, is excluded by the <c>filter</c>.
/// </summary>
/// <param name="folders">Folder to filter</param>
/// <param name="filter">Exclusion filter</param>
/// <param name="cache">Cache of excluded folders (optional)</param>
/// <returns>Filtered folders</returns>
private IEnumerable<string> FilterExcludedFolders(IEnumerable<string> folders,
Utility.Utility.EnumerationFilterDelegate filter, ISet<string> cache = null)
{
var result = new List<string>();
foreach (var folder in folders)
{
try
{
if (!IsFolderOrAncestorsExcluded(folder, filter, cache))
{
result.Add(folder);
}
}
catch (System.Threading.ThreadAbortException)
{
throw;
}
catch (Exception ex)
{
m_errorCallback?.Invoke(folder, folder, ex);
filter(folder, folder, FileAttributes.Directory | Utility.Utility.ATTRIBUTE_ERROR);
}
}
return result;
}
/// <summary>
/// Tests if specified folder, or any of its ancestors, is excluded by the filter
/// </summary>
/// <param name="folder">Folder to test</param>
/// <param name="filter">Filter</param>
/// <param name="cache">Cache of excluded folders (optional)</param>
/// <returns>True if excluded, false otherwise</returns>
private bool IsFolderOrAncestorsExcluded(string folder, Utility.Utility.EnumerationFilterDelegate filter, ISet<string> cache = null)
{
List<string> parents = null;
while (folder != null)
{
// first check cache
if (cache != null)
{
if (cache.Contains(folder))
break; // hit!
// remember folder for cache
if (parents == null)
{
parents = new List<string>(); // create on-demand
}
parents.Add(folder);
}
var attr = m_snapshot.FolderExists(folder) ? m_snapshot.GetAttributes(folder) : FileAttributes.Directory;
if (!filter(folder, folder, attr))
break; // excluded
folder = Utility.Utility.GetParent(folder, true);
}
if (folder != null && parents != null)
{
// update cache
cache.UnionWith(parents);
}
return folder != null;
}
/// <summary>
/// Add ALL sources for volume to result set
/// </summary>
Duplicati/Library/Utility/TempFile.cs
+1 -1
View File
@@ -31,7 +31,7 @@ namespace Duplicati.Library.Utility
/// <summary>
/// The prefix applied to all temporary files
/// </summary>
public static string APPLICATION_PREFIX = Utility.getEntryAssembly().FullName.Substring(0, 3).ToLower() + "-";
public static string APPLICATION_PREFIX = Utility.GetEntryAssembly().FullName.Substring(0, 3).ToLower() + "-";
private string m_path;
private bool m_protect;
Duplicati/Library/Utility/Utility.cs
+106 -157
View File
@@ -38,12 +38,12 @@ namespace Duplicati.Library.Utility
/// <summary>
/// A cache of the FileSystemCaseSensitive property, which is computed upon the first access.
/// </summary>
private static bool? CachedIsFSCaseSensitive = null;
private static bool? CachedIsFSCaseSensitive;
/// <summary>
/// Gets the hash algorithm used for calculating a hash
/// </summary>
public static string HashAlgorithm { get { return "SHA256"; } }
public static string HashAlgorithm => "SHA256";
/// <summary>
/// The EPOCH offset (unix style)
@@ -53,7 +53,7 @@ namespace Duplicati.Library.Utility
/// <summary>
/// The attribute value used to indicate error
/// </summary>
public const System.IO.FileAttributes ATTRIBUTE_ERROR = (System.IO.FileAttributes)(1 << 30);
public const FileAttributes ATTRIBUTE_ERROR = (FileAttributes)(1 << 30);
/// <summary>
/// The callback delegate type used to collecting file information
@@ -62,14 +62,14 @@ namespace Duplicati.Library.Utility
/// <param name="path">The current element</param>
/// <param name="attributes">The attributes of the element</param>
/// <returns>A value indicating if the folder should be recursed, ignored for other types</returns>
public delegate bool EnumerationFilterDelegate(string rootpath, string path, System.IO.FileAttributes attributes);
public delegate bool EnumerationFilterDelegate(string rootpath, string path, FileAttributes attributes);
/// <summary>
/// Copies the content of one stream into another
/// </summary>
/// <param name="source">The stream to read from</param>
/// <param name="target">The stream to write to</param>
public static void CopyStream(System.IO.Stream source, System.IO.Stream target)
public static void CopyStream(Stream source, Stream target)
{
CopyStream(source, target, true);
}
@@ -80,11 +80,15 @@ namespace Duplicati.Library.Utility
/// <param name="source">The stream to read from</param>
/// <param name="target">The stream to write to</param>
/// <param name="tryRewindSource">True if an attempt should be made to rewind the source stream, false otherwise</param>
public static void CopyStream(System.IO.Stream source, System.IO.Stream target, bool tryRewindSource, byte[] buf = null)
/// <param name="buf">Temporary buffer to use (optional)</param>
public static void CopyStream(Stream source, Stream target, bool tryRewindSource, byte[] buf = null)
{
if (tryRewindSource && source.CanSeek)
try { source.Position = 0; }
catch { }
catch
{
// ignored
}
buf = buf ?? new byte[DEFAULT_BUFFER_SIZE];
@@ -96,21 +100,7 @@ namespace Duplicati.Library.Utility
/// <summary>
/// These are characters that must be escaped when using a globbing expression
/// </summary>
private static readonly string BADCHARS = "\\" + string.Join("|\\", new string[] {
"\\",
"+",
"|",
"{",
"[",
"(",
")",
"]",
"}",
"^",
"$",
"#",
"."
});
private static readonly string BADCHARS = @"\\|\+|\||\{|\[|\(|\)|\]|\}|\^|\$|\#|\.";
/// <summary>
/// Most people will probably want to use fileglobbing, but RegExp's are more flexible.
@@ -121,7 +111,7 @@ namespace Duplicati.Library.Utility
public static string ConvertGlobbingToRegExp(string globexp)
{
//First escape all special characters
globexp = Regex.Replace(globexp, BADCHARS, "\\$&");
globexp = Regex.Replace(globexp, BADCHARS, @"\$&");
//Replace the globbing expressions with the corresponding regular expressions
globexp = globexp.Replace('?', '.').Replace("*", ".*");
@@ -194,11 +184,9 @@ namespace Duplicati.Library.Utility
/// <returns>A list of the full filenames and foldernames. Foldernames ends with the directoryseparator char</returns>
public static IEnumerable<string> EnumerateFileSystemEntries(string basepath, IFilter filter)
{
IFilter match;
filter = filter ?? new FilterExpression();
return EnumerateFileSystemEntries(basepath, (rootpath, path, attributes) => {
bool result;
if (!filter.Matches(path, out result, out match))
if (!filter.Matches(path, out var result, out _))
result = true;
return result;
@@ -217,7 +205,7 @@ namespace Duplicati.Library.Utility
/// </summary>
/// <param name="path">The path to return data from</param>
/// <returns>Attributes for the file or folder</returns>
public delegate System.IO.FileAttributes ExtractFileAttributes(string path);
public delegate FileAttributes ExtractFileAttributes(string path);
/// <summary>
/// A callback delegate used for extracting attributes from a file or folder
@@ -236,7 +224,7 @@ namespace Duplicati.Library.Utility
/// <returns>A list of the full filenames</returns>
public static IEnumerable<string> EnumerateFileSystemEntries(string rootpath, EnumerationFilterDelegate callback)
{
return EnumerateFileSystemEntries(rootpath, callback, new FileSystemInteraction(System.IO.Directory.GetDirectories), new FileSystemInteraction(System.IO.Directory.GetFiles));
return EnumerateFileSystemEntries(rootpath, callback, Directory.GetDirectories, Directory.GetFiles);
}
/// <summary>
@@ -250,7 +238,7 @@ namespace Duplicati.Library.Utility
/// <returns>A list of the full filenames</returns>
public static IEnumerable<string> EnumerateFileSystemEntries(string rootpath, EnumerationFilterDelegate callback, FileSystemInteraction folderList, FileSystemInteraction fileList)
{
return EnumerateFileSystemEntries(rootpath, callback, folderList, fileList, null, null);
return EnumerateFileSystemEntries(rootpath, callback, folderList, fileList, null);
}
/// <summary>
@@ -274,7 +262,7 @@ namespace Duplicati.Library.Utility
try
{
System.IO.FileAttributes attr = attributeReader == null ? System.IO.FileAttributes.Directory : attributeReader(rootpath);
FileAttributes attr = attributeReader == null ? FileAttributes.Directory : attributeReader(rootpath);
if (callback(rootpath, rootpath, attr))
lst.Push(rootpath);
}
@@ -286,7 +274,7 @@ namespace Duplicati.Library.Utility
{
if (errorCallback != null)
errorCallback(rootpath, rootpath, ex);
callback(rootpath, rootpath, System.IO.FileAttributes.Directory | ATTRIBUTE_ERROR);
callback(rootpath, rootpath, FileAttributes.Directory | ATTRIBUTE_ERROR);
}
while (lst.Count > 0)
@@ -302,7 +290,7 @@ namespace Duplicati.Library.Utility
var sf = AppendDirSeparator(s);
try
{
System.IO.FileAttributes attr = attributeReader == null ? System.IO.FileAttributes.Directory : attributeReader(sf);
FileAttributes attr = attributeReader == null ? FileAttributes.Directory : attributeReader(sf);
if (callback(rootpath, sf, attr))
lst.Push(sf);
}
@@ -314,7 +302,7 @@ namespace Duplicati.Library.Utility
{
if (errorCallback != null)
errorCallback(rootpath, sf, ex);
callback(rootpath, sf, System.IO.FileAttributes.Directory | ATTRIBUTE_ERROR);
callback(rootpath, sf, FileAttributes.Directory | ATTRIBUTE_ERROR);
}
}
}
@@ -326,7 +314,7 @@ namespace Duplicati.Library.Utility
{
if (errorCallback != null)
errorCallback(rootpath, f, ex);
callback(rootpath, f, System.IO.FileAttributes.Directory | ATTRIBUTE_ERROR);
callback(rootpath, f, FileAttributes.Directory | ATTRIBUTE_ERROR);
}
string[] files = null;
@@ -343,7 +331,7 @@ namespace Duplicati.Library.Utility
{
if (errorCallback != null)
errorCallback(rootpath, f, ex);
callback(rootpath, f, System.IO.FileAttributes.Directory | ATTRIBUTE_ERROR);
callback(rootpath, f, FileAttributes.Directory | ATTRIBUTE_ERROR);
}
if (files != null)
@@ -351,7 +339,7 @@ namespace Duplicati.Library.Utility
{
try
{
System.IO.FileAttributes attr = attributeReader == null ? System.IO.FileAttributes.Normal : attributeReader(s);
FileAttributes attr = attributeReader == null ? FileAttributes.Normal : attributeReader(s);
if (!callback(rootpath, s, attr))
continue;
}
@@ -374,7 +362,7 @@ namespace Duplicati.Library.Utility
{
try
{
System.IO.FileAttributes attr = attributeReader == null ? System.IO.FileAttributes.Normal : attributeReader(rootpath);
FileAttributes attr = attributeReader == null ? FileAttributes.Normal : attributeReader(rootpath);
if (!callback(rootpath, rootpath, attr))
yield break;
}
@@ -407,7 +395,7 @@ namespace Duplicati.Library.Utility
try
{
return attributeReader(path).HasFlag(System.IO.FileAttributes.Directory);
return attributeReader(path).HasFlag(FileAttributes.Directory);
}
catch
{
@@ -416,25 +404,16 @@ namespace Duplicati.Library.Utility
}
/// <summary>
/// Tests if path refers to a file, or folder, below the parent folder
/// Tests if path refers to a file, or folder, <b>below</b> the parent folder
/// </summary>
/// <param name="fileOrFolderPath">File or folder to test</param>
/// <param name="parentFolder">Candidate parent folder</param>
/// <returns>True if below parent folder, false otherwise (note that this returns false if the two argument paths are identical!)</returns>
public static bool IsPathBelowFolder(string fileOrFolderPath, string parentFolder)
{
var candidatePath = fileOrFolderPath;
var sanitizedParentFolder = AppendDirSeparator(parentFolder);
while (true)
{
candidatePath = GetParent(candidatePath, true);
if (candidatePath == null)
return false;
if (candidatePath.Equals(sanitizedParentFolder, ClientFilenameStringComparision))
return true;
}
return fileOrFolderPath.StartsWith(sanitizedParentFolder, ClientFilenameStringComparision) &&
!fileOrFolderPath.Equals(sanitizedParentFolder, ClientFilenameStringComparision);
}
/// <summary>
@@ -535,13 +514,13 @@ namespace Duplicati.Library.Utility
/// <returns>The combined size of all files that match the filter</returns>
public static long GetDirectorySize(string folder, IFilter filter)
{
return EnumerateFolders(folder, filter).Sum((path) => new System.IO.FileInfo(path).Length);
return EnumerateFolders(folder, filter).Sum((path) => new FileInfo(path).Length);
}
/// <summary>
/// A cached instance of the directory separator as a string
/// </summary>
public static readonly string DirectorySeparatorString = System.IO.Path.DirectorySeparatorChar.ToString();
public static readonly string DirectorySeparatorString = Path.DirectorySeparatorChar.ToString();
/// <summary>
/// Appends the appropriate directory separator to paths, depending on OS.
@@ -551,10 +530,9 @@ namespace Duplicati.Library.Utility
/// <returns>The path with the directory separator appended</returns>
public static string AppendDirSeparator(string path)
{
if (!path.EndsWith(DirectorySeparatorString, StringComparison.Ordinal))
return path += DirectorySeparatorString;
else
return path;
return !path.EndsWith(DirectorySeparatorString, StringComparison.Ordinal)
? path + DirectorySeparatorString
: path;
}
/// <summary>
@@ -566,10 +544,7 @@ namespace Duplicati.Library.Utility
/// <returns>The path with the directory separator appended</returns>
public static string AppendDirSeparator(string path, string separator)
{
if (!path.EndsWith(separator, StringComparison.Ordinal))
return path += separator;
else
return path;
return AppendDirSeparator(separator);
}
/// <summary>
@@ -591,7 +566,7 @@ namespace Duplicati.Library.Utility
/// <param name="buf">The buffer to read into</param>
/// <param name="count">The amout of bytes to read</param>
/// <returns>The actual number of bytes read</returns>
public static int ForceStreamRead(System.IO.Stream stream, byte[] buf, int count)
public static int ForceStreamRead(Stream stream, byte[] buf, int count)
{
int a;
int index = 0;
@@ -612,7 +587,7 @@ namespace Duplicati.Library.Utility
/// <param name="stream2">Another stream</param>
/// <param name="checkLength">True if the length of the two streams should be compared</param>
/// <returns>True if they are equal, false otherwise</returns>
public static bool CompareStreams(System.IO.Stream stream1, System.IO.Stream stream2, bool checkLength)
public static bool CompareStreams(Stream stream1, Stream stream2, bool checkLength)
{
if (checkLength)
{
@@ -661,7 +636,7 @@ namespace Duplicati.Library.Utility
/// <returns>The base64 encoded hash</returns>
public static string CalculateHash(string path)
{
using (System.IO.FileStream fs = System.IO.File.Open(path, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read))
using (FileStream fs = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read))
return CalculateHash(fs);
}
@@ -670,7 +645,7 @@ namespace Duplicati.Library.Utility
/// </summary>
/// <param name="stream">The stream to calculate the hash for</param>
/// <returns>The base64 encoded hash</returns>
public static string CalculateHash(System.IO.Stream stream)
public static string CalculateHash(Stream stream)
{
return Convert.ToBase64String(HashAlgorithmHelper.Create(HashAlgorithm).ComputeHash(stream));
}
@@ -685,14 +660,15 @@ namespace Duplicati.Library.Utility
// Since StreamReader defaults to UTF8 and most text files will NOT be UTF8 without BOM,
// we need to detect the encoding (at least that it's not UTF8).
// So we read the first 4096 bytes and try to decode them as UTF8.
byte[] buffer = new byte[4096];
using (System.IO.FileStream file = new System.IO.FileStream(filename, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read))
var buffer = new byte[4096];
using (var file = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))
file.Read(buffer, 0, 4096);
Encoding enc = Encoding.UTF8;
var enc = Encoding.UTF8;
try
{
// this will throw an error if not really UTF8
// ReSharper disable once ReturnValueOfPureMethodIsNotUsed
new UTF8Encoding(false, true).GetString(buffer);
}
catch (Exception)
@@ -701,7 +677,7 @@ namespace Duplicati.Library.Utility
}
// This will load the text using the BOM, or the detected encoding if no BOM.
using (System.IO.StreamReader reader = new System.IO.StreamReader(filename, enc, true))
using (var reader = new StreamReader(filename, enc, true))
{
// Remove all \r from the file and split on \n, then pass directly to ExtractOptions
return reader.ReadToEnd();
@@ -709,7 +685,7 @@ namespace Duplicati.Library.Utility
}
/// <summary>
/// Formats a size into a human readable format, eg. 2048 becomes &quot;2 KB&quot; or -2283 becomes &qout;-2.23 KB%quot.
/// Formats a size into a human readable format, eg. 2048 becomes &quot;2 KB&quot; or -2283 becomes &quot;-2.23 KB%quot.
/// </summary>
/// <param name="size">The size to format</param>
/// <returns>A human readable string representing the size</returns>
@@ -797,7 +773,7 @@ namespace Duplicati.Library.Utility
/// <returns>The parsed value, or the default value.</returns>
public static bool ParseBool(string value, bool @default)
{
return Utility.ParseBool(value, () => @default);
return ParseBool(value, () => @default);
}
/// <summary>
@@ -825,12 +801,7 @@ namespace Duplicati.Library.Utility
/// <typeparam name="T">The enum type parameter.</typeparam>
public static T ParseEnumOption<T>(IDictionary<string, string> options, string value, T @default)
{
string opt;
if (options.TryGetValue(value, out opt))
return ParseEnum<T>(opt, @default);
else
return @default;
return options.TryGetValue(value, out var opt) ? ParseEnum(opt, @default) : @default;
}
/// <summary>
@@ -842,7 +813,7 @@ namespace Duplicati.Library.Utility
/// <typeparam name="T">The enum type parameter.</typeparam>
public static T ParseEnum<T>(string value, T @default)
{
foreach (string s in Enum.GetNames(typeof(T)))
foreach (var s in Enum.GetNames(typeof(T)))
if (s.Equals(value, StringComparison.OrdinalIgnoreCase))
return (T)Enum.Parse(typeof(T), s);
@@ -918,9 +889,11 @@ namespace Duplicati.Library.Utility
{
if (UNAME == null)
{
var psi = new System.Diagnostics.ProcessStartInfo("uname");
psi.RedirectStandardOutput = true;
psi.UseShellExecute = false;
var psi = new System.Diagnostics.ProcessStartInfo("uname")
{
RedirectStandardOutput = true,
UseShellExecute = false
};
var pi = System.Diagnostics.Process.Start(psi);
pi.WaitForExit(5000);
@@ -959,6 +932,7 @@ namespace Duplicati.Library.Utility
}
catch
{
// ignored
}
return null;
@@ -967,24 +941,12 @@ namespace Duplicati.Library.Utility
/// <value>
/// Gets or sets a value indicating if the client is Linux/Unix based
/// </value>
public static bool IsClientLinux
{
get
{
return Environment.OSVersion.Platform == PlatformID.Unix || Environment.OSVersion.Platform == PlatformID.MacOSX;
}
}
public static bool IsClientLinux => Environment.OSVersion.Platform == PlatformID.Unix || Environment.OSVersion.Platform == PlatformID.MacOSX;
/// <summary>
/// Gets a value indicating if the client is Windows based
/// </summary>
public static bool IsClientWindows
{
get
{
return !IsClientLinux;
}
}
public static bool IsClientWindows => !IsClientLinux;
/// <value>
/// Returns a value indicating if the filesystem, is case sensitive
@@ -999,10 +961,7 @@ namespace Duplicati.Library.Utility
// TODO: This should probably be determined by filesystem rather than OS,
// OSX can actually have the disks formated as Case Sensitive, but insensitive is default
Func<bool> defaultReply = () => Utility.IsClientLinux && !Utility.IsClientOSX;
CachedIsFSCaseSensitive = Utility.ParseBool(str, defaultReply);
CachedIsFSCaseSensitive = ParseBool(str, () => IsClientLinux && !IsClientOSX);
}
return CachedIsFSCaseSensitive.Value;
@@ -1012,13 +971,7 @@ namespace Duplicati.Library.Utility
/// <summary>
/// Returns a value indicating if the app is running under Mono
/// </summary>
public static bool IsMono
{
get
{
return Type.GetType("Mono.Runtime") != null;
}
}
public static bool IsMono => Type.GetType("Mono.Runtime") != null;
/// <summary>
/// Gets the current Mono runtime version, will return 0.0 if not running Mono
@@ -1032,7 +985,7 @@ namespace Duplicati.Library.Utility
var v = MonoDisplayVersion;
if (v != null)
{
var regex = new System.Text.RegularExpressions.Regex(@"\d+\.\d+(\.\d+)?(\.\d+)?");
var regex = new Regex(@"\d+\.\d+(\.\d+)?(\.\d+)?");
var match = regex.Match(v);
if (match.Success)
return new Version(match.Value);
@@ -1040,6 +993,7 @@ namespace Duplicati.Library.Utility
}
catch
{
// ignored
}
return new Version();
@@ -1055,16 +1009,17 @@ namespace Duplicati.Library.Utility
{
try
{
Type t = Type.GetType("Mono.Runtime");
var t = Type.GetType("Mono.Runtime");
if (t != null)
{
System.Reflection.MethodInfo mi = t.GetMethod("GetDisplayName", System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.NonPublic);
var mi = t.GetMethod("GetDisplayName", System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.NonPublic);
if (mi != null)
return (string)mi.Invoke(null, null);
}
}
catch
{
// ignored
}
return null;
@@ -1074,28 +1029,17 @@ namespace Duplicati.Library.Utility
/// <summary>
/// Gets the users default UI language
/// </summary>
public static System.Globalization.CultureInfo DefaultCulture
{
get
{
System.Threading.Thread t = new System.Threading.Thread(new System.Threading.ThreadStart(DummyMethod));
return t.CurrentUICulture;
}
}
//Unused function, used to create a dummy thread
private static void DummyMethod()
{
}
public static System.Globalization.CultureInfo DefaultCulture => new System.Threading.Thread(() => { }).CurrentUICulture;
/// <summary>
/// Gets a string comparer that matches the client filesystems case sensitivity
/// </summary>
public static StringComparer ClientFilenameStringComparer { get { return Utility.IsFSCaseSensitive ? StringComparer.CurrentCulture : StringComparer.CurrentCultureIgnoreCase; } }
public static StringComparer ClientFilenameStringComparer => IsFSCaseSensitive ? StringComparer.CurrentCulture : StringComparer.CurrentCultureIgnoreCase;
/// <summary>
/// Gets the string comparision that matches the client filesystems case sensitivity
/// </summary>
public static StringComparison ClientFilenameStringComparision { get { return Utility.IsFSCaseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase; } }
public static StringComparison ClientFilenameStringComparision => IsFSCaseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase;
/// <summary>
/// Searches the system paths for the file specified
@@ -1106,33 +1050,36 @@ namespace Duplicati.Library.Utility
{
try
{
if (System.IO.Path.IsPathRooted(filename))
return System.IO.File.Exists(filename) ? filename : null;
if (Path.IsPathRooted(filename))
return File.Exists(filename) ? filename : null;
try
{
filename = System.IO.Path.GetFileName(filename);
filename = Path.GetFileName(filename);
}
catch
{
// ignored
}
string homedir = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) + System.IO.Path.PathSeparator.ToString();
string homedir = Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) + Path.PathSeparator.ToString();
//Look in application base folder and all system path folders
foreach (string s in (homedir + Environment.GetEnvironmentVariable("PATH")).Split(System.IO.Path.PathSeparator))
foreach (string s in (homedir + Environment.GetEnvironmentVariable("PATH")).Split(Path.PathSeparator))
if (!string.IsNullOrEmpty(s) && s.Trim().Length > 0)
try
{
foreach (string sx in System.IO.Directory.GetFiles(ExpandEnvironmentVariables(s), filename))
foreach (string sx in Directory.GetFiles(ExpandEnvironmentVariables(s), filename))
return sx;
}
catch
{
// ignored
}
}
catch
{
// ignored
}
return null;
@@ -1172,14 +1119,14 @@ namespace Duplicati.Library.Utility
public static string ExpandEnvironmentVariablesRegexp(string str, Func<string, string> lookup = null)
{
if (lookup == null)
lookup = x => Environment.GetEnvironmentVariable(x);
lookup = Environment.GetEnvironmentVariable;
return
// TODO: Should we switch to using the native format, instead of following the Windows scheme?
//IsClientLinux ? ENVIRONMENT_VARIABLE_MATCHER_LINUX : ENVIRONMENT_VARIABLE_MATCHER_WINDOWS
ENVIRONMENT_VARIABLE_MATCHER_WINDOWS.Replace(str, (m) => Regex.Escape(lookup(m.Groups["name"].Value)));
ENVIRONMENT_VARIABLE_MATCHER_WINDOWS.Replace(str, m => Regex.Escape(lookup(m.Groups["name"].Value)));
}
/// <summary>
@@ -1232,8 +1179,7 @@ namespace Duplicati.Library.Utility
/// <returns>The parsed <see cref="System.DateTime"/> instance</returns>
public static DateTime DeserializeDateTime(string str)
{
DateTime dt;
if (!TryDeserializeDateTime(str, out dt))
if (!TryDeserializeDateTime(str, out var dt))
throw new Exception(Strings.Utility.InvalidDateError(str));
return dt;
@@ -1248,7 +1194,7 @@ namespace Duplicati.Library.Utility
/// <returns>The unique items from <paramref name="collection"/>.</returns>
public static ISet<T> GetUniqueItems<T>(IEnumerable<T> collection, out ISet<T> duplicateItems)
{
return Utility.GetUniqueItems(collection, EqualityComparer<T>.Default, out duplicateItems);
return GetUniqueItems(collection, EqualityComparer<T>.Default, out duplicateItems);
}
/// <summary>
@@ -1261,12 +1207,14 @@ namespace Duplicati.Library.Utility
/// <returns>The unique items from <paramref name="collection"/>.</returns>
public static ISet<T> GetUniqueItems<T>(IEnumerable<T> collection, IEqualityComparer<T> comparer, out ISet<T> duplicateItems)
{
HashSet<T> uniqueItems = new HashSet<T>(comparer);
var uniqueItems = new HashSet<T>(comparer);
duplicateItems = new HashSet<T>(comparer);
foreach (T item in collection)
foreach (var item in collection)
{
if (!uniqueItems.Add(item))
duplicateItems.Add(item);
}
return uniqueItems;
}
@@ -1278,8 +1226,8 @@ namespace Duplicati.Library.Utility
/// <param name="sourcefile">The file to replace with</param>
public static void ReplaceFile(string target, string sourcefile)
{
if (System.IO.File.Exists(target))
System.IO.File.Delete(target);
if (File.Exists(target))
File.Delete(target);
//Nasty workaround for the fact that a recently deleted file occasionally blocks a new write
long i = 5;
@@ -1287,13 +1235,14 @@ namespace Duplicati.Library.Utility
{
try
{
System.IO.File.Move(sourcefile, target);
File.Move(sourcefile, target);
break;
}
catch (Exception ex)
{
if (i == 0)
throw new Exception(string.Format("Failed to replace the file \"{0}\" volume with the \"{1}\", error: {2}", target, sourcefile, ex.Message));
throw new Exception(
$"Failed to replace the file \"{target}\" volume with the \"{sourcefile}\", error: {ex.Message}");
System.Threading.Thread.Sleep(250);
}
} while (i-- > 0);
@@ -1305,7 +1254,7 @@ namespace Duplicati.Library.Utility
// so this layer of indirection is necessary
// </summary>
// <returns>entry assembly or reasonable approximation</returns>
public static System.Reflection.Assembly getEntryAssembly()
public static System.Reflection.Assembly GetEntryAssembly()
{
return System.Reflection.Assembly.GetEntryAssembly() ?? System.Reflection.Assembly.GetExecutingAssembly();
}
@@ -1394,7 +1343,7 @@ namespace Duplicati.Library.Utility
/// <returns><c>true</c>, the item was printed, <c>false</c> otherwise.</returns>
/// <param name="item">The item to write.</param>
/// <param name="writer">The target writer.</param>
private static bool PrintSerializeIfPrimitive(object item, System.IO.TextWriter writer)
private static bool PrintSerializeIfPrimitive(object item, TextWriter writer)
{
if (item == null)
{
@@ -1430,7 +1379,7 @@ namespace Duplicati.Library.Utility
/// <param name="indentation">The string indentation</param>
/// <param name="visited">A lookup table with visited objects, used to avoid inifinite recursion</param>
/// <param name="collectionlimit">The maximum number of items to report from an IEnumerable instance</param>
public static void PrintSerializeObject(object item, System.IO.TextWriter writer, Func<System.Reflection.PropertyInfo, object, bool> filter = null, bool recurseobjects = false, int indentation = 0, int collectionlimit = 0, Dictionary<object, object> visited = null)
public static void PrintSerializeObject(object item, TextWriter writer, Func<System.Reflection.PropertyInfo, object, bool> filter = null, bool recurseobjects = false, int indentation = 0, int collectionlimit = 0, Dictionary<object, object> visited = null)
{
visited = visited ?? new Dictionary<object, object>();
var indentstring = new string(' ', indentation);
@@ -1549,7 +1498,7 @@ namespace Duplicati.Library.Utility
public static StringBuilder PrintSerializeObject(object item, StringBuilder sb = null, Func<System.Reflection.PropertyInfo, object, bool> filter = null, bool recurseobjects = false, int indentation = 0, int collectionlimit = 10)
{
sb = sb ?? new StringBuilder();
using (var sw = new System.IO.StringWriter(sb))
using (var sw = new StringWriter(sb))
PrintSerializeObject(item, sw, filter, recurseobjects, indentation, collectionlimit);
return sb;
}
@@ -1566,8 +1515,8 @@ namespace Duplicati.Library.Utility
public static byte[] RepeatedHashWithSalt(string data, string salt, int repeats = 1200)
{
return RepeatedHashWithSalt(
System.Text.Encoding.UTF8.GetBytes(data ?? ""),
System.Text.Encoding.UTF8.GetBytes(salt ?? ""),
Encoding.UTF8.GetBytes(data ?? ""),
Encoding.UTF8.GetBytes(salt ?? ""),
repeats);
}
@@ -1645,16 +1594,16 @@ namespace Duplicati.Library.Utility
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)]
public static IEnumerable<KeyValuePair<string, string>> GetVolumeGuidsAndDriveLetters()
{
using (System.Management.ManagementObjectSearcher searcher = new System.Management.ManagementObjectSearcher("Select * from Win32_Volume"))
using (var searcher = new System.Management.ManagementObjectSearcher("Select * from Win32_Volume"))
{
foreach (System.Management.ManagementObject obj in searcher.Get())
foreach (var obj in searcher.Get())
{
object deviceIdObj = obj["DeviceID"];
object driveLetterObj = obj["DriveLetter"];
var deviceIdObj = obj["DeviceID"];
var driveLetterObj = obj["DriveLetter"];
if (deviceIdObj != null && driveLetterObj != null)
{
string deviceId = deviceIdObj.ToString();
string driveLetter = driveLetterObj.ToString();
var deviceId = deviceIdObj.ToString();
var driveLetter = driveLetterObj.ToString();
if (!string.IsNullOrEmpty(deviceId) && !string.IsNullOrEmpty(driveLetter))
{
yield return new KeyValuePair<string, string>(driveLetter + @"\", deviceId);
@@ -1684,7 +1633,7 @@ namespace Duplicati.Library.Utility
if (string.IsNullOrWhiteSpace(arg))
return arg;
if (!Library.Utility.Utility.IsClientWindows)
if (!IsClientWindows)
{
// We could consider using single quotes that prevents all expansions
//if (!allowEnvExpansion)
@@ -1694,12 +1643,12 @@ namespace Duplicati.Library.Utility
arg = COMMANDLINE_ESCAPED_LINUX.Replace(arg, (match) =>
{
if (match.Value == "!")
return "\"'!'\"";
return @"""'!'""";
if (match.Value == "$" && allowEnvExpansion)
return match.Value;
return "\\" + match.Value;
return @"\" + match.Value;
});
}
else
@@ -1714,16 +1663,16 @@ namespace Duplicati.Library.Utility
// and sadly it expands only if the variable exists
// making it even rarer and harder to diagnose when
// it happens
arg = arg.Replace("\"", "\"\"");
arg = arg.Replace(@"""", @"""""");
// Also fix the case where the argument ends with a slash
if (arg[arg.Length - 1] == '\\')
arg += "\\";
arg += @"\";
}
// Check that all characters are in the safe set
if (COMMANDLINE_SAFE.Match(arg).Length != arg.Length)
return "\"" + arg + "\"";
return @"""" + arg + @"""";
else
return arg;
}
Duplicati/Server/WebServer/RESTMethods/Licenses.cs
+1 -1
View File
@@ -22,7 +22,7 @@ namespace Duplicati.Server.WebServer.RESTMethods
{
public void GET(string key, RequestInfo info)
{
var path = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(Duplicati.Library.Utility.Utility.getEntryAssembly().Location), "licenses");
var path = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(Duplicati.Library.Utility.Utility.GetEntryAssembly().Location), "licenses");
info.OutputOK(Duplicati.License.LicenseReader.ReadLicenses(path));
}
}