From cc053ff4b0274b2fa77f90d2bd530ce636ccee43 Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Tue, 6 Feb 2018 12:28:54 -0700 Subject: [PATCH 1/2] Normalize whitespace --- Duplicati/Library/Main/Operation/BackupHandler.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 1bf05ab9b..0123c2b82 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -825,12 +825,12 @@ namespace Duplicati.Library.Main.Operation backend.WaitForEmpty(m_database, m_transaction); if (m_result.TaskControlRendevouz() != TaskControlState.Stop) - CompactIfRequired(backend, lastVolumeSize); - - using (new Logging.Timer("Async backend wait")) - backend.WaitForComplete(m_database, m_transaction); - - if (m_options.UploadVerificationFile) + CompactIfRequired(backend, lastVolumeSize); + + using (new Logging.Timer("Async backend wait")) + backend.WaitForComplete(m_database, m_transaction); + + if (m_options.UploadVerificationFile) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload); FilelistProcessor.UploadVerificationFile(backend.BackendUrl, m_options, m_result.BackendWriter, m_database, m_transaction); From 08e0708a9534464a7238d61b242acbffa189257d Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Tue, 6 Feb 2018 13:24:49 -0700 Subject: [PATCH 2/2] Change Duplicati's handling of reparse points, so that not all reparse points are treated as symlinks. This is inspired largely by the new (/ returning) feature of OneDrive in Windows 10 Fall Creator's Update, which downloads files on-demand. A side effect of that change is that the OneDrive folder (and subfolders of it) are marked as reparse points, even though they are not technically standard symlinks. With this change, a new extension method IsSymlink is added for ISnapshotService and ISystemIO, which checks both the file attributes (for reparse point) and the symlink target path (if null, the path is not treated as a symlink). All places that previously checked only the file attributes have been updated to use either this method (or at least the same logic, in the case of the core BackupHandler file check). One side effect of this change is that '--symlink-policy=store' no longer ignores empty symlinks - they are now treated as regular files. If there are empty symlinks, they will now be backed up as if they were regular files, but I don't know what the conditions are that create symlinks like that, so this might not effect anything in practice. --- .../Library/Main/Operation/BackupHandler.cs | 60 +++++++++++-------- .../Main/Operation/TestFilterHandler.cs | 3 +- Duplicati/Library/Snapshots/ISystemIO.cs | 1 + Duplicati/Library/Snapshots/LinuxSnapshot.cs | 2 +- .../Library/Snapshots/NoSnapshotLinux.cs | 2 +- .../Library/Snapshots/NoSnapshotWindows.cs | 6 +- .../Library/Snapshots/SnapshotUtility.cs | 55 +++++++++++++++++ Duplicati/Library/Snapshots/SystemIOLinux.cs | 5 ++ .../Library/Snapshots/SystemIOWindows.cs | 28 +++++++++ .../Library/Snapshots/WindowsSnapshot.cs | 30 ++++------ .../WebServer/RESTMethods/Filesystem.cs | 3 +- 11 files changed, 143 insertions(+), 52 deletions(-) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 0123c2b82..78c02a64e 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -6,6 +6,7 @@ using System.IO; using Duplicati.Library.Main.Database; using Duplicati.Library.Main.Volumes; using Duplicati.Library.Interface; +using Duplicati.Library.Snapshots; namespace Duplicati.Library.Main.Operation { @@ -187,7 +188,7 @@ namespace Duplicati.Library.Main.Operation m_logWriter.AddVerboseMessage("Including path due to filter: {0} => {1}", path, match.ToString()); } - var isSymlink = (attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint; + var isSymlink = m_snapshot.IsSymlink(path, attributes); if (isSymlink && m_symlinkPolicy == Options.SymlinkStrategy.Ignore) { if (m_logWriter != null) @@ -281,8 +282,8 @@ namespace Duplicati.Library.Main.Operation var fa = FileAttributes.Normal; try { fa = snapshot.GetAttributes(path); } catch { } - - if (followSymlinks && ((fa & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint)) + + if (followSymlinks && snapshot.IsSymlink(path, fa)) continue; else if ((fa & FileAttributes.Directory) == FileAttributes.Directory) continue; @@ -970,32 +971,39 @@ namespace Duplicati.Library.Main.Operation if ((attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint) { - if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Ignore) + // Not all reparse points are symlinks. + // For example, on Windows 10 Fall Creator's Update, the OneDrive folder (and all subfolders) + // are reparse points, which allows the folder to hook into the OneDrive service and download things on-demand. + // If we can't find a symlink target for the current path, we won't treat it as a symlink. + string symlinkTarget = snapshot.GetSymlinkTarget(path); + if (!string.IsNullOrWhiteSpace(symlinkTarget)) { - m_result.AddVerboseMessage("Ignoring symlink {0}", path); - return false; - } - - if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Store) - { - Dictionary metadata = GenerateMetadata(snapshot, path, attributes); - - if (!metadata.ContainsKey("CoreSymlinkTarget")) + if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Ignore) { - var p = snapshot.GetSymlinkTarget(path); - - if (string.IsNullOrWhiteSpace(p)) - m_result.AddVerboseMessage("Ignoring empty symlink {0}", path); - else - metadata["CoreSymlinkTarget"] = p; + m_result.AddVerboseMessage("Ignoring symlink {0}", path); + return false; } - - var metahash = Utility.WrapMetadata(metadata, m_options); - AddSymlinkToOutput(backend, path, DateTime.UtcNow, metahash); - - m_result.AddVerboseMessage("Stored symlink {0}", path); - //Do not recurse symlinks - return false; + + if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Store) + { + Dictionary metadata = GenerateMetadata(snapshot, path, attributes); + + if (!metadata.ContainsKey("CoreSymlinkTarget")) + { + metadata["CoreSymlinkTarget"] = symlinkTarget; + } + + var metahash = Utility.WrapMetadata(metadata, m_options); + AddSymlinkToOutput(backend, path, DateTime.UtcNow, metahash); + + m_result.AddVerboseMessage("Stored symlink {0}", path); + //Do not recurse symlinks + return false; + } + } + else + { + m_result.AddVerboseMessage("Treating empty symlink as regular path {0}", path); } } diff --git a/Duplicati/Library/Main/Operation/TestFilterHandler.cs b/Duplicati/Library/Main/Operation/TestFilterHandler.cs index 366a2ff4b..7b4b4775b 100644 --- a/Duplicati/Library/Main/Operation/TestFilterHandler.cs +++ b/Duplicati/Library/Main/Operation/TestFilterHandler.cs @@ -17,6 +17,7 @@ // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA using System; using System.IO; +using Duplicati.Library.Snapshots; namespace Duplicati.Library.Main.Operation { @@ -44,7 +45,7 @@ namespace Duplicati.Library.Main.Operation try { fa = snapshot.GetAttributes(path); } catch { } - if (storeSymlinks && ((fa & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint)) + if (storeSymlinks && snapshot.IsSymlink(path, fa)) { m_result.AddVerboseMessage("Storing symlink: {0}", path); } diff --git a/Duplicati/Library/Snapshots/ISystemIO.cs b/Duplicati/Library/Snapshots/ISystemIO.cs index cf3fef902..1e4d01bd1 100644 --- a/Duplicati/Library/Snapshots/ISystemIO.cs +++ b/Duplicati/Library/Snapshots/ISystemIO.cs @@ -50,6 +50,7 @@ namespace Duplicati.Library.Snapshots FileAttributes GetFileAttributes(string path); void SetFileAttributes(string path, FileAttributes attributes); void CreateSymlink(string symlinkfile, string target, bool asDir); + string GetSymlinkTarget(string path); string PathGetDirectoryName(string path); string PathGetFileName(string path); string PathGetExtension(string path); diff --git a/Duplicati/Library/Snapshots/LinuxSnapshot.cs b/Duplicati/Library/Snapshots/LinuxSnapshot.cs index 714ea00ef..f0723bee9 100644 --- a/Duplicati/Library/Snapshots/LinuxSnapshot.cs +++ b/Duplicati/Library/Snapshots/LinuxSnapshot.cs @@ -455,7 +455,7 @@ namespace Duplicati.Library.Snapshots public string GetSymlinkTarget(string file) { var local = ConvertToSnapshotPath(FindSnapShotByLocalPath(file), file); - return UnixSupport.File.GetSymlinkTarget(NoSnapshot.NormalizePath(local)); + return _sysIO.GetSymlinkTarget(local); } /// diff --git a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs index b4e8dc79a..3aa6f8343 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs @@ -45,7 +45,7 @@ namespace Duplicati.Library.Snapshots /// The symlink target public override string GetSymlinkTarget(string file) { - return UnixSupport.File.GetSymlinkTarget(NormalizePath(file)); + return _sysIO.GetSymlinkTarget(file); } /// diff --git a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs index 3aa2fa3e1..d4a692ee8 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs @@ -46,11 +46,7 @@ namespace Duplicati.Library.Snapshots /// The symlink target public override string GetSymlinkTarget(string file) { - try { return File.GetLinkTargetInfo(SystemIOWindows.PrefixWithUNC(file)).PrintName; } - catch (NotAReparsePointException) { } - catch (UnrecognizedReparsePointException) { } - - return null; + return m_sysIO.GetSymlinkTarget(file); } /// diff --git a/Duplicati/Library/Snapshots/SnapshotUtility.cs b/Duplicati/Library/Snapshots/SnapshotUtility.cs index 9d02515be..4875b8239 100644 --- a/Duplicati/Library/Snapshots/SnapshotUtility.cs +++ b/Duplicati/Library/Snapshots/SnapshotUtility.cs @@ -19,6 +19,7 @@ #endregion using System; using System.Collections.Generic; +using System.IO; using System.Text; namespace Duplicati.Library.Snapshots @@ -71,6 +72,60 @@ namespace Duplicati.Library.Snapshots return new WindowsSnapshot(folders, options); } + /// + /// Extension method for ISnapshotService which determines whether the given path is a symlink. + /// + /// ISnapshotService implementation + /// File or folder path + /// Whether the path is a symlink + public static bool IsSymlink(this ISnapshotService snapshot, string path) + { + return snapshot.IsSymlink(path, snapshot.GetAttributes(path)); + } + + /// + /// Extension method for ISnapshotService which determines whether the given path is a symlink. + /// + /// ISnapshotService implementation + /// File or folder path + /// File attributes + /// Whether the path is a symlink + public static bool IsSymlink(this ISnapshotService snapshot, string path, FileAttributes attributes) + { + // Not all reparse points are symlinks. + // For example, on Windows 10 Fall Creator's Update, the OneDrive folder (and all subfolders) + // are reparse points, which allows the folder to hook into the OneDrive service and download things on-demand. + // If we can't find a symlink target for the current path, we won't treat it as a symlink. + return (attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint && !string.IsNullOrEmpty(snapshot.GetSymlinkTarget(path)); + } + + /// + /// Extension method for ISystemIO which determines whether the given path is a symlink. + /// + /// ISystemIO implementation + /// File or folder path + /// Whether the path is a symlink + public static bool IsSymlink(this ISystemIO systemIO, string path) + { + return systemIO.IsSymlink(path, systemIO.GetFileAttributes(path)); + } + + /// + /// Extension method for ISystemIO which determines whether the given path is a symlink. + /// + /// ISystemIO implementation + /// File or folder path + /// File attributes + /// Whether the path is a symlink + public static bool IsSymlink(this ISystemIO systemIO, string path, FileAttributes attributes) + { + // Not all reparse points are symlinks. + // For example, on Windows 10 Fall Creator's Update, the OneDrive folder (and all subfolders) + // are reparse points, which allows the folder to hook into the OneDrive service and download things on-demand. + // If we can't find a symlink target for the current path, we won't treat it as a symlink. + return (attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint && !string.IsNullOrEmpty(systemIO.GetSymlinkTarget(path)); + } + /// /// Gets an interface for System.IO, which wraps all operations in a platform consistent manner. /// diff --git a/Duplicati/Library/Snapshots/SystemIOLinux.cs b/Duplicati/Library/Snapshots/SystemIOLinux.cs index f97462c66..56ea72b4b 100644 --- a/Duplicati/Library/Snapshots/SystemIOLinux.cs +++ b/Duplicati/Library/Snapshots/SystemIOLinux.cs @@ -104,6 +104,11 @@ namespace Duplicati.Library.Snapshots { UnixSupport.File.CreateSymlink(symlinkfile, target); } + + public string GetSymlinkTarget(string path) + { + return UnixSupport.File.GetSymlinkTarget(NoSnapshot.NormalizePath(path)); + } public string PathGetDirectoryName(string path) { diff --git a/Duplicati/Library/Snapshots/SystemIOWindows.cs b/Duplicati/Library/Snapshots/SystemIOWindows.cs index aba8c26de..01b079b0c 100644 --- a/Duplicati/Library/Snapshots/SystemIOWindows.cs +++ b/Duplicati/Library/Snapshots/SystemIOWindows.cs @@ -21,6 +21,8 @@ using System.Collections.Generic; using System.Security.AccessControl; using System.IO; +using AlphaFS = Alphaleonis.Win32.Filesystem; + namespace Duplicati.Library.Snapshots { @@ -261,6 +263,32 @@ namespace Duplicati.Library.Snapshots throw new System.IO.IOException(string.Format("Unable to create symlink, check account permissions: {0}", symlinkfile)); } + /// + /// Returns the symlink target if the entry is a symlink, and null otherwise + /// + /// The file or folder to examine + /// The symlink target + public string GetSymlinkTarget(string file) + { + try + { + try + { + return AlphaFS.File.GetLinkTargetInfo(file).PrintName; + } + catch (PathTooLongException) { } + + return AlphaFS.File.GetLinkTargetInfo(SystemIOWindows.PrefixWithUNC(file)).PrintName; + } + catch (AlphaFS.NotAReparsePointException) { } + catch (AlphaFS.UnrecognizedReparsePointException) { } + + // This path looks like it isn't actually a symlink + // (Note that some reparse points aren't actually symlinks - + // things like the OneDrive folder in the Windows 10 Fall Creator's Update for example) + return null; + } + public IEnumerable EnumerateFileSystemEntries(string path) { if (!IsPathTooLong(path)) diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index 14104c9c5..075784ce9 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -25,6 +25,8 @@ using System.Collections.Generic; using System.IO; using Alphaleonis.Win32.Vss; +using AlphaFS = Alphaleonis.Win32.Filesystem; + namespace Duplicati.Library.Snapshots { /// @@ -129,7 +131,7 @@ namespace Duplicati.Library.Snapshots m_volumes = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (string s in m_sourcepaths) { - string drive = Alphaleonis.Win32.Filesystem.Path.GetPathRoot(s); + string drive = AlphaFS.Path.GetPathRoot(s); if (!m_volumes.ContainsKey(drive)) { if (!m_backup.IsVolumeSupported(drive)) @@ -198,14 +200,14 @@ namespace Duplicati.Library.Snapshots /// A list of non-shadow paths private string[] ListFolders(string folder) { - string root = Utility.Utility.AppendDirSeparator(Alphaleonis.Win32.Filesystem.Path.GetPathRoot(folder)); + string root = Utility.Utility.AppendDirSeparator(AlphaFS.Path.GetPathRoot(folder)); string volumePath = Utility.Utility.AppendDirSeparator(GetSnapshotPath(root)); string[] tmp = null; string spath = GetSnapshotPath(folder); if (SystemIOWindows.IsPathTooLong(spath)) - try { tmp = Alphaleonis.Win32.Filesystem.Directory.GetDirectories(spath); } + try { tmp = AlphaFS.Directory.GetDirectories(spath); } catch (PathTooLongException) { } catch (DirectoryNotFoundException) { } else @@ -216,7 +218,7 @@ namespace Duplicati.Library.Snapshots { spath = SystemIOWindows.PrefixWithUNC(spath); volumePath = SystemIOWindows.PrefixWithUNC(volumePath); - tmp = Alphaleonis.Win32.Filesystem.Directory.GetDirectories(spath); + tmp = AlphaFS.Directory.GetDirectories(spath); } volumePath = SystemIOWindows.PrefixWithUNC(volumePath); @@ -235,14 +237,14 @@ namespace Duplicati.Library.Snapshots /// A list of non-shadow paths private string[] ListFiles(string folder) { - string root = Utility.Utility.AppendDirSeparator(Alphaleonis.Win32.Filesystem.Path.GetPathRoot(folder)); + string root = Utility.Utility.AppendDirSeparator(AlphaFS.Path.GetPathRoot(folder)); string volumePath = Utility.Utility.AppendDirSeparator(GetSnapshotPath(root)); string[] tmp = null; string spath = GetSnapshotPath(folder); if (SystemIOWindows.IsPathTooLong(spath)) - try { tmp = Alphaleonis.Win32.Filesystem.Directory.GetFiles(spath); } + try { tmp = AlphaFS.Directory.GetFiles(spath); } catch (PathTooLongException) { } catch (DirectoryNotFoundException) { } else @@ -253,7 +255,7 @@ namespace Duplicati.Library.Snapshots { spath = SystemIOWindows.PrefixWithUNC(spath); volumePath = SystemIOWindows.PrefixWithUNC(volumePath); - tmp = Alphaleonis.Win32.Filesystem.Directory.GetFiles(spath); + tmp = AlphaFS.Directory.GetFiles(spath); } volumePath = SystemIOWindows.PrefixWithUNC(volumePath); @@ -273,7 +275,7 @@ namespace Duplicati.Library.Snapshots if (!Path.IsPathRooted(localPath)) throw new InvalidOperationException(); - string root = Alphaleonis.Win32.Filesystem.Path.GetPathRoot(localPath); + string root = AlphaFS.Path.GetPathRoot(localPath); string volumePath; if (!m_volumeMap.TryGetValue(root, out volumePath)) @@ -318,7 +320,7 @@ namespace Duplicati.Library.Snapshots } catch (PathTooLongException) { } - return Alphaleonis.Win32.Filesystem.File.GetLastWriteTimeUtc(SystemIOWindows.PrefixWithUNC(spath)); + return AlphaFS.File.GetLastWriteTimeUtc(SystemIOWindows.PrefixWithUNC(spath)); } /// @@ -336,7 +338,7 @@ namespace Duplicati.Library.Snapshots } catch (PathTooLongException) { } - return Alphaleonis.Win32.Filesystem.File.GetCreationTimeUtc(SystemIOWindows.PrefixWithUNC(spath)); + return AlphaFS.File.GetCreationTimeUtc(SystemIOWindows.PrefixWithUNC(spath)); } /// @@ -377,13 +379,7 @@ namespace Duplicati.Library.Snapshots public string GetSymlinkTarget(string file) { string spath = GetSnapshotPath(file); - try - { - return Alphaleonis.Win32.Filesystem.File.GetLinkTargetInfo(spath).PrintName; - } - catch (PathTooLongException) { } - - return Alphaleonis.Win32.Filesystem.File.GetLinkTargetInfo(SystemIOWindows.PrefixWithUNC(spath)).PrintName; + return _ioWin.GetSymlinkTarget(spath); } /// diff --git a/Duplicati/Server/WebServer/RESTMethods/Filesystem.cs b/Duplicati/Server/WebServer/RESTMethods/Filesystem.cs index f93f72013..a6553a527 100644 --- a/Duplicati/Server/WebServer/RESTMethods/Filesystem.cs +++ b/Duplicati/Server/WebServer/RESTMethods/Filesystem.cs @@ -18,6 +18,7 @@ using System; using System.Collections.Generic; using System.Linq; using System.IO; +using Duplicati.Library.Snapshots; namespace Duplicati.Server.WebServer.RESTMethods { @@ -215,7 +216,7 @@ namespace Duplicati.Server.WebServer.RESTMethods try { var attr = systemIO.GetFileAttributes(s); - var isSymlink = (attr & FileAttributes.ReparsePoint) != 0; + var isSymlink = systemIO.IsSymlink(s, attr); var isFolder = (attr & FileAttributes.Directory) != 0; var isFile = !isFolder; var isHidden = (attr & FileAttributes.Hidden) != 0;