diff --git a/Duplicati/GUI/HelperControls/TestSearchSelection.cs b/Duplicati/GUI/HelperControls/TestSearchSelection.cs index fb54f313e..2c93e5a2a 100644 --- a/Duplicati/GUI/HelperControls/TestSearchSelection.cs +++ b/Duplicati/GUI/HelperControls/TestSearchSelection.cs @@ -219,11 +219,11 @@ namespace Duplicati.GUI.HelperControls { try { - CallbackHandler handler = new CallbackHandler(scanWorker); + CallbackHandler handler = new CallbackHandler(scanWorker, fn); foreach (string scanDir in Paths) { if (!scanWorker.CancellationPending) - Library.Utility.Utility.EnumerateFileSystemEntries(scanDir, fn, handler.AddFileToList_Callback); + Library.Utility.Utility.EnumerateFileSystemEntries(scanDir, handler.AddFileToList_Callback); } return handler.Results; @@ -239,26 +239,29 @@ namespace Duplicati.GUI.HelperControls { { public List Results = new List(); private BackgroundWorker Worker; + private FilenameFilter Filter; - public CallbackHandler(BackgroundWorker worker) + public CallbackHandler(BackgroundWorker worker, FilenameFilter filter) { Worker = worker; + Filter = filter; } - public void AddFileToList_Callback(string rootpath, string path, Duplicati.Library.Utility.Utility.EnumeratedFileStatus status) + public bool AddFileToList_Callback(string rootpath, string path, FileAttributes attributes) { if (Worker.CancellationPending) throw new AbortException(); - if (status == Utility.EnumeratedFileStatus.File) + if (!Filter.ShouldInclude(rootpath, path)) + return false; + + if ((attributes & FileAttributes.Directory) == 0 && (attributes & Utility.ATTRIBUTE_ERROR) == 0) { Results.Add(new ResultItem(path)); } - else if (status == Utility.EnumeratedFileStatus.Folder) - { - //Should be displayed in tree so the user can exclude with right click? - } + + return true; } } diff --git a/Duplicati/Library/Main/Duplicati.Library.Main.csproj b/Duplicati/Library/Main/Duplicati.Library.Main.csproj index e0afaf590..456508777 100644 --- a/Duplicati/Library/Main/Duplicati.Library.Main.csproj +++ b/Duplicati/Library/Main/Duplicati.Library.Main.csproj @@ -1,4 +1,4 @@ - + Debug diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index a29a0109e..e420d95a7 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -74,6 +74,27 @@ namespace Duplicati.Library.Main Copy } + /// + /// The possible settings for the symlink strategy + /// + public enum SymlinkStrategy + { + /// + /// Store information about the symlink + /// + Store, + + /// + /// Treat symlinks as normal files or folders + /// + Follow, + + /// + /// Ignore all symlinks + /// + Ignore + } + /// /// Lock that protects the options collection /// @@ -150,6 +171,8 @@ namespace Duplicati.Library.Main "usn-policy", "open-file-policy", "exclude-empty-folders", + "symlink-policy", + "exclude-files-attributes" }; } } @@ -368,6 +391,9 @@ namespace Duplicati.Library.Main new CommandLineArgument("backend-log-database", CommandLineArgument.ArgumentType.Path, Strings.Options.BackendlogdatabaseShort, Strings.Options.BackendlogdatabaseLong), new CommandLineArgument("quota-size", CommandLineArgument.ArgumentType.Size, Strings.Options.QuotasizeShort, Strings.Options.QuotasizeLong), + + new CommandLineArgument("symlink-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.SymlinkpolicyShort, string.Format(Strings.Options.SymlinkpolicyLong, "store", "ignore", "follow"), "store", null, Enum.GetNames(typeof(SymlinkStrategy))), + new CommandLineArgument("exclude-files-attributes", CommandLineArgument.ArgumentType.String, Strings.Options.ExcludefilesattributesShort, string.Format(Strings.Options.ExcludefilesattributesLong, string.Join(", ", Enum.GetNames(typeof(System.IO.FileAttributes))))), }); } } @@ -1006,6 +1032,24 @@ namespace Duplicati.Library.Main } } + /// + /// Gets the snapshot strategy to use + /// + public SymlinkStrategy SymlinkPolicy + { + get + { + string strategy; + if (!m_options.TryGetValue("symlink-policy", out strategy)) + strategy = ""; + + SymlinkStrategy r; + if (!Enum.TryParse(strategy, true, out r)) + r = SymlinkStrategy.Store; + + return r; + } + } /// /// Gets the snapshot strategy to use /// @@ -1144,6 +1188,29 @@ namespace Duplicati.Library.Main } } + /// + /// Gets the attribute filter used to exclude files and folders. + /// + public System.IO.FileAttributes FileAttributeFilter + { + get + { + System.IO.FileAttributes res = (System.IO.FileAttributes)0; + string v; + if (!m_options.TryGetValue("", out v)) + return res; + + foreach(string s in v.Split(new string[] {","}, StringSplitOptions.RemoveEmptyEntries)) + { + System.IO.FileAttributes f; + if (Enum.TryParse(s.Trim(), true, out f)) + res |= f; + } + + return res; + } + } + /// /// A value indicating if a verification file is placed on the server /// diff --git a/Duplicati/Library/Main/RSyncDir.cs b/Duplicati/Library/Main/RSyncDir.cs index baf702411..4bf0013ce 100644 --- a/Duplicati/Library/Main/RSyncDir.cs +++ b/Duplicati/Library/Main/RSyncDir.cs @@ -439,14 +439,15 @@ namespace Duplicati.Library.Main.RSync #endregion #region Filenames - internal static readonly string COMBINED_SIGNATURE_ROOT = "signature"; - internal static readonly string CONTENT_SIGNATURE_ROOT = "content_signature"; - internal static readonly string DELTA_SIGNATURE_ROOT = "delta_signature"; + internal const string COMBINED_SIGNATURE_ROOT = "signature"; + internal const string CONTENT_SIGNATURE_ROOT = "content_signature"; + internal const string DELTA_SIGNATURE_ROOT = "delta_signature"; - internal static readonly string CONTENT_ROOT = "snapshot"; - internal static readonly string DELTA_ROOT = "diff"; - //private static readonly string DACL_ROOT = "dacl"; - internal static readonly string CONTROL_ROOT = "controlfiles"; + internal const string CONTENT_ROOT = "snapshot"; + internal const string DELTA_ROOT = "diff"; + //private const string DACL_ROOT = "dacl"; + internal const string CONTROL_ROOT = "controlfiles"; + internal const string SYMLINK_ROOT = "symlinks"; internal static readonly string DELETED_FILES = "deleted_files.txt"; internal static readonly string DELETED_FOLDERS = "deleted_folders.txt"; @@ -481,6 +482,10 @@ namespace Duplicati.Library.Main.RSync /// This is a list of existing folders. /// private Dictionary m_oldFolders; + /// + /// This is a list of old symlinks + /// + private Dictionary m_oldSymlinks; /// /// This is the list of added files @@ -672,6 +677,7 @@ namespace Duplicati.Library.Main.RSync { m_oldSignatures.Remove(s); m_lastVerificationTime.Remove(s); + m_oldSymlinks.Remove(s); } foreach (string prefix in prefixes) @@ -685,6 +691,13 @@ namespace Duplicati.Library.Main.RSync } } + string symlinkprefix = Utility.Utility.AppendDirSeparator(SYMLINK_ROOT); + foreach(string s in FilenamesFromPlatformIndependant(patch.Value.ListFiles(symlinkprefix))) + { + string tmp = FilenamesFromPlatformIndependant( new string[] { Encoding.UTF8.GetString(patch.Value.ReadAllBytes(s)) })[0]; + m_oldSymlinks[s.Substring(symlinkprefix.Length)] = tmp; + } + if (z.FileExists(UNMODIFIED_FILES)) foreach (string s in FilenamesFromPlatformIndependant(z.ReadAllLines(UNMODIFIED_FILES))) m_lastVerificationTime[s] = patch.Key.Time.ToUniversalTime(); @@ -745,6 +758,7 @@ namespace Duplicati.Library.Main.RSync m_metadata = metadata == null ? new Backupmetadata() : metadata; m_oldSignatures = new Dictionary(Utility.Utility.ClientFilenameStringComparer); m_oldFolders = new Dictionary(Utility.Utility.ClientFilenameStringComparer); + m_oldSymlinks = new Dictionary(Utility.Utility.ClientFilenameStringComparer); m_lastVerificationTime = new Dictionary(Utility.Utility.ClientFilenameStringComparer); for (int i = 0; i < sourcefolder.Length; i++) @@ -793,6 +807,7 @@ namespace Duplicati.Library.Main.RSync { m_oldFolders = new Dictionary(); m_oldSignatures = new Dictionary(); + m_oldSymlinks = new Dictionary(); } m_newfiles = new Dictionary(); @@ -824,13 +839,16 @@ namespace Duplicati.Library.Main.RSync //Failsafe, just use a plain implementation if (m_snapshot == null) { - m_snapshot = new Duplicati.Library.Snapshots.NoSnapshot(m_sourcefolder, options.RawOptions); + m_snapshot = Utility.Utility.IsClientLinux ? + (Library.Snapshots.ISnapshotService)new Duplicati.Library.Snapshots.NoSnapshotLinux(m_sourcefolder, options.RawOptions) + : + (Library.Snapshots.ISnapshotService)new Duplicati.Library.Snapshots.NoSnapshotWindows(m_sourcefolder, options.RawOptions); m_openfilepolicy = options.OpenFilePolicy; } Dictionary usnHelpers = null; List unchanged = new List(); - m_unproccesed = new PathCollector(); + m_unproccesed = new PathCollector(m_snapshot, options.SymlinkPolicy, options.FileAttributeFilter, m_filter, m_stat); try { @@ -904,7 +922,7 @@ namespace Duplicati.Library.Main.RSync } else */ { - usnHelpers[rootFolder].EnumerateFilesAndFolders(s, m_filter, m_unproccesed.Callback); + usnHelpers[rootFolder].EnumerateFilesAndFolders(s, m_unproccesed.Callback); } } @@ -930,7 +948,7 @@ namespace Duplicati.Library.Main.RSync //If we get here, something went really wrong with USN, so we disable it m_currentUSN = null; - m_unproccesed = new PathCollector(); + m_unproccesed = new PathCollector(m_snapshot, options.SymlinkPolicy, options.FileAttributeFilter, m_filter, m_stat); unchanged = null; } finally @@ -948,7 +966,7 @@ namespace Duplicati.Library.Main.RSync if (m_currentUSN == null) { - m_snapshot.EnumerateFilesAndFolders(m_filter, m_unproccesed.Callback); + m_snapshot.EnumerateFilesAndFolders(m_unproccesed.Callback); } else { @@ -958,12 +976,13 @@ namespace Duplicati.Library.Main.RSync string relpath = GetRelativeName(x); m_oldSignatures.Remove(relpath); m_oldFolders.Remove(relpath); + m_oldSymlinks.Remove(relpath); } //If some folders did not support USN, add their files now foreach (string s in m_sourcefolder) if (!m_currentUSN.ContainsKey(System.IO.Path.GetPathRoot(s))) - m_snapshot.EnumerateFilesAndFolders(s, m_filter, m_unproccesed.Callback); + m_snapshot.EnumerateFilesAndFolders(s, m_unproccesed.Callback); } m_totalfiles = m_unproccesed.Files.Count; @@ -1023,7 +1042,7 @@ namespace Duplicati.Library.Main.RSync string relpath = GetRelativeName(s); if (relpath.Trim().Length != 0) { - DateTime lastWrite = Directory.GetLastWriteTimeUtc(s); + DateTime lastWrite = m_snapshot.GetLastWriteTime(s).ToUniversalTime(); //Cut off as we only have seconds stored lastWrite = new DateTime(lastWrite.Year, lastWrite.Month, lastWrite.Day, lastWrite.Hour, lastWrite.Minute, lastWrite.Second, DateTimeKind.Utc); @@ -1050,6 +1069,39 @@ namespace Duplicati.Library.Main.RSync if (!m_unproccesed.IsAffectedByError(s)) m_deletedfolders.Add(s); + //Build symlink diffs + if (m_oldSymlinks.Count > 0) + { + for (int i = m_unproccesed.Symlinks.Count - 1; i >= 0; i--) + { + string s = m_unproccesed.Symlinks[i].Key; + try + { + string relpath = GetRelativeName(s); + if (relpath.Trim().Length != 0) + { + string oldLink; + if (m_oldSymlinks.TryGetValue(relpath, out oldLink)) + { + m_oldSymlinks.Remove(relpath); + if (string.Equals(oldLink, m_unproccesed.Symlinks[i].Value, Utility.Utility.ClientFilenameStringComparision)) + { + m_unproccesed.Symlinks.RemoveAt(i); + } + } + } + } + catch (Exception ex) + { + m_unproccesed.Errors.Add(s); + m_stat.LogError(string.Format(Strings.RSyncDir.SymlinkReadError, s, ex.Message), ex); + } + } + } + + m_deletedfiles.AddRange(m_oldSymlinks.Keys); + m_oldSymlinks.Clear(); + m_sortedfilelist = options.SortedFilelist; if (m_sortedfilelist) m_unproccesed.Files.Sort(Utility.Utility.ClientFilenameStringComparer); @@ -1173,7 +1225,7 @@ namespace Duplicati.Library.Main.RSync if (!m_unproccesed.IsAffectedByError(fullpath)) { m_deletedfiles.Add(s); - stringsize += System.Text.Encoding.UTF8.GetByteCount(s + Environment.NewLine); + stringsize += System.Text.Encoding.UTF8.GetByteCount(s + "\r\n"); } } catch (Exception ex) @@ -1185,6 +1237,8 @@ namespace Duplicati.Library.Main.RSync } } + m_oldSignatures.Clear(); + if (m_deletedfiles.Count > 0) { //The +100 is a safety margin @@ -1194,6 +1248,7 @@ namespace Duplicati.Library.Main.RSync signaturefile.WriteAllLines(DELETED_FILES, m_deletedfiles.ToArray()); contentfile.WriteAllLines(DELETED_FILES, m_deletedfiles.ToArray()); + m_deletedfiles.Clear(); } //We only write the USN if all files were processed @@ -1204,6 +1259,20 @@ namespace Duplicati.Library.Main.RSync //Only write this if all files were processed if (m_checkedUnchangedFiles.Count > 0) signaturefile.WriteAllLines(UNMODIFIED_FILES, m_checkedUnchangedFiles.ToArray()); + + if (m_unproccesed.Symlinks.Count > 0) + { + foreach(KeyValuePair kvp in m_unproccesed.Symlinks) + { + string target = FilenamesToPlatformIndependant(new string[] { kvp.Value })[0]; + string source = Path.Combine(SYMLINK_ROOT, GetRelativeName(kvp.Key)); + byte[] targetBytes = Encoding.UTF8.GetBytes(target); + + contentfile.WriteAllBytes(source, targetBytes); + signaturefile.WriteAllBytes(source, targetBytes); + } + m_unproccesed.Symlinks.Clear(); + } } m_finalized = true; @@ -1299,9 +1368,14 @@ namespace Duplicati.Library.Main.RSync m_lastPartialFile = WritePossiblePartial(m_lastPartialFile, contentfile, signaturefile, volumesize); } + if (m_lastPartialFile != null) + return false; - while (m_unproccesed.Files.Count > 0 && totalSize < volumesize && m_lastPartialFile == null) + while (m_unproccesed.Files.Count > 0) { + if (totalSize >= volumesize) + break; + int next = m_sortedfilelist ? 0 : r.Next(0, m_unproccesed.Files.Count); string s = m_unproccesed.Files[next]; m_unproccesed.Files.RemoveAt(next); @@ -1504,7 +1578,7 @@ namespace Duplicati.Library.Main.RSync } } - if (m_unproccesed.Files.Count == 0 && m_lastPartialFile == null) + if (m_unproccesed.Files.Count == 0) return FinalizeMultiPass(signaturefile, contentfile, volumesize); else return false; @@ -1740,15 +1814,16 @@ namespace Duplicati.Library.Main.RSync /// public void FinalizeRestore() { + Snapshots.ISystemIO SystemIO = Utility.Utility.IsClientLinux ? (Snapshots.ISystemIO)new Snapshots.SystemIOLinux() : (Snapshots.ISystemIO)new Snapshots.SystemIOWindows(); if (m_folders_to_delete != null) { foreach (string s in m_folders_to_delete) { - if (System.IO.Directory.Exists(s)) + if (SystemIO.DirectoryExists(s)) try { //TODO: Perhaps read ahead in patches to prevent creation - System.IO.Directory.Delete(s, false); + SystemIO.DirectoryDelete(s, false); if (m_stat as RestoreStatistics != null) { (m_stat as RestoreStatistics).FoldersDeleted++; @@ -1790,8 +1865,8 @@ namespace Duplicati.Library.Main.RSync if (m_folderTimestamps != null) foreach (KeyValuePair t in m_folderTimestamps) - if (System.IO.Directory.Exists(t.Key)) - try { System.IO.Directory.SetLastWriteTimeUtc(t.Key, t.Value); } + if (SystemIO.DirectoryExists(t.Key)) + try { SystemIO.DirectorySetLastWriteTimeUtc(t.Key, t.Value); } catch (Exception ex) { m_stat.LogWarning(string.Format(Strings.RSyncDir.FailedToSetFolderWriteTime, t.Key, ex.Message), ex); @@ -1881,6 +1956,8 @@ namespace Duplicati.Library.Main.RSync /// The content file that the destination is patched with public void Patch(string[] destination, Library.Interface.ICompression patch) { + Snapshots.ISystemIO SystemIO = Utility.Utility.IsClientLinux ? (Snapshots.ISystemIO)new Snapshots.SystemIOLinux() : (Snapshots.ISystemIO)new Snapshots.SystemIOWindows(); + if (m_partialDeltas == null) m_partialDeltas = new Dictionary(); @@ -1899,14 +1976,14 @@ namespace Duplicati.Library.Main.RSync if (patch.FileExists(DELETED_FILES)) foreach (string s in fh.Filterlist(FilenamesFromPlatformIndependant(patch.ReadAllLines(DELETED_FILES)), false)) { - if (System.IO.File.Exists(s)) + if (SystemIO.FileExists(s)) { try { //TODO: Perhaps read ahead in patches to prevent creation - long size = new FileInfo(s).Length; + long size = SystemIO.FileLength(s); - System.IO.File.Delete(s); + SystemIO.FileDelete(s); if (m_stat as RestoreStatistics != null) { (m_stat as RestoreStatistics).FilesRestored--; @@ -1955,10 +2032,10 @@ namespace Duplicati.Library.Main.RSync foreach (string s in addedfolders) { - if (!System.IO.Directory.Exists(s)) + if (!SystemIO.DirectoryExists(s)) try { - System.IO.Directory.CreateDirectory(s); + SystemIO.DirectoryCreate(s); if (m_stat as RestoreStatistics != null) (m_stat as RestoreStatistics).FoldersRestored++; } @@ -2009,6 +2086,9 @@ namespace Duplicati.Library.Main.RSync string deltaprefix = Utility.Utility.AppendDirSeparator(DELTA_ROOT); List deltafiles = m_filter.FilterList(deltaprefix, patch.ListFiles(deltaprefix)); + string symlinkprefix = Utility.Utility.AppendDirSeparator(SYMLINK_ROOT); + List symlinks = m_filter.FilterList(symlinkprefix, patch.ListFiles(symlinkprefix)); + long totalfiles = deltafiles.Count + contentfiles.Count; long fileindex = 0; @@ -2018,10 +2098,10 @@ namespace Duplicati.Library.Main.RSync string target = GetFullPathFromRelname(destination, s.Substring(contentprefix.Length)); try { - if (!System.IO.Directory.Exists(System.IO.Path.GetDirectoryName(target))) + if (!SystemIO.DirectoryExists(SystemIO.PathGetDirectoryName(target))) { Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.RestoreFolderMissingError, target), Duplicati.Library.Logging.LogMessageType.Warning); - System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(target)); + SystemIO.DirectoryCreate(SystemIO.PathGetDirectoryName(target)); } //Update each 0.5% @@ -2058,7 +2138,7 @@ namespace Duplicati.Library.Main.RSync throw new Exception(string.Format(Strings.RSyncDir.FileShouldBePartialError, s)); long startOffset = pex == null ? 0 : pex.StartOffset; - using (System.IO.FileStream s2 = System.IO.File.OpenWrite(partialFile == null ? target : (string)partialFile)) + using (System.IO.Stream s2 = SystemIO.FileOpenWrite(partialFile == null ? target : (string)partialFile)) { if (s2.Length != startOffset) throw new Exception(string.Format(Strings.RSyncDir.InvalidPartialFileEntry, s)); @@ -2072,9 +2152,9 @@ namespace Duplicati.Library.Main.RSync if (pex != null && pex == fe) { - if (System.IO.File.Exists(target)) - System.IO.File.Delete(target); - System.IO.File.Move(partialFile, target); + if (SystemIO.FileExists(target)) + SystemIO.FileDelete(target); + SystemIO.FileMove(partialFile, target); partialFile.Dispose(); m_partialDeltas.Remove(s); } @@ -2082,7 +2162,7 @@ namespace Duplicati.Library.Main.RSync if (m_stat is RestoreStatistics && (partialFile == null || pex == fe)) { (m_stat as RestoreStatistics).FilesRestored++; - (m_stat as RestoreStatistics).SizeOfRestoredFiles += new FileInfo(target).Length; + (m_stat as RestoreStatistics).SizeOfRestoredFiles += SystemIO.FileLength(target); } } @@ -2091,7 +2171,7 @@ namespace Duplicati.Library.Main.RSync DateTime t = patch.GetLastWriteTime(s); if (!isUtc) t = t.ToUniversalTime(); - try { File.SetLastWriteTimeUtc(target, t); } + try { SystemIO.FileSetLastWriteTimeUtc(target, t); } catch (Exception ex) { if (m_stat != null) @@ -2122,10 +2202,10 @@ namespace Duplicati.Library.Main.RSync lastPg = pg; } - if (!System.IO.Directory.Exists(System.IO.Path.GetDirectoryName(target))) + if (!SystemIO.DirectoryExists(SystemIO.PathGetDirectoryName(target))) { Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.RestoreFolderDeltaError, target), Duplicati.Library.Logging.LogMessageType.Warning); - System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(target)); + SystemIO.DirectoryCreate(SystemIO.PathGetDirectoryName(target)); } PartialEntryRecord pex = null; @@ -2147,7 +2227,7 @@ namespace Duplicati.Library.Main.RSync m_partialDeltas.Add(s, new Duplicati.Library.Utility.TempFile()); //Dump the content in the temp file at the specified offset - using (System.IO.Stream st = System.IO.File.OpenWrite(m_partialDeltas[s])) + using (System.IO.Stream st = SystemIO.FileOpenWrite(m_partialDeltas[s])) { if (st.Length != pex.StartOffset) throw new Exception(string.Format(Strings.RSyncDir.InvalidPartialFileEntry, s)); @@ -2171,28 +2251,28 @@ namespace Duplicati.Library.Main.RSync using (tempDelta) //May be null, but the using directive does not care { //Use either the patch directly, or the partial temp file - System.IO.Stream deltaStream = tempDelta == null ? patch.OpenRead(s) : System.IO.File.OpenRead(tempDelta); + System.IO.Stream deltaStream = tempDelta == null ? patch.OpenRead(s) : SystemIO.FileOpenRead(tempDelta); using (System.IO.Stream s2 = deltaStream) - using (System.IO.FileStream s1 = System.IO.File.OpenRead(target)) - using (System.IO.FileStream s3 = System.IO.File.Create(tempfile)) + using (System.IO.Stream s1 = SystemIO.FileOpenRead(target)) + using (System.IO.Stream s3 = SystemIO.FileCreate(tempfile)) SharpRSync.Interface.PatchFile(s1, s2, s3); if (m_stat as RestoreStatistics != null) { - (m_stat as RestoreStatistics).SizeOfRestoredFiles -= new FileInfo(target).Length; - (m_stat as RestoreStatistics).SizeOfRestoredFiles += new FileInfo(tempfile).Length; + (m_stat as RestoreStatistics).SizeOfRestoredFiles -= SystemIO.FileLength(target); + (m_stat as RestoreStatistics).SizeOfRestoredFiles += SystemIO.FileLength(tempfile); (m_stat as RestoreStatistics).FilesPatched++; } - System.IO.File.Delete(target); + SystemIO.FileDelete(target); - try { System.IO.File.Move(tempfile, target); } + try { SystemIO.FileMove(tempfile, target); } catch { //The OS sometimes reports the file as existing even after a delete // this seems to be related to MS Security Essentials? System.Threading.Thread.Sleep(500); - System.IO.File.Move(tempfile, target); + SystemIO.FileMove(tempfile, target); } } @@ -2202,7 +2282,7 @@ namespace Duplicati.Library.Main.RSync if (!isUtc) t = t.ToUniversalTime(); - try { File.SetLastWriteTimeUtc(target, t); } + try { SystemIO.FileSetLastWriteTimeUtc(target, t); } catch (Exception ex) { if (m_stat != null) @@ -2216,11 +2296,58 @@ namespace Duplicati.Library.Main.RSync m_stat.LogError(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), ex); Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), Duplicati.Library.Logging.LogMessageType.Error, ex); - try { System.IO.File.Delete(target); } + try { SystemIO.FileDelete(target); } catch { } } fileindex++; } + + //Re-create symlinks (no progress report here, should be really fast) + foreach (string s in symlinks) + { + string target = GetFullPathFromRelname(destination, s.Substring(symlinkprefix.Length)); + string symlinktarget = ""; + try + { + symlinktarget = FilenamesFromPlatformIndependant(new string[] { Encoding.UTF8.GetString(patch.ReadAllBytes(s)) })[0]; + bool isDir = symlinktarget[symlinktarget.Length - 1] == Path.DirectorySeparatorChar; + if (isDir) + symlinktarget = symlinktarget.Substring(0, symlinktarget.Length - 1); + + try + { + //In case another symlink is present, we "update" it + if (SystemIO.FileExists(target) && (SystemIO.GetFileAttributes(target) & FileAttributes.ReparsePoint) != 0) + SystemIO.FileDelete(target); + } + catch (Exception ex) + { + Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), Duplicati.Library.Logging.LogMessageType.Error, ex); + } + + SystemIO.CreateSymlink(target, symlinktarget, isDir); + } + catch (Exception ex) + { + if (m_stat != null) + m_stat.LogError(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), ex); + Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), Duplicati.Library.Logging.LogMessageType.Error, ex); + + try { SystemIO.FileDelete(target); } + catch { } + + try + { + if (!string.IsNullOrEmpty(symlinktarget)) + using (System.IO.StreamWriter sw = new System.IO.StreamWriter(SystemIO.FileOpenWrite(target))) + sw.Write(symlinktarget); + } + catch + { + } + } + } + } @@ -2379,6 +2506,7 @@ namespace Duplicati.Library.Main.RSync List lst = new List(); lst.AddRange(m_oldFolders.Keys); lst.AddRange(m_oldSignatures.Keys); + lst.AddRange(m_oldSymlinks.Keys); return lst; } @@ -2508,20 +2636,79 @@ namespace Duplicati.Library.Main.RSync /// private class PathCollector { + private readonly Options.SymlinkStrategy m_symlinkHandling; + private readonly FileAttributes m_attributeMask; + private readonly Utility.FilenameFilter m_filter; + private readonly Snapshots.ISnapshotService m_snapshot; + private readonly CommunicationStatistics m_stat; + private List m_files = new List(); private List m_folders = new List(); private List m_errors = new List(); private List m_filesWithError = new List(); private List m_filesTooLarge = new List(); + private List> m_symlinks = new List>(); - public void Callback(string rootpath, string path, Utility.Utility.EnumeratedFileStatus status) + public PathCollector(Snapshots.ISnapshotService snapshot, Options.SymlinkStrategy symlinkHandling, FileAttributes attributeMask, Utility.FilenameFilter filter, CommunicationStatistics stat) { - if (status == Utility.Utility.EnumeratedFileStatus.Folder) - m_folders.Add(path); - else if (status == Utility.Utility.EnumeratedFileStatus.File) - m_files.Add(path); - else if (status == Utility.Utility.EnumeratedFileStatus.Error) + m_symlinkHandling = symlinkHandling; + m_filter = filter; + m_attributeMask = attributeMask; + m_snapshot = snapshot; + m_stat = stat; + } + + public bool Callback(string rootpath, string path, FileAttributes attributes) + { + if ((attributes & Utility.Utility.ATTRIBUTE_ERROR) != 0) + { m_errors.Add(path); + return false; + } + + if ((attributes & m_attributeMask) != 0) + return false; + + if (!m_filter.ShouldInclude(rootpath, path)) + return false; + + if ((attributes & FileAttributes.ReparsePoint) != 0) + { + if (m_symlinkHandling == Options.SymlinkStrategy.Ignore) + return false; + else if (m_symlinkHandling == Options.SymlinkStrategy.Store) + { + try + { + //We treat symlinks as files, even if they point to folders + if (path[path.Length - 1] == Path.DirectorySeparatorChar) + path = path.Substring(0, path.Length - 1); + + string s = m_snapshot.GetSymlinkTarget(path); + if (s != null) + { + if ((attributes & FileAttributes.Directory) != 0) + s = Utility.Utility.AppendDirSeparator(s); + m_symlinks.Add(new KeyValuePair(path, s)); + } + } + catch(Exception ex) + { + Logging.Log.WriteMessage(string.Format("Failed to obtain symlink target for {0}, message: {1}", path, ex.Message), Duplicati.Library.Logging.LogMessageType.Warning, ex); + if (m_stat != null) + m_stat.LogWarning(string.Format("Failed to obtain symlink target for {0}, message: {1}", path, ex.Message), ex); + m_errors.Add(path); + } + return false; + } + } + + if ((attributes & FileAttributes.Directory) != 0) + m_folders.Add(path); + else + m_files.Add(path); + + return true; } public List Files { get { return m_files; } } @@ -2529,6 +2716,7 @@ namespace Duplicati.Library.Main.RSync public List Errors { get { return m_errors; } } public List FilesWithError { get { return m_filesWithError; } } public List FilesTooLarge { get { return m_filesTooLarge; } } + public List> Symlinks { get { return m_symlinks; } } public bool IsAffectedByError(string path) { diff --git a/Duplicati/Library/Main/Strings/Options.Designer.cs b/Duplicati/Library/Main/Strings/Options.Designer.cs index 8613d912d..7034f76ae 100644 --- a/Duplicati/Library/Main/Strings/Options.Designer.cs +++ b/Duplicati/Library/Main/Strings/Options.Designer.cs @@ -1265,5 +1265,42 @@ namespace Duplicati.Library.Main.Strings { return ResourceManager.GetString("VssexcludewritersShort", resourceCulture); } } + + /// + /// Looks up a localized string similar to Using this option to handle symlinks different. The "{0}" option will simply record a symlink with its name and destination, and a restore will recreate the symlink as a link. Use the option "{1}" to ignore all symlinks and not store any information about them. Previous versions of Duplicati used the setting "{2}", which will cause symlinked files to be included and restore as normal files.. + /// + internal static string SymlinkpolicyLong { + get { + return ResourceManager.GetString("SymlinkpolicyLong", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Symlink handling. + /// + internal static string SymlinkpolicyShort { + get { + return ResourceManager.GetString("SymlinkpolicyShort", resourceCulture); + } + } + + + /// + /// Exclude files by attribute. + /// + internal static string ExcludefilesattributesShort { + get { + return ResourceManager.GetString("ExcludefilesattributesShort", resourceCulture); + } + } + + /// + /// Use this option to exclude files with certain attributes. Use a comma separated list of attribute names to specify more that one. Possible values are: {0}. + /// + internal static string ExcludefilesattributesLong { + get { + return ResourceManager.GetString("ExcludefilesattributesLong", resourceCulture); + } + } } } diff --git a/Duplicati/Library/Main/Strings/Options.resx b/Duplicati/Library/Main/Strings/Options.resx index c77efec96..50c50a0c6 100644 --- a/Duplicati/Library/Main/Strings/Options.resx +++ b/Duplicati/Library/Main/Strings/Options.resx @@ -518,5 +518,18 @@ A reported maximum storage + + + Symlink handling + + Using this option to handle symlinks different. The "{0}" option will simply record a symlink with its name and destination, and a restore will recreate the symlink as a link. Use the option "{1}" to ignore all symlinks and not store any information about them. Previous versions of Duplicati used the setting "{2}", which will cause symlinked files to be included and restore as normal files. + + + Exclude files by attribute + + + Use this option to exclude files with certain attributes. Use a comma separated list of attribute names to specify more that one. Possible values are: {0} + + \ No newline at end of file diff --git a/Duplicati/Library/Main/Strings/RSyncDir.Designer.cs b/Duplicati/Library/Main/Strings/RSyncDir.Designer.cs index 070442519..03621e2f4 100644 --- a/Duplicati/Library/Main/Strings/RSyncDir.Designer.cs +++ b/Duplicati/Library/Main/Strings/RSyncDir.Designer.cs @@ -332,5 +332,14 @@ namespace Duplicati.Library.Main.Strings { return ResourceManager.GetString("UsnNumberingFaultWarning", resourceCulture); } } - } + + /// + /// Looks up a localized string similar to The symlink information for {0} could not be read: {1}. + /// + internal static string SymlinkReadError { + get { + return ResourceManager.GetString("SymlinkReadError", resourceCulture); + } + } +} } diff --git a/Duplicati/Library/Main/Strings/RSyncDir.resx b/Duplicati/Library/Main/Strings/RSyncDir.resx index 4f000fe4e..a1392cb09 100644 --- a/Duplicati/Library/Main/Strings/RSyncDir.resx +++ b/Duplicati/Library/Main/Strings/RSyncDir.resx @@ -210,4 +210,7 @@ Error message: {1} The timestamp information for file {0} could not be read: {1} + + The symlink information for {0} could not be read: {1} + \ No newline at end of file diff --git a/Duplicati/Library/Snapshots/Duplicati.Library.Snapshots.csproj b/Duplicati/Library/Snapshots/Duplicati.Library.Snapshots.csproj index f1f1ef261..7801eca99 100644 --- a/Duplicati/Library/Snapshots/Duplicati.Library.Snapshots.csproj +++ b/Duplicati/Library/Snapshots/Duplicati.Library.Snapshots.csproj @@ -64,6 +64,9 @@ ..\..\..\thirdparty\alphavss\Bin\AlphaVSS.Common.dll + + ..\..\..\thirdparty\UnixSupport\UnixSupport.dll + @@ -97,6 +100,11 @@ + + + + + diff --git a/Duplicati/Library/Snapshots/ISnapshotService.cs b/Duplicati/Library/Snapshots/ISnapshotService.cs index 8ee382f4b..2163a278d 100644 --- a/Duplicati/Library/Snapshots/ISnapshotService.cs +++ b/Duplicati/Library/Snapshots/ISnapshotService.cs @@ -31,17 +31,15 @@ namespace Duplicati.Library.Snapshots /// /// Enumerates all files and folders in the snapshot /// - /// The filter to apply when evaluating files and folders /// The callback to invoke with each found path - void EnumerateFilesAndFolders(Duplicati.Library.Utility.FilenameFilter filter, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback); + void EnumerateFilesAndFolders(Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback); /// /// Enumerates all files and folders in the snapshot /// /// The path from which to retrieve files and folders - /// The filter to apply when evaluating files and folders /// The callback to invoke with each found path - void EnumerateFilesAndFolders(string startpath, Duplicati.Library.Utility.FilenameFilter filter, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback); + void EnumerateFilesAndFolders(string startpath, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback); /// /// Gets the last write time of a given file @@ -63,5 +61,19 @@ namespace Duplicati.Library.Snapshots /// The full path to the file in non-snapshot format /// The lenth of the file long GetFileSize(string file); + + /// + /// Returns the symlink target if the entry is a symlink, and null otherwise + /// + /// The file or folder to examine + /// The symlink target + string GetSymlinkTarget(string file); + + /// + /// Gets the attributes for the given file or folder + /// + /// The file attributes + /// The file or folder to examine + System.IO.FileAttributes GetAttributes(string file); } } diff --git a/Duplicati/Library/Snapshots/ISystemIO.cs b/Duplicati/Library/Snapshots/ISystemIO.cs new file mode 100644 index 000000000..2c9a64478 --- /dev/null +++ b/Duplicati/Library/Snapshots/ISystemIO.cs @@ -0,0 +1,47 @@ +// Copyright (C) 2011, Kenneth Skovhede + +// http://www.hexad.dk, opensource@hexad.dk +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +using System; +using System.IO; + +namespace Duplicati.Library.Snapshots +{ + /// + /// Interface for wrapping System.IO operations. + /// + public interface ISystemIO + { + void DirectoryDelete(string path); + void DirectoryDelete(string path, bool recursive); + void DirectoryCreate(string path); + bool DirectoryExists(string path); + void DirectorySetLastWriteTimeUtc(string path, DateTime time); + + void FileMove(string source, string target); + void FileDelete(string path); + void FileSetLastWriteTimeUtc(string path, DateTime time); + bool FileExists(string path); + long FileLength(string path); + Stream FileOpenRead(string path); + Stream FileOpenWrite(string path); + Stream FileCreate(string path); + FileAttributes GetFileAttributes(string path); + void CreateSymlink(string symlinkfile, string target, bool asDir); + string PathGetDirectoryName(string path); + } +} + diff --git a/Duplicati/Library/Snapshots/LinuxSnapshot.cs b/Duplicati/Library/Snapshots/LinuxSnapshot.cs index dbdb00ce7..340892787 100644 --- a/Duplicati/Library/Snapshots/LinuxSnapshot.cs +++ b/Duplicati/Library/Snapshots/LinuxSnapshot.cs @@ -386,12 +386,12 @@ namespace Duplicati.Library.Snapshots /// The path from which to retrieve files and folders /// The filter to apply when evaluating files and folders /// The callback to invoke with each found path - public void EnumerateFilesAndFolders(string startpath, Duplicati.Library.Utility.FilenameFilter filter, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) + public void EnumerateFilesAndFolders(string startpath, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) { foreach (KeyValuePair s in m_entries) if (s.Key.Equals(startpath, Utility.Utility.ClientFilenameStringComparision)) { - Utility.Utility.EnumerateFileSystemEntries(s.Key, filter, callback, this.ListFolders, this.ListFiles); + Utility.Utility.EnumerateFileSystemEntries(s.Key, callback, this.ListFolders, this.ListFiles); return; } @@ -403,10 +403,10 @@ namespace Duplicati.Library.Snapshots /// /// The filter to apply when evaluating files and folders /// The callback to invoke with each found path - public void EnumerateFilesAndFolders(Duplicati.Library.Utility.FilenameFilter filter, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) + public void EnumerateFilesAndFolders(Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) { foreach (KeyValuePair s in m_entries) - Utility.Utility.EnumerateFileSystemEntries(s.Key, filter, callback, this.ListFolders, this.ListFiles); + Utility.Utility.EnumerateFileSystemEntries(s.Key, callback, this.ListFolders, this.ListFiles); } /// @@ -439,6 +439,26 @@ namespace Duplicati.Library.Snapshots return new System.IO.FileInfo(ConvertToSnapshotPath(FindSnapShotByLocalPath(file), file)).Length; } + /// + /// Gets the attributes for the given file or folder + /// + /// The file attributes + /// The file or folder to examine + public System.IO.FileAttributes GetAttributes(string file) + { + return System.IO.File.GetAttributes(file); + } + + /// + /// Returns the symlink target if the entry is a symlink, and null otherwise + /// + /// The file or folder to examine + /// The symlink target + public string GetSymlinkTarget(string file) + { + return UnixSupport.File.GetSymlinkTarget(file); + } + #endregion #region IDisposable Members diff --git a/Duplicati/Library/Snapshots/NoSnapshot.cs b/Duplicati/Library/Snapshots/NoSnapshot.cs index 1ebf9cdc1..a1bf5519a 100644 --- a/Duplicati/Library/Snapshots/NoSnapshot.cs +++ b/Duplicati/Library/Snapshots/NoSnapshot.cs @@ -27,12 +27,17 @@ namespace Duplicati.Library.Snapshots /// A simple implementation of snapshots that only uses the regular System.IO operations, /// used to simplify code that uses snapshots, but also handles a non-snapshot enabled systems /// - public class NoSnapshot : ISnapshotService + public abstract class NoSnapshot : ISnapshotService { /// /// The list of all folders in the snapshot /// - private string[] m_sourcefolders; + protected string[] m_sourcefolders; + + /// + /// A frequently used char-as-string + /// + protected readonly string DIR_SEP = System.IO.Path.DirectorySeparatorChar.ToString(); /// /// Constructs a new backup snapshot, using all the required disks @@ -47,7 +52,7 @@ namespace Duplicati.Library.Snapshots /// Constructs a new backup snapshot, using all the required disks /// /// The folders that are about to be backed up - /// True if the file may be opened with read/write permissions + /// A set of system options public NoSnapshot(string[] folders, Dictionary options) { m_sourcefolders = new string[folders.Length]; @@ -55,6 +60,29 @@ namespace Duplicati.Library.Snapshots m_sourcefolders[i] = Utility.Utility.AppendDirSeparator(folders[i]); } + #region Private Methods + /// + /// Lists all folders in the given folder + /// + /// All folders found in the folder + /// The folder to examinate + protected virtual string[] ListFolders(string folder) + { + return System.IO.Directory.GetDirectories(folder); + } + + + /// + /// Lists all files in the given folder + /// + /// All folders found in the folder + /// The folder to examinate + protected virtual string[] ListFiles(string folder) + { + return System.IO.Directory.GetFiles(folder); + } + #endregion + #region ISnapshotService Members /// @@ -68,14 +96,13 @@ namespace Duplicati.Library.Snapshots /// Enumerates all files and folders in the snapshot /// /// The path from which to retrieve files and folders - /// The filter to apply when evaluating files and folders /// The callback to invoke with each found path - public void EnumerateFilesAndFolders(string startpath, Duplicati.Library.Utility.FilenameFilter filter, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) + public void EnumerateFilesAndFolders(string startpath, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) { foreach (string s in m_sourcefolders) if (s.Equals(startpath, Utility.Utility.ClientFilenameStringComparision)) { - Utility.Utility.EnumerateFileSystemEntries(s, filter, callback); + Utility.Utility.EnumerateFileSystemEntries(s, callback, this.ListFolders, this.ListFiles, this.GetAttributes); return; } @@ -85,12 +112,11 @@ namespace Duplicati.Library.Snapshots /// /// Enumerates all files and folders in the snapshot /// - /// The filter to apply when evaluating files and folders /// The callback to invoke with each found path - public void EnumerateFilesAndFolders(Duplicati.Library.Utility.FilenameFilter filter, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) + public void EnumerateFilesAndFolders(Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) { foreach (string s in m_sourcefolders) - Utility.Utility.EnumerateFileSystemEntries(s, filter, callback); + Utility.Utility.EnumerateFileSystemEntries(s, callback, this.ListFolders, this.ListFiles, this.GetAttributes); } /// @@ -98,7 +124,7 @@ namespace Duplicati.Library.Snapshots /// /// The full path to the file in non-snapshot format /// The last write time of the file - public DateTime GetLastWriteTime(string file) + public virtual DateTime GetLastWriteTime(string file) { return System.IO.File.GetLastWriteTime(file); } @@ -108,7 +134,7 @@ namespace Duplicati.Library.Snapshots /// /// The full path to the file in non-snapshot format /// An open filestream that can be read - public System.IO.Stream OpenRead(string file) + public virtual System.IO.Stream OpenRead(string file) { return System.IO.File.OpenRead(file); } @@ -118,7 +144,7 @@ namespace Duplicati.Library.Snapshots /// /// The full path to the file in non-snapshot format /// An open filestream that can be read - public System.IO.Stream OpenLockedRead(string file) + public virtual System.IO.Stream OpenLockedRead(string file) { return System.IO.File.Open(file, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.ReadWrite); } @@ -128,10 +154,30 @@ namespace Duplicati.Library.Snapshots /// /// The full path to the file in non-snapshot format /// The lenth of the file - public long GetFileSize(string file) + public virtual long GetFileSize(string file) { return new System.IO.FileInfo(file).Length; } + + /// + /// Returns the symlink target if the entry is a symlink, and null otherwise + /// + /// The file or folder to examine + /// The symlink target + public abstract string GetSymlinkTarget(string file); + + /// + /// Gets the attributes for the given file or folder + /// + /// The file attributes + /// The file or folder to examine + public virtual System.IO.FileAttributes GetAttributes(string file) + { + if (file.EndsWith(DIR_SEP)) + return System.IO.File.GetAttributes(file.Substring(0, file.Length - 1)); + else + return System.IO.File.GetAttributes(file); + } #endregion } } diff --git a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs new file mode 100644 index 000000000..5a62ab847 --- /dev/null +++ b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs @@ -0,0 +1,53 @@ +// Copyright (C) 2011, Kenneth Skovhede + +// http://www.hexad.dk, opensource@hexad.dk +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +using System; +using System.Text; +using System.Collections.Generic; + +namespace Duplicati.Library.Snapshots +{ + /// + /// Handler for providing a snapshot like access to files and folders + /// + public class NoSnapshotLinux : NoSnapshot + { + public NoSnapshotLinux(string[] sourcefolders) + : base(sourcefolders) + { + } + + public NoSnapshotLinux(string[] sourcefolders, Dictionary options) + : base(sourcefolders, options) + { + } + + /// + /// Returns the symlink target if the entry is a symlink, and null otherwise + /// + /// The file or folder to examine + /// The symlink target + public override string GetSymlinkTarget(string file) + { + if (file.EndsWith(DIR_SEP)) + return UnixSupport.File.GetSymlinkTarget(file.Substring(0, file.Length - 1)); + else + return UnixSupport.File.GetSymlinkTarget(file); + } + } +} + diff --git a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs new file mode 100644 index 000000000..a8e9dfc6d --- /dev/null +++ b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs @@ -0,0 +1,153 @@ +// Copyright (C) 2011, Kenneth Skovhede + +// http://www.hexad.dk, opensource@hexad.dk +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +using System; +using System.Text; +using System.Collections.Generic; +using Alphaleonis.Win32.Filesystem; + +namespace Duplicati.Library.Snapshots +{ + /// + /// Handler for providing a snapshot like access to files and folders + /// + public class NoSnapshotWindows : NoSnapshot + { + private SystemIOWindows m_sysIO = new SystemIOWindows(); + + public NoSnapshotWindows(string[] sourcefolders) + : base(sourcefolders) + { + } + + public NoSnapshotWindows(string[] sourcefolders, Dictionary options) + : base(sourcefolders, options) + { + } + + /// + /// Returns the symlink target if the entry is a symlink, and null otherwise + /// + /// The file or folder to examine + /// The symlink target + public override string GetSymlinkTarget(string file) + { + return File.GetLinkTargetInfo(SystemIOWindows.PrefixWithUNC(file)).PrintName; + } + + /// + /// Gets the attributes for the given file or folder + /// + /// The file attributes + /// The file or folder to examine + public override System.IO.FileAttributes GetAttributes (string file) + { + return m_sysIO.GetFileAttributes(file); + } + + /// + /// Returns the size of a file + /// + /// The full path to the file in non-snapshot format + /// The lenth of the file + public override long GetFileSize (string file) + { + return m_sysIO.FileLength(file); + } + + /// + /// Gets the last write time of a given file + /// + /// The full path to the file in non-snapshot format + /// The last write time of the file + public override DateTime GetLastWriteTime (string file) + { + if (!SystemIOWindows.IsPathTooLong(file)) + try { return base.GetLastWriteTime(file); } + catch (System.IO.PathTooLongException) { } + + return File.GetLastWriteTime(SystemIOWindows.PrefixWithUNC(file)); + } + + /// + /// Opens a locked file for reading + /// + /// The full path to the file in non-snapshot format + /// An open filestream that can be read + public override System.IO.Stream OpenLockedRead (string file) + { + if (!SystemIOWindows.IsPathTooLong(file)) + try { return base.OpenLockedRead(file); } + catch (System.IO.PathTooLongException) { } + + return File.Open(SystemIOWindows.PrefixWithUNC(file), FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + } + + /// + /// Opens a file for reading + /// + /// The full path to the file in non-snapshot format + /// An open filestream that can be read + public override System.IO.Stream OpenRead (string file) + { + if (!SystemIOWindows.IsPathTooLong(file)) + try { return base.OpenRead(file); } + catch (System.IO.PathTooLongException) { } + + return File.OpenRead(SystemIOWindows.PrefixWithUNC(file)); + } + + /// + /// Lists all files in the given folder + /// + /// All folders found in the folder + /// The folder to examinate + protected override string[] ListFiles (string folder) + { + if (!SystemIOWindows.IsPathTooLong(folder)) + try { return base.ListFiles(folder); } + catch (System.IO.PathTooLongException) { } + + string[] tmp = Directory.GetFiles(SystemIOWindows.PrefixWithUNC(folder)); + string[] res = new string[tmp.Length]; + for(int i = 0; i < tmp.Length; i++) + res[i] = SystemIOWindows.StripUNCPrefix(tmp[i]); + + return res; + } + + /// + /// Lists all folders in the given folder + /// + /// All folders found in the folder + /// The folder to examinate + protected override string[] ListFolders (string folder) + { + if (!SystemIOWindows.IsPathTooLong(folder)) + try { return base.ListFolders(folder); } + catch (System.IO.PathTooLongException) { } + + string[] tmp = Directory.GetDirectories(SystemIOWindows.PrefixWithUNC(folder)); + string[] res = new string[tmp.Length]; + for (int i = 0; i < tmp.Length; i++) + res[i] = SystemIOWindows.StripUNCPrefix(tmp[i]); + + return res; + } + } +} + diff --git a/Duplicati/Library/Snapshots/SystemIOLinux.cs b/Duplicati/Library/Snapshots/SystemIOLinux.cs new file mode 100644 index 000000000..a5a17556b --- /dev/null +++ b/Duplicati/Library/Snapshots/SystemIOLinux.cs @@ -0,0 +1,110 @@ +// Copyright (C) 2011, Kenneth Skovhede + +// http://www.hexad.dk, opensource@hexad.dk +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +using System; +using System.IO; + +namespace Duplicati.Library.Snapshots +{ + public struct SystemIOLinux : ISystemIO + { + #region ISystemIO implementation + public void DirectoryDelete(string path) + { + Directory.Delete(path); + } + + public void DirectoryCreate(string path) + { + Directory.CreateDirectory(path); + } + + public bool DirectoryExists(string path) + { + return Directory.Exists(path); + } + + public void FileDelete(string path) + { + File.Delete(path); + } + + public void FileSetLastWriteTimeUtc(string path, DateTime time) + { + File.SetLastWriteTimeUtc(path, time); + } + + public bool FileExists(string path) + { + return File.Exists(path); + } + + public Stream FileOpenRead(string path) + { + return File.OpenRead(path); + } + + public Stream FileOpenWrite(string path) + { + return File.OpenWrite(path); + } + + public Stream FileCreate(string path) + { + return File.Create(path); + } + + public FileAttributes GetFileAttributes(string path) + { + return File.GetAttributes(path); + } + + public void CreateSymlink(string symlinkfile, string target, bool asDir) + { + UnixSupport.File.CreateSymlink(symlinkfile, target); + } + public string PathGetDirectoryName(string path) + { + return Path.GetDirectoryName(path); + } + + public void DirectorySetLastWriteTimeUtc(string path, DateTime time) + { + Directory.SetLastWriteTimeUtc(path, time); + } + + public void FileMove(string source, string target) + { + File.Move(source, target); + } + + public long FileLength(string path) + { + return new System.IO.FileInfo(path).Length; + } + + public void DirectoryDelete(string path, bool recursive) + { + Directory.Delete(path, recursive); + } + #endregion + + + } + +} + diff --git a/Duplicati/Library/Snapshots/SystemIOWindows.cs b/Duplicati/Library/Snapshots/SystemIOWindows.cs new file mode 100644 index 000000000..fc6fc9b36 --- /dev/null +++ b/Duplicati/Library/Snapshots/SystemIOWindows.cs @@ -0,0 +1,236 @@ +// Copyright (C) 2011, Kenneth Skovhede + +// http://www.hexad.dk, opensource@hexad.dk +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +using System; + + +namespace Duplicati.Library.Snapshots +{ + public struct SystemIOWindows : ISystemIO + { + private const string UNCPREFIX = @"\\?\"; + + public static bool IsPathTooLong(string path) + { + if (path.StartsWith(UNCPREFIX) || path.Length > 260) + return true; + + return false; + } + + public static string PrefixWithUNC(string path) + { + if (!path.StartsWith(UNCPREFIX)) + return UNCPREFIX + path; + else + return path; + } + + public static string StripUNCPrefix(string path) + { + if (path.StartsWith(UNCPREFIX)) + return path.Substring(UNCPREFIX.Length); + else + return path; + } + + #region ISystemIO implementation + public void DirectoryDelete(string path) + { + if (!IsPathTooLong(path)) + try + { + System.IO.Directory.Delete(path); + return; + } + catch (System.IO.PathTooLongException) { } + + Alphaleonis.Win32.Filesystem.Directory.Delete(PrefixWithUNC(path)); + } + + public void DirectoryCreate(string path) + { + if (!IsPathTooLong(path)) + try + { + System.IO.Directory.CreateDirectory(path); + return; + } + catch (System.IO.PathTooLongException) { } + + Alphaleonis.Win32.Filesystem.Directory.CreateDirectory(PrefixWithUNC(path)); + } + + public bool DirectoryExists(string path) + { + if (!IsPathTooLong(path)) + try { return System.IO.Directory.Exists(path); } + catch (System.IO.PathTooLongException) { } + + return Alphaleonis.Win32.Filesystem.Directory.Exists(PrefixWithUNC(path)); + } + + public void FileDelete(string path) + { + if (!IsPathTooLong(path)) + try + { + System.IO.File.Delete(path); + return; + } + catch (System.IO.PathTooLongException) { } + + Alphaleonis.Win32.Filesystem.File.Delete(PrefixWithUNC(path)); + } + + public void FileSetLastWriteTimeUtc(string path, DateTime time) + { + if (!IsPathTooLong(path)) + try + { + System.IO.File.SetLastWriteTimeUtc(path, time); + return; + } + catch (System.IO.PathTooLongException) { } + + Alphaleonis.Win32.Filesystem.File.SetLastWriteTimeUtc(PrefixWithUNC(path), time); + } + + public bool FileExists(string path) + { + if (!IsPathTooLong(path)) + try { return System.IO.File.Exists(path); } + catch (System.IO.PathTooLongException) { } + + return Alphaleonis.Win32.Filesystem.File.Exists(PrefixWithUNC(path)); + } + + public System.IO.Stream FileOpenRead(string path) + { + if (!IsPathTooLong(path)) + try { return System.IO.File.OpenRead(path); } + catch (System.IO.PathTooLongException) { } + + return Alphaleonis.Win32.Filesystem.File.OpenRead(PrefixWithUNC(path)); + } + + public System.IO.Stream FileOpenWrite(string path) + { + if (!IsPathTooLong(path)) + try { return System.IO.File.OpenWrite(path); } + catch (System.IO.PathTooLongException) { } + + if (FileExists(path)) + return Alphaleonis.Win32.Filesystem.File.OpenWrite(PrefixWithUNC(path)); + else + return FileCreate(path); + } + + public System.IO.Stream FileCreate(string path) + { + if (!IsPathTooLong(path)) + try { return System.IO.File.Create(path); } + catch (System.IO.PathTooLongException) { } + + return Alphaleonis.Win32.Filesystem.File.Create(PrefixWithUNC(path)); + } + + public System.IO.FileAttributes GetFileAttributes(string path) + { + if (!IsPathTooLong(path)) + try { return System.IO.File.GetAttributes(path); } + catch (System.IO.PathTooLongException) { } + + return (System.IO.FileAttributes)Alphaleonis.Win32.Filesystem.File.GetAttributes(PrefixWithUNC(path)); + } + + public void CreateSymlink(string symlinkfile, string target, bool asDir) + { + if (FileExists(symlinkfile) || DirectoryExists(symlinkfile)) + throw new System.IO.IOException(string.Format("File already exists: {0}", symlinkfile)); + Alphaleonis.Win32.Filesystem.File.CreateSymbolicLink(target, PrefixWithUNC(symlinkfile), asDir ? Alphaleonis.Win32.Filesystem.SymbolicLinkTarget.Directory : Alphaleonis.Win32.Filesystem.SymbolicLinkTarget.File); + + //Sadly we do not get a notification if the creation fails :( + System.IO.FileAttributes attr = 0; + if ((!asDir && FileExists(symlinkfile)) || (asDir && DirectoryExists(symlinkfile))) + try { attr = GetFileAttributes(symlinkfile); } + catch { } + + if ((attr & System.IO.FileAttributes.ReparsePoint) == 0) + throw new System.IO.IOException(string.Format("Unable to create symlink, check account permissions: {0}", symlinkfile)); + } + + public string PathGetDirectoryName(string path) + { + if (!IsPathTooLong(path)) + try { return System.IO.Path.GetDirectoryName(path); } + catch (System.IO.PathTooLongException) { } + + return StripUNCPrefix(Alphaleonis.Win32.Filesystem.Path.GetDirectoryName(PrefixWithUNC(path))); + } + + public void DirectorySetLastWriteTimeUtc(string path, DateTime time) + { + if (!IsPathTooLong(path)) + try + { + System.IO.Directory.SetLastWriteTimeUtc(path, time); + return; + } + catch (System.IO.PathTooLongException) { } + + Alphaleonis.Win32.Filesystem.File.SetLastWriteTimeUtc(PrefixWithUNC(path), time); + } + + public void FileMove(string source, string target) + { + if (!IsPathTooLong(source) && !IsPathTooLong(target)) + try + { + System.IO.File.Move(source, target); + return; + } + catch (System.IO.PathTooLongException) { } + + Alphaleonis.Win32.Filesystem.File.Move(PrefixWithUNC(source), PrefixWithUNC(target)); + } + + public long FileLength(string path) + { + if (!IsPathTooLong(path)) + try { return new System.IO.FileInfo(path).Length; } + catch (System.IO.PathTooLongException) { } + + return new Alphaleonis.Win32.Filesystem.FileInfo(PrefixWithUNC(path)).Length; + } + + public void DirectoryDelete(string path, bool recursive) + { + if (!IsPathTooLong(path)) + try + { + System.IO.Directory.Delete(path, recursive); + return; + } + catch (System.IO.PathTooLongException) { } + + Alphaleonis.Win32.Filesystem.Directory.Delete(PrefixWithUNC(path), recursive); + } + #endregion + } +} + diff --git a/Duplicati/Library/Snapshots/USNHelper.cs b/Duplicati/Library/Snapshots/USNHelper.cs index dfbd4d47e..cad697193 100644 --- a/Duplicati/Library/Snapshots/USNHelper.cs +++ b/Duplicati/Library/Snapshots/USNHelper.cs @@ -121,9 +121,8 @@ namespace Duplicati.Library.Snapshots /// Helper function to support the Duplicati enumeration callback system /// /// The root path to look in and use as filter base - /// The filter to apply /// The callback function that collects the output - public void EnumerateFilesAndFolders(string rootpath, Duplicati.Library.Utility.FilenameFilter filter, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) + public void EnumerateFilesAndFolders(string rootpath, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) { //Under normal enumeration, the filter will prevent visiting subfolders for excluded folders //But when using USN all files/folders are present in the list, so we have to maintain @@ -141,9 +140,7 @@ namespace Duplicati.Library.Snapshots if (isFolder) { - if (filter == null || filter.ShouldInclude(rootpath, Utility.Utility.AppendDirSeparator(r.Key))) - callback(rootpath, r.Key, Duplicati.Library.Utility.Utility.EnumeratedFileStatus.Folder); - else + if (!callback(rootpath, r.Key, (System.IO.FileAttributes)((int)r.Value.FileAttributes & 0xff))) { if (local_filter.Length != 0) local_filter.Append("|"); @@ -159,8 +156,8 @@ namespace Duplicati.Library.Snapshots } else { - if (filter == null || filter.ShouldInclude(rootpath, r.Key)) - callback(rootpath, r.Key, Duplicati.Library.Utility.Utility.EnumeratedFileStatus.File); + if (local_filter.Length == 0 || !excludedFolders.IsMatch(r.Key)) + callback(rootpath, r.Key, (System.IO.FileAttributes)((int)r.Value.FileAttributes & 0xff)); } } } diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index 204237fd1..9fb814000 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -178,14 +178,16 @@ namespace Duplicati.Library.Snapshots /// The source folder /// The full entry path /// The entry type - public void callback(string rootpath, string path, Utility.Utility.EnumeratedFileStatus status) + public bool callback(string rootpath, string path, FileAttributes attributes) { - if (status == Duplicati.Library.Utility.Utility.EnumeratedFileStatus.File) - files.Add(path); - else if (status == Duplicati.Library.Utility.Utility.EnumeratedFileStatus.Folder) + if ((attributes & Utility.Utility.ATTRIBUTE_ERROR) != 0) + errors.Add(path); + else if ((attributes & FileAttributes.Directory) != 0) folders.Add(path); else - errors.Add(path); + files.Add(path); + + return true; } } #endif @@ -262,12 +264,12 @@ namespace Duplicati.Library.Snapshots /// The path from which to retrieve files and folders /// The filter to apply when evaluating files and folders /// The callback to invoke with each found path - public void EnumerateFilesAndFolders(string startpath, Duplicati.Library.Utility.FilenameFilter filter, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) + public void EnumerateFilesAndFolders(string startpath, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) { foreach (string s in m_sourcepaths) if (s.Equals(startpath, Utility.Utility.ClientFilenameStringComparision)) { - Utility.Utility.EnumerateFileSystemEntries(s, filter, callback, this.ListFolders, this.ListFiles); + Utility.Utility.EnumerateFileSystemEntries(s, callback, this.ListFolders, this.ListFiles, this.GetAttributes); return; } @@ -279,10 +281,10 @@ namespace Duplicati.Library.Snapshots /// /// The filter to apply when evaluating files and folders /// The callback to invoke with each found path - public void EnumerateFilesAndFolders(Utility.FilenameFilter filter, Utility.Utility.EnumerationCallbackDelegate callback) + public void EnumerateFilesAndFolders(Utility.Utility.EnumerationCallbackDelegate callback) { foreach (string s in m_sourcepaths) - Utility.Utility.EnumerateFileSystemEntries(s, filter, callback, this.ListFolders, this.ListFiles); + Utility.Utility.EnumerateFileSystemEntries(s, callback, this.ListFolders, this.ListFiles, this.GetAttributes); } /// @@ -315,6 +317,25 @@ namespace Duplicati.Library.Snapshots return new Alphaleonis.Win32.Filesystem.FileInfo(GetSnapshotPath(file)).Length; } + /// + /// Gets the attributes for the given file or folder + /// + /// The file attributes + /// The file or folder to examine + public System.IO.FileAttributes GetAttributes(string file) + { + return (System.IO.FileAttributes)Alphaleonis.Win32.Filesystem.File.GetAttributes(GetSnapshotPath(file)); + } + + /// + /// Returns the symlink target if the entry is a symlink, and null otherwise + /// + /// The file or folder to examine + /// The symlink target + public string GetSymlinkTarget(string file) + { + return Alphaleonis.Win32.Filesystem.File.GetLinkTargetInfo(file).PrintName; + } #endregion #region IDisposable Members diff --git a/Duplicati/Library/Utility/BlockingQueue.cs b/Duplicati/Library/Utility/BlockingQueue.cs new file mode 100644 index 000000000..caa9c0431 --- /dev/null +++ b/Duplicati/Library/Utility/BlockingQueue.cs @@ -0,0 +1,229 @@ +#region Disclaimer / License +// Copyright (C) 2011, Kenneth Skovhede +// http://www.hexad.dk, opensource@hexad.dk +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// +#endregion +using System; +using System.Collections.Generic; +using System.Text; + +namespace Duplicati.Library.Utility +{ + /// + /// Class for wrapping a as an + /// + public class BlockingQueueAsEnumerable : IEnumerable + { + private IEnumerator m_enumerator; + + public BlockingQueueAsEnumerable(BlockingQueue queue) + { + m_enumerator = new BlockingQueueEnumerator(queue); + } + + #region IEnumerable implementation + public System.Collections.IEnumerator GetEnumerator() + { + return ((IEnumerable)this).GetEnumerator(); + } + #endregion + + #region IEnumerable implementation + IEnumerator IEnumerable.GetEnumerator() + { + if (m_enumerator == null) + throw new Exception("Can only return a single enumerator for a blocking queue"); + + IEnumerator tmp = m_enumerator; + m_enumerator = null; + return tmp; + } + #endregion + + private class BlockingQueueEnumerator : IEnumerator + { + private BlockingQueue m_queue; + private bool m_first = true; + private T m_current; + + public BlockingQueueEnumerator(BlockingQueue queue) + { + m_queue = queue; + } + + #region IDisposable implementation + public void Dispose() + { + } + #endregion + + #region IEnumerator implementation + public bool MoveNext() + { + m_current = m_queue.Dequeue(); + return !(object.Equals(m_current, default(T)) && m_queue.Completed); + + } + + public void Reset() + { + if (!m_first) + throw new System.NotImplementedException(); + } + + public object Current + { + get + { + return m_current; + } + } + #endregion + + #region IEnumerator implementation + T IEnumerator.Current + { + get + { + return m_current; + } + } + #endregion + } + } + + /// + /// Implementation of a blocking queue that serves as a producer/consumer contact point + /// + /// The type of data in the queue + public class BlockingQueue + { + /// + /// The lock that protects the data structures + /// + private object m_lock = new object(); + /// + /// The queue storing the elements produced + /// + private Queue m_queue = new Queue(); + /// + /// A flag indicating if the queue is now empty forever + /// + private bool m_completed = false; + /// + /// The event that signals that items have been produced (consumers wait for this) + /// + private System.Threading.ManualResetEvent m_itemsProduced = new System.Threading.ManualResetEvent(false); + /// + /// The event that signals that items have been consumed (producers wait for this) + /// + private System.Threading.ManualResetEvent m_itemsConsumed = new System.Threading.ManualResetEvent(false); + /// + /// The maximum capacity of the queue, the producers will be blocked if more elements are put into the queue + /// + private long m_maxCapacity = 100; + + /// + /// Initializes a new instance of the class with the default capacity + /// + public BlockingQueue() + { + } + + /// + /// Gets a value indicating whether this is completed + /// + public bool Completed { get { return m_completed; } } + + /// + /// Initializes a new instance of the class + /// + /// The maximum capacity of the queue, the producers will be blocked if more elements are put into the queue + public BlockingQueue(long maxCapacity) + : this() + { + if (maxCapacity <= 0) + throw new ArgumentException("maxCapacity", "maxCapacity must be a positive number"); + + m_maxCapacity = maxCapacity; + } + + /// + /// Signals that the producers are done and no more elements will be put into the queue + /// + public void SetCompleted() + { + m_completed = true; + m_itemsProduced.Set(); + m_itemsConsumed.Set(); + } + + /// + /// Enqueues the specified item. + /// + /// The item to put into the queue + public void Enqueue(T item) + { + while (true) + { + lock(m_lock) + { + if (m_completed) + throw new InvalidOperationException("Cannot insert items after complete flag has been set"); + + if (m_queue.Count < m_maxCapacity) + { + m_queue.Enqueue(item); + m_itemsProduced.Set(); + return; + } + + m_itemsConsumed.Reset(); + } + m_itemsConsumed.WaitOne(); + } + } + + /// + /// Dequeues an element from the queue + /// + /// The dequed element or default(T) if there are no more elements + public T Dequeue() + { + while(true) + { + lock(m_lock) + { + if (m_queue.Count > 0) + { + m_itemsConsumed.Reset(); + return m_queue.Dequeue(); + } + + if (m_completed) + return default(T); + + m_itemsProduced.Reset(); + } + + m_itemsProduced.WaitOne(); + } + } + + } +} + diff --git a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj index e600e5352..848a4d43f 100644 --- a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj +++ b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj @@ -113,6 +113,7 @@ + diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 8b3ac7dbc..b8efb69a1 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -46,22 +46,18 @@ namespace Duplicati.Library.Utility public static readonly DateTime EPOCH = new DateTime(1970, 1, 1, 0, 0, 0); /// - /// An enumeration used to identify file types when performing file listings + /// The attribute value used to indicate error /// - public enum EnumeratedFileStatus - { - File, - Folder, - Error - }; + public const System.IO.FileAttributes ATTRIBUTE_ERROR = (System.IO.FileAttributes)(1 << 30); /// /// The callback delegate type used to collecting file information /// /// The path that the file enumeration started at /// The current element - /// The type of the element - public delegate void EnumerationCallbackDelegate(string rootpath, string path, EnumeratedFileStatus status); + /// The attributes of the element + /// A value indicating if the folder should be recursed, ignored for other types + public delegate bool EnumerationCallbackDelegate(string rootpath, string path, System.IO.FileAttributes attributes); /// /// Copies the content of one stream into another @@ -92,11 +88,63 @@ namespace Duplicati.Library.Utility target.Write(buf, 0, read); } + /// + /// Returns an IEnumerable with all filesystem entries, recursive + /// + /// The folder to look in + /// An optional filter to apply to the filenames + /// The function to call with the filenames + /// A function to call that lists all folders in the supplied folder + /// A function to call that lists all files in the supplied folder + /// A function to call that obtains the attributes for an element, set to null to skip symlink detection + /// The file system entries + public static IEnumerable EnumerateFileSystemEntriesBlocked(string rootpath, FilenameFilter filter, FileSystemInteraction folderList, FileSystemInteraction fileList, ExtractFileAttributes attributeReader) + { + BlockingQueue queue = new BlockingQueue(); + new BlockedCollector(queue, rootpath, filter, folderList, fileList); + return new BlockingQueueAsEnumerable(queue); + } + + /// + /// Class used to implement blocked collection of files and folders + /// + private class BlockedCollector + { + private BlockingQueue m_queue; + public BlockedCollector(BlockingQueue queue, string rootpath, FilenameFilter filter, FileSystemInteraction folderList, FileSystemInteraction fileList) + { + m_queue = queue; + System.Threading.Thread t = new System.Threading.Thread(BlockedThreadRunner); + t.Start(new object[] { rootpath, filter, folderList, fileList}); + } + + private bool Callback(string rootpath, string path, System.IO.FileAttributes attributes) + { + if ((attributes & ATTRIBUTE_ERROR) == 0) + m_queue.Enqueue(path); + + return true; + } + + private void BlockedThreadRunner(object _args) + { + object[] args = (object[])_args; + string roothpath = (string)args[0]; + FilenameFilter filter = (FilenameFilter)args[1]; + FileSystemInteraction folderList = (FileSystemInteraction)args[2]; + FileSystemInteraction fileList = (FileSystemInteraction)args[3]; + + EnumerateFileSystemEntries(roothpath, Callback, folderList, fileList); + m_queue.SetCompleted(); + } + } + /// /// Returns a list of all files found in the given folder. /// The search is recursive. /// /// The folder to look in + /// The filter to apply. /// A list of the full filenames public static List EnumerateFiles(string basepath) { @@ -114,6 +162,59 @@ namespace Duplicati.Library.Utility return EnumerateFolders(basepath, null); } + /// + /// Returns a list of all files and subfolders found in the given folder. + /// The search is recursive. + /// + /// The folder to look in. + /// A list of the full filenames and foldernames. Foldernames ends with the directoryseparator char + public static List EnumerateFileSystemEntries(string basepath) + { + return EnumerateFileSystemEntries(basepath, (FilenameFilter)null); + } + + /// + /// Returns a list of all files found in the given folder. + /// The search is recursive. + /// + /// The folder to look in + /// The filter to apply. + /// A list of the full filenames + public static List EnumerateFiles(string basepath, FilenameFilter filter) + { + PathCollector c = new PathCollector(false, true, filter); + EnumerateFileSystemEntries(basepath, new EnumerationCallbackDelegate(c.Callback)); + return c.Files; + } + + /// + /// Returns a list of folder names found in the given folder. + /// The search is recursive. + /// + /// The folder to look in + /// The filter to apply. + /// A list of the full paths + public static List EnumerateFolders(string basepath, FilenameFilter filter) + { + PathCollector c = new PathCollector(true, false, filter); + EnumerateFileSystemEntries(basepath, new EnumerationCallbackDelegate(c.Callback)); + return c.Files; + } + + /// + /// Returns a list of all files and subfolders found in the given folder. + /// The search is recursive. + /// + /// The folder to look in. + /// The filter to apply. + /// A list of the full filenames and foldernames. Foldernames ends with the directoryseparator char + public static List EnumerateFileSystemEntries(string basepath, FilenameFilter filter) + { + PathCollector c = new PathCollector(true, true, filter); + EnumerateFileSystemEntries(basepath, new EnumerationCallbackDelegate(c.Callback)); + return c.Files; + } + /// /// A callback delegate used for applying alternate enumeration of filesystems /// @@ -122,17 +223,11 @@ namespace Duplicati.Library.Utility public delegate string[] FileSystemInteraction(string path); /// - /// Returns a list of all files found in the given folder. - /// The search is recursive. + /// A callback delegate used for extracting attributes from a file or folder /// - /// The folder to look in - /// An optional filter to apply to the filenames - /// The function to call with the filenames - /// A list of the full filenames - public static void EnumerateFileSystemEntries(string rootpath, FilenameFilter filter, EnumerationCallbackDelegate callback) - { - EnumerateFileSystemEntries(rootpath, filter, callback, new FileSystemInteraction(System.IO.Directory.GetDirectories), new FileSystemInteraction(System.IO.Directory.GetFiles)); - } + /// The path to return data from + /// Attributes for the file or folder + public delegate System.IO.FileAttributes ExtractFileAttributes(string path); /// /// Returns a list of all files found in the given folder. @@ -141,10 +236,38 @@ namespace Duplicati.Library.Utility /// The folder to look in /// An optional filter to apply to the filenames /// The function to call with the filenames + /// A list of the full filenames + public static void EnumerateFileSystemEntries(string rootpath, EnumerationCallbackDelegate callback) + { + EnumerateFileSystemEntries(rootpath, callback, new FileSystemInteraction(System.IO.Directory.GetDirectories), new FileSystemInteraction(System.IO.Directory.GetFiles)); + } + + /// + /// Returns a list of all files found in the given folder. + /// The search is recursive. + /// + /// The folder to look in + /// The function to call with the filenames /// A function to call that lists all folders in the supplied folder /// A function to call that lists all files in the supplied folder /// A list of the full filenames - public static void EnumerateFileSystemEntries(string rootpath, FilenameFilter filter, EnumerationCallbackDelegate callback, FileSystemInteraction folderList, FileSystemInteraction fileList) + public static void EnumerateFileSystemEntries(string rootpath, EnumerationCallbackDelegate callback, FileSystemInteraction folderList, FileSystemInteraction fileList) + { + + EnumerateFileSystemEntries(rootpath, callback, folderList, fileList, null); + } + + /// + /// Returns a list of all files found in the given folder. + /// The search is recursive. + /// + /// The folder to look in + /// The function to call with the filenames + /// A function to call that lists all folders in the supplied folder + /// A function to call that lists all files in the supplied folder + /// A function to call that obtains the attributes for an element, set to null to avoid reading attributes + /// A list of the full filenames + public static void EnumerateFileSystemEntries(string rootpath, EnumerationCallbackDelegate callback, FileSystemInteraction folderList, FileSystemInteraction fileList, ExtractFileAttributes attributeReader) { if (!System.IO.Directory.Exists(rootpath)) return; @@ -157,11 +280,12 @@ namespace Duplicati.Library.Utility string f = AppendDirSeparator(lst.Dequeue()); try { - foreach (string s in folderList(f)) - if (filter == null || filter.ShouldInclude(rootpath, AppendDirSeparator(s))) - lst.Enqueue(s); + System.IO.FileAttributes attr = attributeReader == null ? System.IO.FileAttributes.Directory : attributeReader(f); + if (!callback(rootpath, f, attr)) + continue; - callback(rootpath, f, EnumeratedFileStatus.Folder); + foreach (string s in folderList(f)) + lst.Enqueue(s); } catch (System.Threading.ThreadAbortException) { @@ -169,14 +293,28 @@ namespace Duplicati.Library.Utility } catch (Exception) { - callback(rootpath, f, EnumeratedFileStatus.Error); + callback(rootpath, f, ATTRIBUTE_ERROR | System.IO.FileAttributes.Directory); } try { - foreach (string s in fileList(f)) - if (filter == null || filter.ShouldInclude(rootpath, s)) - callback(rootpath, s, EnumeratedFileStatus.File); + if (fileList != null) + foreach (string s in fileList(f)) + { + try + { + System.IO.FileAttributes attr = attributeReader == null ? System.IO.FileAttributes.Normal : attributeReader(s); + callback(rootpath, s, attr); + } + catch (System.Threading.ThreadAbortException) + { + throw; + } + catch (Exception) + { + callback(rootpath, s, ATTRIBUTE_ERROR); + } + } } catch (System.Threading.ThreadAbortException) { @@ -184,9 +322,8 @@ namespace Duplicati.Library.Utility } catch (Exception) { - callback(rootpath, f, EnumeratedFileStatus.Error); + callback(rootpath, f, ATTRIBUTE_ERROR); } - } } @@ -196,22 +333,37 @@ namespace Duplicati.Library.Utility private class PathCollector { private List m_files; + private FilenameFilter m_filter; private bool m_includeFolders; private bool m_includeFiles; - public PathCollector(bool includeFolders, bool includeFiles) + public PathCollector(bool includeFolders, bool includeFiles, FilenameFilter filter) { m_files = new List(); + m_filter = filter; m_includeFolders = includeFolders; m_includeFiles = includeFiles; } - public void Callback(string rootpath, string path, EnumeratedFileStatus status) + public bool Callback(string rootpath, string path, System.IO.FileAttributes attributes) { - if (m_includeFolders && status == EnumeratedFileStatus.Folder) - m_files.Add(path); - else if (m_includeFiles && status == EnumeratedFileStatus.File) - m_files.Add(path); + if ((attributes & ATTRIBUTE_ERROR) != 0) + { + //Don't care + } + else + { + if (m_filter != null && !m_filter.ShouldInclude(rootpath, path)) + return false; + + bool isDirectory = (attributes & System.IO.FileAttributes.Directory) != 0; + if (m_includeFolders && isDirectory) + m_files.Add(path); + else if (m_includeFiles && !isDirectory) + m_files.Add(path); + } + + return true; } public List Files { get { return m_files; } } @@ -223,12 +375,23 @@ namespace Duplicati.Library.Utility private class PathSizeCalculator { private long m_size = 0; + private FilenameFilter m_filter; - public void Callback(string rootpath, string path, EnumeratedFileStatus status) + public PathSizeCalculator(FilenameFilter filter) { - if (status == EnumeratedFileStatus.File) + m_filter = filter; + } + + public bool Callback(string rootpath, string path, System.IO.FileAttributes attributes) + { + if (m_filter != null && !m_filter.ShouldInclude(rootpath, path)) + return false; + + if ((attributes & System.IO.FileAttributes.Directory) == 0) try { m_size += new System.IO.FileInfo(path).Length; } catch { } + + return true; } public long Size { get { return m_size; } } @@ -242,64 +405,11 @@ namespace Duplicati.Library.Utility /// The combined size of all files that match the filter public static long GetDirectorySize(string folder, FilenameFilter filter) { - PathSizeCalculator c = new PathSizeCalculator(); - EnumerateFileSystemEntries(folder, filter, new EnumerationCallbackDelegate(c.Callback)); + PathSizeCalculator c = new PathSizeCalculator(filter); + EnumerateFileSystemEntries(folder, new EnumerationCallbackDelegate(c.Callback)); return c.Size; } - /// - /// Returns a list of all files found in the given folder. - /// The search is recursive. - /// - /// The folder to look in - /// An optional filter to apply to the filenames - /// A list of the full filenames - public static List EnumerateFiles(string basepath, FilenameFilter filter) - { - PathCollector c = new PathCollector(false, true); - EnumerateFileSystemEntries(basepath, filter, new EnumerationCallbackDelegate(c.Callback)); - return c.Files; - } - - /// - /// Returns a list of all files and subfolders found in the given folder. - /// The search is recursive. - /// - /// The folder to look in. - /// A list of the full filenames and foldernames. Foldernames ends with the directoryseparator char - public static List EnumerateFileSystemEntries(string basepath) - { - return EnumerateFileSystemEntries(basepath, null); - } - - /// - /// Returns a list of all files and subfolders found in the given folder. - /// The search is recursive. - /// - /// The folder to look in. - /// An optional filter to apply to the filenames - /// A list of the full filenames and foldernames. Foldernames ends with the directoryseparator char - public static List EnumerateFileSystemEntries(string basepath, FilenameFilter filter) - { - PathCollector c = new PathCollector(true, true); - EnumerateFileSystemEntries(basepath, filter, new EnumerationCallbackDelegate(c.Callback)); - return c.Files; - } - - /// - /// Returns a list of folder names found in the given folder. - /// The search is recursive. - /// - /// The folder to look in - /// An optional filter to apply to the folder names - /// A list of the full paths - public static List EnumerateFolders(string basepath, FilenameFilter filter) - { - PathCollector c = new PathCollector(true, false); - EnumerateFileSystemEntries(basepath, filter, new EnumerationCallbackDelegate(c.Callback)); - return c.Files; - } - /// /// A cached instance of the directory separator as a string /// diff --git a/thirdparty/UnixSupport/File.cs b/thirdparty/UnixSupport/File.cs index 46a968283..caa2ae30f 100644 --- a/thirdparty/UnixSupport/File.cs +++ b/thirdparty/UnixSupport/File.cs @@ -9,7 +9,7 @@ namespace UnixSupport /// Opens the file and honors advisory locking. /// /// A open stream that references the file - /// The full path to the file + /// The full path to the file public static System.IO.Stream OpenExclusive(string path, System.IO.FileAccess mode) { return OpenExclusive(path, mode, (int)Mono.Unix.Native.FilePermissions.DEFFILEMODE); @@ -80,6 +80,31 @@ namespace UnixSupport { } } + + /// + /// Gets the symlink target for the given path + /// + /// The path to get the symlink target for + /// The symlink target + public static string GetSymlinkTarget(string path) + { + System.Text.StringBuilder sb = new System.Text.StringBuilder(2048); //2kb, should cover utf16 * 1023 chars + if (Mono.Unix.Native.Syscall.readlink(path, sb, (ulong)sb.Capacity) >= 0) + return sb.ToString(); + + throw new System.IO.FileLoadException(string.Format("Unable to get symlink for \"{0}\", error: {1} ({2})", path, Syscall.GetLastError(), (int)Syscall.GetLastError())); + } + + /// + /// Creates a new symlink + /// + /// The path to create the symbolic link entry + /// The path the symbolic link points to + public static void CreateSymlink(string path, string target) + { + if (Mono.Unix.Native.Syscall.symlink(target, path) != 0) + throw new System.IO.IOException(string.Format("Unable to create symlink from \"{0}\" to \"{1}\", error: {2} ({3})", path, target, Syscall.GetLastError(), (int)Syscall.GetLastError())); + } } } diff --git a/thirdparty/UnixSupport/UnixSupport.dll b/thirdparty/UnixSupport/UnixSupport.dll index 64701577a..022fb2b8f 100755 Binary files a/thirdparty/UnixSupport/UnixSupport.dll and b/thirdparty/UnixSupport/UnixSupport.dll differ