From b277f6325db1833a7643d03f2111f3ea5e6509cf Mon Sep 17 00:00:00 2001 From: "kenneth@hexad.dk" Date: Sun, 29 Jul 2012 20:07:10 +0000 Subject: [PATCH] Fairly large rewrite of how files and paths are handled. This fixes issues #144, #577 and #320. This update introduces a new options called "symlink-policy" that can be set to "store", "ignore" or "follow". The default setting "store" means all symlinks are recorded as symlinks and recreated as symlinks on restore. This is chosen as default because it is the best replication of the actual disk contents. Setting it to "ignore" simply means that no information is recorded about any symlinks. And lastly, setting it to "follow" reverts to the previous default of treating symlinks as if they are normal files/folders. Caveat: On Windows, you need Administrator privileges to create symlinks (when restoring), but not for reading them (on backup). Since this change meant that the file attributes must be read, I also added support for the option "exclude-files-attributes", that can be used to ignore/exclude files with certain attributes, such as "hidden", "archive" or "encrypted". Since I had to much about so much with getting the symlinks supported on Windows, I also added some extra logic to the Windows file handler, so it can now handle files and folders with up to 32k chars in the path. The logic is done so that the normal System.IO operations are used for paths with less than 260 characters (MAX_PATH), and on error or longer paths, the Win32 Unicode API is invoked. This should give maximal backwards compatibility and still allow backup and restore of files with long paths. This commit also includes a BlockingQueue implementation that is supposed to make file/folder enumeration faster, but this has not been implemented because Duplicati depends on having a "whole world" view of the files and folders before running the backup (i.e. deeper changes required for this). git-svn-id: https://duplicati.googlecode.com/svn/trunk@1381 59da171f-624f-0410-aa54-27559c288bec --- .../GUI/HelperControls/TestSearchSelection.cs | 21 +- .../Main/Duplicati.Library.Main.csproj | 2 +- Duplicati/Library/Main/Options.cs | 67 ++++ Duplicati/Library/Main/RSyncDir.cs | 294 +++++++++++++---- .../Library/Main/Strings/Options.Designer.cs | 37 +++ Duplicati/Library/Main/Strings/Options.resx | 13 + .../Library/Main/Strings/RSyncDir.Designer.cs | 11 +- Duplicati/Library/Main/Strings/RSyncDir.resx | 3 + .../Duplicati.Library.Snapshots.csproj | 8 + .../Library/Snapshots/ISnapshotService.cs | 20 +- Duplicati/Library/Snapshots/ISystemIO.cs | 47 +++ Duplicati/Library/Snapshots/LinuxSnapshot.cs | 28 +- Duplicati/Library/Snapshots/NoSnapshot.cs | 72 ++++- .../Library/Snapshots/NoSnapshotLinux.cs | 53 ++++ .../Library/Snapshots/NoSnapshotWindows.cs | 153 +++++++++ Duplicati/Library/Snapshots/SystemIOLinux.cs | 110 +++++++ .../Library/Snapshots/SystemIOWindows.cs | 236 ++++++++++++++ Duplicati/Library/Snapshots/USNHelper.cs | 11 +- .../Library/Snapshots/WindowsSnapshot.cs | 39 ++- Duplicati/Library/Utility/BlockingQueue.cs | 229 ++++++++++++++ .../Utility/Duplicati.Library.Utility.csproj | 1 + Duplicati/Library/Utility/Utility.cs | 296 ++++++++++++------ thirdparty/UnixSupport/File.cs | 27 +- thirdparty/UnixSupport/UnixSupport.dll | Bin 4608 -> 5120 bytes 24 files changed, 1583 insertions(+), 195 deletions(-) create mode 100644 Duplicati/Library/Snapshots/ISystemIO.cs create mode 100644 Duplicati/Library/Snapshots/NoSnapshotLinux.cs create mode 100644 Duplicati/Library/Snapshots/NoSnapshotWindows.cs create mode 100644 Duplicati/Library/Snapshots/SystemIOLinux.cs create mode 100644 Duplicati/Library/Snapshots/SystemIOWindows.cs create mode 100644 Duplicati/Library/Utility/BlockingQueue.cs diff --git a/Duplicati/GUI/HelperControls/TestSearchSelection.cs b/Duplicati/GUI/HelperControls/TestSearchSelection.cs index fb54f313e..2c93e5a2a 100644 --- a/Duplicati/GUI/HelperControls/TestSearchSelection.cs +++ b/Duplicati/GUI/HelperControls/TestSearchSelection.cs @@ -219,11 +219,11 @@ namespace Duplicati.GUI.HelperControls { try { - CallbackHandler handler = new CallbackHandler(scanWorker); + CallbackHandler handler = new CallbackHandler(scanWorker, fn); foreach (string scanDir in Paths) { if (!scanWorker.CancellationPending) - Library.Utility.Utility.EnumerateFileSystemEntries(scanDir, fn, handler.AddFileToList_Callback); + Library.Utility.Utility.EnumerateFileSystemEntries(scanDir, handler.AddFileToList_Callback); } return handler.Results; @@ -239,26 +239,29 @@ namespace Duplicati.GUI.HelperControls { { public List Results = new List(); private BackgroundWorker Worker; + private FilenameFilter Filter; - public CallbackHandler(BackgroundWorker worker) + public CallbackHandler(BackgroundWorker worker, FilenameFilter filter) { Worker = worker; + Filter = filter; } - public void AddFileToList_Callback(string rootpath, string path, Duplicati.Library.Utility.Utility.EnumeratedFileStatus status) + public bool AddFileToList_Callback(string rootpath, string path, FileAttributes attributes) { if (Worker.CancellationPending) throw new AbortException(); - if (status == Utility.EnumeratedFileStatus.File) + if (!Filter.ShouldInclude(rootpath, path)) + return false; + + if ((attributes & FileAttributes.Directory) == 0 && (attributes & Utility.ATTRIBUTE_ERROR) == 0) { Results.Add(new ResultItem(path)); } - else if (status == Utility.EnumeratedFileStatus.Folder) - { - //Should be displayed in tree so the user can exclude with right click? - } + + return true; } } diff --git a/Duplicati/Library/Main/Duplicati.Library.Main.csproj b/Duplicati/Library/Main/Duplicati.Library.Main.csproj index e0afaf590..456508777 100644 --- a/Duplicati/Library/Main/Duplicati.Library.Main.csproj +++ b/Duplicati/Library/Main/Duplicati.Library.Main.csproj @@ -1,4 +1,4 @@ - + Debug diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index a29a0109e..e420d95a7 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -74,6 +74,27 @@ namespace Duplicati.Library.Main Copy } + /// + /// The possible settings for the symlink strategy + /// + public enum SymlinkStrategy + { + /// + /// Store information about the symlink + /// + Store, + + /// + /// Treat symlinks as normal files or folders + /// + Follow, + + /// + /// Ignore all symlinks + /// + Ignore + } + /// /// Lock that protects the options collection /// @@ -150,6 +171,8 @@ namespace Duplicati.Library.Main "usn-policy", "open-file-policy", "exclude-empty-folders", + "symlink-policy", + "exclude-files-attributes" }; } } @@ -368,6 +391,9 @@ namespace Duplicati.Library.Main new CommandLineArgument("backend-log-database", CommandLineArgument.ArgumentType.Path, Strings.Options.BackendlogdatabaseShort, Strings.Options.BackendlogdatabaseLong), new CommandLineArgument("quota-size", CommandLineArgument.ArgumentType.Size, Strings.Options.QuotasizeShort, Strings.Options.QuotasizeLong), + + new CommandLineArgument("symlink-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.SymlinkpolicyShort, string.Format(Strings.Options.SymlinkpolicyLong, "store", "ignore", "follow"), "store", null, Enum.GetNames(typeof(SymlinkStrategy))), + new CommandLineArgument("exclude-files-attributes", CommandLineArgument.ArgumentType.String, Strings.Options.ExcludefilesattributesShort, string.Format(Strings.Options.ExcludefilesattributesLong, string.Join(", ", Enum.GetNames(typeof(System.IO.FileAttributes))))), }); } } @@ -1006,6 +1032,24 @@ namespace Duplicati.Library.Main } } + /// + /// Gets the snapshot strategy to use + /// + public SymlinkStrategy SymlinkPolicy + { + get + { + string strategy; + if (!m_options.TryGetValue("symlink-policy", out strategy)) + strategy = ""; + + SymlinkStrategy r; + if (!Enum.TryParse(strategy, true, out r)) + r = SymlinkStrategy.Store; + + return r; + } + } /// /// Gets the snapshot strategy to use /// @@ -1144,6 +1188,29 @@ namespace Duplicati.Library.Main } } + /// + /// Gets the attribute filter used to exclude files and folders. + /// + public System.IO.FileAttributes FileAttributeFilter + { + get + { + System.IO.FileAttributes res = (System.IO.FileAttributes)0; + string v; + if (!m_options.TryGetValue("", out v)) + return res; + + foreach(string s in v.Split(new string[] {","}, StringSplitOptions.RemoveEmptyEntries)) + { + System.IO.FileAttributes f; + if (Enum.TryParse(s.Trim(), true, out f)) + res |= f; + } + + return res; + } + } + /// /// A value indicating if a verification file is placed on the server /// diff --git a/Duplicati/Library/Main/RSyncDir.cs b/Duplicati/Library/Main/RSyncDir.cs index baf702411..4bf0013ce 100644 --- a/Duplicati/Library/Main/RSyncDir.cs +++ b/Duplicati/Library/Main/RSyncDir.cs @@ -439,14 +439,15 @@ namespace Duplicati.Library.Main.RSync #endregion #region Filenames - internal static readonly string COMBINED_SIGNATURE_ROOT = "signature"; - internal static readonly string CONTENT_SIGNATURE_ROOT = "content_signature"; - internal static readonly string DELTA_SIGNATURE_ROOT = "delta_signature"; + internal const string COMBINED_SIGNATURE_ROOT = "signature"; + internal const string CONTENT_SIGNATURE_ROOT = "content_signature"; + internal const string DELTA_SIGNATURE_ROOT = "delta_signature"; - internal static readonly string CONTENT_ROOT = "snapshot"; - internal static readonly string DELTA_ROOT = "diff"; - //private static readonly string DACL_ROOT = "dacl"; - internal static readonly string CONTROL_ROOT = "controlfiles"; + internal const string CONTENT_ROOT = "snapshot"; + internal const string DELTA_ROOT = "diff"; + //private const string DACL_ROOT = "dacl"; + internal const string CONTROL_ROOT = "controlfiles"; + internal const string SYMLINK_ROOT = "symlinks"; internal static readonly string DELETED_FILES = "deleted_files.txt"; internal static readonly string DELETED_FOLDERS = "deleted_folders.txt"; @@ -481,6 +482,10 @@ namespace Duplicati.Library.Main.RSync /// This is a list of existing folders. /// private Dictionary m_oldFolders; + /// + /// This is a list of old symlinks + /// + private Dictionary m_oldSymlinks; /// /// This is the list of added files @@ -672,6 +677,7 @@ namespace Duplicati.Library.Main.RSync { m_oldSignatures.Remove(s); m_lastVerificationTime.Remove(s); + m_oldSymlinks.Remove(s); } foreach (string prefix in prefixes) @@ -685,6 +691,13 @@ namespace Duplicati.Library.Main.RSync } } + string symlinkprefix = Utility.Utility.AppendDirSeparator(SYMLINK_ROOT); + foreach(string s in FilenamesFromPlatformIndependant(patch.Value.ListFiles(symlinkprefix))) + { + string tmp = FilenamesFromPlatformIndependant( new string[] { Encoding.UTF8.GetString(patch.Value.ReadAllBytes(s)) })[0]; + m_oldSymlinks[s.Substring(symlinkprefix.Length)] = tmp; + } + if (z.FileExists(UNMODIFIED_FILES)) foreach (string s in FilenamesFromPlatformIndependant(z.ReadAllLines(UNMODIFIED_FILES))) m_lastVerificationTime[s] = patch.Key.Time.ToUniversalTime(); @@ -745,6 +758,7 @@ namespace Duplicati.Library.Main.RSync m_metadata = metadata == null ? new Backupmetadata() : metadata; m_oldSignatures = new Dictionary(Utility.Utility.ClientFilenameStringComparer); m_oldFolders = new Dictionary(Utility.Utility.ClientFilenameStringComparer); + m_oldSymlinks = new Dictionary(Utility.Utility.ClientFilenameStringComparer); m_lastVerificationTime = new Dictionary(Utility.Utility.ClientFilenameStringComparer); for (int i = 0; i < sourcefolder.Length; i++) @@ -793,6 +807,7 @@ namespace Duplicati.Library.Main.RSync { m_oldFolders = new Dictionary(); m_oldSignatures = new Dictionary(); + m_oldSymlinks = new Dictionary(); } m_newfiles = new Dictionary(); @@ -824,13 +839,16 @@ namespace Duplicati.Library.Main.RSync //Failsafe, just use a plain implementation if (m_snapshot == null) { - m_snapshot = new Duplicati.Library.Snapshots.NoSnapshot(m_sourcefolder, options.RawOptions); + m_snapshot = Utility.Utility.IsClientLinux ? + (Library.Snapshots.ISnapshotService)new Duplicati.Library.Snapshots.NoSnapshotLinux(m_sourcefolder, options.RawOptions) + : + (Library.Snapshots.ISnapshotService)new Duplicati.Library.Snapshots.NoSnapshotWindows(m_sourcefolder, options.RawOptions); m_openfilepolicy = options.OpenFilePolicy; } Dictionary usnHelpers = null; List unchanged = new List(); - m_unproccesed = new PathCollector(); + m_unproccesed = new PathCollector(m_snapshot, options.SymlinkPolicy, options.FileAttributeFilter, m_filter, m_stat); try { @@ -904,7 +922,7 @@ namespace Duplicati.Library.Main.RSync } else */ { - usnHelpers[rootFolder].EnumerateFilesAndFolders(s, m_filter, m_unproccesed.Callback); + usnHelpers[rootFolder].EnumerateFilesAndFolders(s, m_unproccesed.Callback); } } @@ -930,7 +948,7 @@ namespace Duplicati.Library.Main.RSync //If we get here, something went really wrong with USN, so we disable it m_currentUSN = null; - m_unproccesed = new PathCollector(); + m_unproccesed = new PathCollector(m_snapshot, options.SymlinkPolicy, options.FileAttributeFilter, m_filter, m_stat); unchanged = null; } finally @@ -948,7 +966,7 @@ namespace Duplicati.Library.Main.RSync if (m_currentUSN == null) { - m_snapshot.EnumerateFilesAndFolders(m_filter, m_unproccesed.Callback); + m_snapshot.EnumerateFilesAndFolders(m_unproccesed.Callback); } else { @@ -958,12 +976,13 @@ namespace Duplicati.Library.Main.RSync string relpath = GetRelativeName(x); m_oldSignatures.Remove(relpath); m_oldFolders.Remove(relpath); + m_oldSymlinks.Remove(relpath); } //If some folders did not support USN, add their files now foreach (string s in m_sourcefolder) if (!m_currentUSN.ContainsKey(System.IO.Path.GetPathRoot(s))) - m_snapshot.EnumerateFilesAndFolders(s, m_filter, m_unproccesed.Callback); + m_snapshot.EnumerateFilesAndFolders(s, m_unproccesed.Callback); } m_totalfiles = m_unproccesed.Files.Count; @@ -1023,7 +1042,7 @@ namespace Duplicati.Library.Main.RSync string relpath = GetRelativeName(s); if (relpath.Trim().Length != 0) { - DateTime lastWrite = Directory.GetLastWriteTimeUtc(s); + DateTime lastWrite = m_snapshot.GetLastWriteTime(s).ToUniversalTime(); //Cut off as we only have seconds stored lastWrite = new DateTime(lastWrite.Year, lastWrite.Month, lastWrite.Day, lastWrite.Hour, lastWrite.Minute, lastWrite.Second, DateTimeKind.Utc); @@ -1050,6 +1069,39 @@ namespace Duplicati.Library.Main.RSync if (!m_unproccesed.IsAffectedByError(s)) m_deletedfolders.Add(s); + //Build symlink diffs + if (m_oldSymlinks.Count > 0) + { + for (int i = m_unproccesed.Symlinks.Count - 1; i >= 0; i--) + { + string s = m_unproccesed.Symlinks[i].Key; + try + { + string relpath = GetRelativeName(s); + if (relpath.Trim().Length != 0) + { + string oldLink; + if (m_oldSymlinks.TryGetValue(relpath, out oldLink)) + { + m_oldSymlinks.Remove(relpath); + if (string.Equals(oldLink, m_unproccesed.Symlinks[i].Value, Utility.Utility.ClientFilenameStringComparision)) + { + m_unproccesed.Symlinks.RemoveAt(i); + } + } + } + } + catch (Exception ex) + { + m_unproccesed.Errors.Add(s); + m_stat.LogError(string.Format(Strings.RSyncDir.SymlinkReadError, s, ex.Message), ex); + } + } + } + + m_deletedfiles.AddRange(m_oldSymlinks.Keys); + m_oldSymlinks.Clear(); + m_sortedfilelist = options.SortedFilelist; if (m_sortedfilelist) m_unproccesed.Files.Sort(Utility.Utility.ClientFilenameStringComparer); @@ -1173,7 +1225,7 @@ namespace Duplicati.Library.Main.RSync if (!m_unproccesed.IsAffectedByError(fullpath)) { m_deletedfiles.Add(s); - stringsize += System.Text.Encoding.UTF8.GetByteCount(s + Environment.NewLine); + stringsize += System.Text.Encoding.UTF8.GetByteCount(s + "\r\n"); } } catch (Exception ex) @@ -1185,6 +1237,8 @@ namespace Duplicati.Library.Main.RSync } } + m_oldSignatures.Clear(); + if (m_deletedfiles.Count > 0) { //The +100 is a safety margin @@ -1194,6 +1248,7 @@ namespace Duplicati.Library.Main.RSync signaturefile.WriteAllLines(DELETED_FILES, m_deletedfiles.ToArray()); contentfile.WriteAllLines(DELETED_FILES, m_deletedfiles.ToArray()); + m_deletedfiles.Clear(); } //We only write the USN if all files were processed @@ -1204,6 +1259,20 @@ namespace Duplicati.Library.Main.RSync //Only write this if all files were processed if (m_checkedUnchangedFiles.Count > 0) signaturefile.WriteAllLines(UNMODIFIED_FILES, m_checkedUnchangedFiles.ToArray()); + + if (m_unproccesed.Symlinks.Count > 0) + { + foreach(KeyValuePair kvp in m_unproccesed.Symlinks) + { + string target = FilenamesToPlatformIndependant(new string[] { kvp.Value })[0]; + string source = Path.Combine(SYMLINK_ROOT, GetRelativeName(kvp.Key)); + byte[] targetBytes = Encoding.UTF8.GetBytes(target); + + contentfile.WriteAllBytes(source, targetBytes); + signaturefile.WriteAllBytes(source, targetBytes); + } + m_unproccesed.Symlinks.Clear(); + } } m_finalized = true; @@ -1299,9 +1368,14 @@ namespace Duplicati.Library.Main.RSync m_lastPartialFile = WritePossiblePartial(m_lastPartialFile, contentfile, signaturefile, volumesize); } + if (m_lastPartialFile != null) + return false; - while (m_unproccesed.Files.Count > 0 && totalSize < volumesize && m_lastPartialFile == null) + while (m_unproccesed.Files.Count > 0) { + if (totalSize >= volumesize) + break; + int next = m_sortedfilelist ? 0 : r.Next(0, m_unproccesed.Files.Count); string s = m_unproccesed.Files[next]; m_unproccesed.Files.RemoveAt(next); @@ -1504,7 +1578,7 @@ namespace Duplicati.Library.Main.RSync } } - if (m_unproccesed.Files.Count == 0 && m_lastPartialFile == null) + if (m_unproccesed.Files.Count == 0) return FinalizeMultiPass(signaturefile, contentfile, volumesize); else return false; @@ -1740,15 +1814,16 @@ namespace Duplicati.Library.Main.RSync /// public void FinalizeRestore() { + Snapshots.ISystemIO SystemIO = Utility.Utility.IsClientLinux ? (Snapshots.ISystemIO)new Snapshots.SystemIOLinux() : (Snapshots.ISystemIO)new Snapshots.SystemIOWindows(); if (m_folders_to_delete != null) { foreach (string s in m_folders_to_delete) { - if (System.IO.Directory.Exists(s)) + if (SystemIO.DirectoryExists(s)) try { //TODO: Perhaps read ahead in patches to prevent creation - System.IO.Directory.Delete(s, false); + SystemIO.DirectoryDelete(s, false); if (m_stat as RestoreStatistics != null) { (m_stat as RestoreStatistics).FoldersDeleted++; @@ -1790,8 +1865,8 @@ namespace Duplicati.Library.Main.RSync if (m_folderTimestamps != null) foreach (KeyValuePair t in m_folderTimestamps) - if (System.IO.Directory.Exists(t.Key)) - try { System.IO.Directory.SetLastWriteTimeUtc(t.Key, t.Value); } + if (SystemIO.DirectoryExists(t.Key)) + try { SystemIO.DirectorySetLastWriteTimeUtc(t.Key, t.Value); } catch (Exception ex) { m_stat.LogWarning(string.Format(Strings.RSyncDir.FailedToSetFolderWriteTime, t.Key, ex.Message), ex); @@ -1881,6 +1956,8 @@ namespace Duplicati.Library.Main.RSync /// The content file that the destination is patched with public void Patch(string[] destination, Library.Interface.ICompression patch) { + Snapshots.ISystemIO SystemIO = Utility.Utility.IsClientLinux ? (Snapshots.ISystemIO)new Snapshots.SystemIOLinux() : (Snapshots.ISystemIO)new Snapshots.SystemIOWindows(); + if (m_partialDeltas == null) m_partialDeltas = new Dictionary(); @@ -1899,14 +1976,14 @@ namespace Duplicati.Library.Main.RSync if (patch.FileExists(DELETED_FILES)) foreach (string s in fh.Filterlist(FilenamesFromPlatformIndependant(patch.ReadAllLines(DELETED_FILES)), false)) { - if (System.IO.File.Exists(s)) + if (SystemIO.FileExists(s)) { try { //TODO: Perhaps read ahead in patches to prevent creation - long size = new FileInfo(s).Length; + long size = SystemIO.FileLength(s); - System.IO.File.Delete(s); + SystemIO.FileDelete(s); if (m_stat as RestoreStatistics != null) { (m_stat as RestoreStatistics).FilesRestored--; @@ -1955,10 +2032,10 @@ namespace Duplicati.Library.Main.RSync foreach (string s in addedfolders) { - if (!System.IO.Directory.Exists(s)) + if (!SystemIO.DirectoryExists(s)) try { - System.IO.Directory.CreateDirectory(s); + SystemIO.DirectoryCreate(s); if (m_stat as RestoreStatistics != null) (m_stat as RestoreStatistics).FoldersRestored++; } @@ -2009,6 +2086,9 @@ namespace Duplicati.Library.Main.RSync string deltaprefix = Utility.Utility.AppendDirSeparator(DELTA_ROOT); List deltafiles = m_filter.FilterList(deltaprefix, patch.ListFiles(deltaprefix)); + string symlinkprefix = Utility.Utility.AppendDirSeparator(SYMLINK_ROOT); + List symlinks = m_filter.FilterList(symlinkprefix, patch.ListFiles(symlinkprefix)); + long totalfiles = deltafiles.Count + contentfiles.Count; long fileindex = 0; @@ -2018,10 +2098,10 @@ namespace Duplicati.Library.Main.RSync string target = GetFullPathFromRelname(destination, s.Substring(contentprefix.Length)); try { - if (!System.IO.Directory.Exists(System.IO.Path.GetDirectoryName(target))) + if (!SystemIO.DirectoryExists(SystemIO.PathGetDirectoryName(target))) { Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.RestoreFolderMissingError, target), Duplicati.Library.Logging.LogMessageType.Warning); - System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(target)); + SystemIO.DirectoryCreate(SystemIO.PathGetDirectoryName(target)); } //Update each 0.5% @@ -2058,7 +2138,7 @@ namespace Duplicati.Library.Main.RSync throw new Exception(string.Format(Strings.RSyncDir.FileShouldBePartialError, s)); long startOffset = pex == null ? 0 : pex.StartOffset; - using (System.IO.FileStream s2 = System.IO.File.OpenWrite(partialFile == null ? target : (string)partialFile)) + using (System.IO.Stream s2 = SystemIO.FileOpenWrite(partialFile == null ? target : (string)partialFile)) { if (s2.Length != startOffset) throw new Exception(string.Format(Strings.RSyncDir.InvalidPartialFileEntry, s)); @@ -2072,9 +2152,9 @@ namespace Duplicati.Library.Main.RSync if (pex != null && pex == fe) { - if (System.IO.File.Exists(target)) - System.IO.File.Delete(target); - System.IO.File.Move(partialFile, target); + if (SystemIO.FileExists(target)) + SystemIO.FileDelete(target); + SystemIO.FileMove(partialFile, target); partialFile.Dispose(); m_partialDeltas.Remove(s); } @@ -2082,7 +2162,7 @@ namespace Duplicati.Library.Main.RSync if (m_stat is RestoreStatistics && (partialFile == null || pex == fe)) { (m_stat as RestoreStatistics).FilesRestored++; - (m_stat as RestoreStatistics).SizeOfRestoredFiles += new FileInfo(target).Length; + (m_stat as RestoreStatistics).SizeOfRestoredFiles += SystemIO.FileLength(target); } } @@ -2091,7 +2171,7 @@ namespace Duplicati.Library.Main.RSync DateTime t = patch.GetLastWriteTime(s); if (!isUtc) t = t.ToUniversalTime(); - try { File.SetLastWriteTimeUtc(target, t); } + try { SystemIO.FileSetLastWriteTimeUtc(target, t); } catch (Exception ex) { if (m_stat != null) @@ -2122,10 +2202,10 @@ namespace Duplicati.Library.Main.RSync lastPg = pg; } - if (!System.IO.Directory.Exists(System.IO.Path.GetDirectoryName(target))) + if (!SystemIO.DirectoryExists(SystemIO.PathGetDirectoryName(target))) { Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.RestoreFolderDeltaError, target), Duplicati.Library.Logging.LogMessageType.Warning); - System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(target)); + SystemIO.DirectoryCreate(SystemIO.PathGetDirectoryName(target)); } PartialEntryRecord pex = null; @@ -2147,7 +2227,7 @@ namespace Duplicati.Library.Main.RSync m_partialDeltas.Add(s, new Duplicati.Library.Utility.TempFile()); //Dump the content in the temp file at the specified offset - using (System.IO.Stream st = System.IO.File.OpenWrite(m_partialDeltas[s])) + using (System.IO.Stream st = SystemIO.FileOpenWrite(m_partialDeltas[s])) { if (st.Length != pex.StartOffset) throw new Exception(string.Format(Strings.RSyncDir.InvalidPartialFileEntry, s)); @@ -2171,28 +2251,28 @@ namespace Duplicati.Library.Main.RSync using (tempDelta) //May be null, but the using directive does not care { //Use either the patch directly, or the partial temp file - System.IO.Stream deltaStream = tempDelta == null ? patch.OpenRead(s) : System.IO.File.OpenRead(tempDelta); + System.IO.Stream deltaStream = tempDelta == null ? patch.OpenRead(s) : SystemIO.FileOpenRead(tempDelta); using (System.IO.Stream s2 = deltaStream) - using (System.IO.FileStream s1 = System.IO.File.OpenRead(target)) - using (System.IO.FileStream s3 = System.IO.File.Create(tempfile)) + using (System.IO.Stream s1 = SystemIO.FileOpenRead(target)) + using (System.IO.Stream s3 = SystemIO.FileCreate(tempfile)) SharpRSync.Interface.PatchFile(s1, s2, s3); if (m_stat as RestoreStatistics != null) { - (m_stat as RestoreStatistics).SizeOfRestoredFiles -= new FileInfo(target).Length; - (m_stat as RestoreStatistics).SizeOfRestoredFiles += new FileInfo(tempfile).Length; + (m_stat as RestoreStatistics).SizeOfRestoredFiles -= SystemIO.FileLength(target); + (m_stat as RestoreStatistics).SizeOfRestoredFiles += SystemIO.FileLength(tempfile); (m_stat as RestoreStatistics).FilesPatched++; } - System.IO.File.Delete(target); + SystemIO.FileDelete(target); - try { System.IO.File.Move(tempfile, target); } + try { SystemIO.FileMove(tempfile, target); } catch { //The OS sometimes reports the file as existing even after a delete // this seems to be related to MS Security Essentials? System.Threading.Thread.Sleep(500); - System.IO.File.Move(tempfile, target); + SystemIO.FileMove(tempfile, target); } } @@ -2202,7 +2282,7 @@ namespace Duplicati.Library.Main.RSync if (!isUtc) t = t.ToUniversalTime(); - try { File.SetLastWriteTimeUtc(target, t); } + try { SystemIO.FileSetLastWriteTimeUtc(target, t); } catch (Exception ex) { if (m_stat != null) @@ -2216,11 +2296,58 @@ namespace Duplicati.Library.Main.RSync m_stat.LogError(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), ex); Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), Duplicati.Library.Logging.LogMessageType.Error, ex); - try { System.IO.File.Delete(target); } + try { SystemIO.FileDelete(target); } catch { } } fileindex++; } + + //Re-create symlinks (no progress report here, should be really fast) + foreach (string s in symlinks) + { + string target = GetFullPathFromRelname(destination, s.Substring(symlinkprefix.Length)); + string symlinktarget = ""; + try + { + symlinktarget = FilenamesFromPlatformIndependant(new string[] { Encoding.UTF8.GetString(patch.ReadAllBytes(s)) })[0]; + bool isDir = symlinktarget[symlinktarget.Length - 1] == Path.DirectorySeparatorChar; + if (isDir) + symlinktarget = symlinktarget.Substring(0, symlinktarget.Length - 1); + + try + { + //In case another symlink is present, we "update" it + if (SystemIO.FileExists(target) && (SystemIO.GetFileAttributes(target) & FileAttributes.ReparsePoint) != 0) + SystemIO.FileDelete(target); + } + catch (Exception ex) + { + Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), Duplicati.Library.Logging.LogMessageType.Error, ex); + } + + SystemIO.CreateSymlink(target, symlinktarget, isDir); + } + catch (Exception ex) + { + if (m_stat != null) + m_stat.LogError(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), ex); + Logging.Log.WriteMessage(string.Format(Strings.RSyncDir.RestoreFileError, s, ex.Message), Duplicati.Library.Logging.LogMessageType.Error, ex); + + try { SystemIO.FileDelete(target); } + catch { } + + try + { + if (!string.IsNullOrEmpty(symlinktarget)) + using (System.IO.StreamWriter sw = new System.IO.StreamWriter(SystemIO.FileOpenWrite(target))) + sw.Write(symlinktarget); + } + catch + { + } + } + } + } @@ -2379,6 +2506,7 @@ namespace Duplicati.Library.Main.RSync List lst = new List(); lst.AddRange(m_oldFolders.Keys); lst.AddRange(m_oldSignatures.Keys); + lst.AddRange(m_oldSymlinks.Keys); return lst; } @@ -2508,20 +2636,79 @@ namespace Duplicati.Library.Main.RSync /// private class PathCollector { + private readonly Options.SymlinkStrategy m_symlinkHandling; + private readonly FileAttributes m_attributeMask; + private readonly Utility.FilenameFilter m_filter; + private readonly Snapshots.ISnapshotService m_snapshot; + private readonly CommunicationStatistics m_stat; + private List m_files = new List(); private List m_folders = new List(); private List m_errors = new List(); private List m_filesWithError = new List(); private List m_filesTooLarge = new List(); + private List> m_symlinks = new List>(); - public void Callback(string rootpath, string path, Utility.Utility.EnumeratedFileStatus status) + public PathCollector(Snapshots.ISnapshotService snapshot, Options.SymlinkStrategy symlinkHandling, FileAttributes attributeMask, Utility.FilenameFilter filter, CommunicationStatistics stat) { - if (status == Utility.Utility.EnumeratedFileStatus.Folder) - m_folders.Add(path); - else if (status == Utility.Utility.EnumeratedFileStatus.File) - m_files.Add(path); - else if (status == Utility.Utility.EnumeratedFileStatus.Error) + m_symlinkHandling = symlinkHandling; + m_filter = filter; + m_attributeMask = attributeMask; + m_snapshot = snapshot; + m_stat = stat; + } + + public bool Callback(string rootpath, string path, FileAttributes attributes) + { + if ((attributes & Utility.Utility.ATTRIBUTE_ERROR) != 0) + { m_errors.Add(path); + return false; + } + + if ((attributes & m_attributeMask) != 0) + return false; + + if (!m_filter.ShouldInclude(rootpath, path)) + return false; + + if ((attributes & FileAttributes.ReparsePoint) != 0) + { + if (m_symlinkHandling == Options.SymlinkStrategy.Ignore) + return false; + else if (m_symlinkHandling == Options.SymlinkStrategy.Store) + { + try + { + //We treat symlinks as files, even if they point to folders + if (path[path.Length - 1] == Path.DirectorySeparatorChar) + path = path.Substring(0, path.Length - 1); + + string s = m_snapshot.GetSymlinkTarget(path); + if (s != null) + { + if ((attributes & FileAttributes.Directory) != 0) + s = Utility.Utility.AppendDirSeparator(s); + m_symlinks.Add(new KeyValuePair(path, s)); + } + } + catch(Exception ex) + { + Logging.Log.WriteMessage(string.Format("Failed to obtain symlink target for {0}, message: {1}", path, ex.Message), Duplicati.Library.Logging.LogMessageType.Warning, ex); + if (m_stat != null) + m_stat.LogWarning(string.Format("Failed to obtain symlink target for {0}, message: {1}", path, ex.Message), ex); + m_errors.Add(path); + } + return false; + } + } + + if ((attributes & FileAttributes.Directory) != 0) + m_folders.Add(path); + else + m_files.Add(path); + + return true; } public List Files { get { return m_files; } } @@ -2529,6 +2716,7 @@ namespace Duplicati.Library.Main.RSync public List Errors { get { return m_errors; } } public List FilesWithError { get { return m_filesWithError; } } public List FilesTooLarge { get { return m_filesTooLarge; } } + public List> Symlinks { get { return m_symlinks; } } public bool IsAffectedByError(string path) { diff --git a/Duplicati/Library/Main/Strings/Options.Designer.cs b/Duplicati/Library/Main/Strings/Options.Designer.cs index 8613d912d..7034f76ae 100644 --- a/Duplicati/Library/Main/Strings/Options.Designer.cs +++ b/Duplicati/Library/Main/Strings/Options.Designer.cs @@ -1265,5 +1265,42 @@ namespace Duplicati.Library.Main.Strings { return ResourceManager.GetString("VssexcludewritersShort", resourceCulture); } } + + /// + /// Looks up a localized string similar to Using this option to handle symlinks different. The "{0}" option will simply record a symlink with its name and destination, and a restore will recreate the symlink as a link. Use the option "{1}" to ignore all symlinks and not store any information about them. Previous versions of Duplicati used the setting "{2}", which will cause symlinked files to be included and restore as normal files.. + /// + internal static string SymlinkpolicyLong { + get { + return ResourceManager.GetString("SymlinkpolicyLong", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Symlink handling. + /// + internal static string SymlinkpolicyShort { + get { + return ResourceManager.GetString("SymlinkpolicyShort", resourceCulture); + } + } + + + /// + /// Exclude files by attribute. + /// + internal static string ExcludefilesattributesShort { + get { + return ResourceManager.GetString("ExcludefilesattributesShort", resourceCulture); + } + } + + /// + /// Use this option to exclude files with certain attributes. Use a comma separated list of attribute names to specify more that one. Possible values are: {0}. + /// + internal static string ExcludefilesattributesLong { + get { + return ResourceManager.GetString("ExcludefilesattributesLong", resourceCulture); + } + } } } diff --git a/Duplicati/Library/Main/Strings/Options.resx b/Duplicati/Library/Main/Strings/Options.resx index c77efec96..50c50a0c6 100644 --- a/Duplicati/Library/Main/Strings/Options.resx +++ b/Duplicati/Library/Main/Strings/Options.resx @@ -518,5 +518,18 @@ A reported maximum storage + + + Symlink handling + + Using this option to handle symlinks different. The "{0}" option will simply record a symlink with its name and destination, and a restore will recreate the symlink as a link. Use the option "{1}" to ignore all symlinks and not store any information about them. Previous versions of Duplicati used the setting "{2}", which will cause symlinked files to be included and restore as normal files. + + + Exclude files by attribute + + + Use this option to exclude files with certain attributes. Use a comma separated list of attribute names to specify more that one. Possible values are: {0} + + \ No newline at end of file diff --git a/Duplicati/Library/Main/Strings/RSyncDir.Designer.cs b/Duplicati/Library/Main/Strings/RSyncDir.Designer.cs index 070442519..03621e2f4 100644 --- a/Duplicati/Library/Main/Strings/RSyncDir.Designer.cs +++ b/Duplicati/Library/Main/Strings/RSyncDir.Designer.cs @@ -332,5 +332,14 @@ namespace Duplicati.Library.Main.Strings { return ResourceManager.GetString("UsnNumberingFaultWarning", resourceCulture); } } - } + + /// + /// Looks up a localized string similar to The symlink information for {0} could not be read: {1}. + /// + internal static string SymlinkReadError { + get { + return ResourceManager.GetString("SymlinkReadError", resourceCulture); + } + } +} } diff --git a/Duplicati/Library/Main/Strings/RSyncDir.resx b/Duplicati/Library/Main/Strings/RSyncDir.resx index 4f000fe4e..a1392cb09 100644 --- a/Duplicati/Library/Main/Strings/RSyncDir.resx +++ b/Duplicati/Library/Main/Strings/RSyncDir.resx @@ -210,4 +210,7 @@ Error message: {1} The timestamp information for file {0} could not be read: {1} + + The symlink information for {0} could not be read: {1} + \ No newline at end of file diff --git a/Duplicati/Library/Snapshots/Duplicati.Library.Snapshots.csproj b/Duplicati/Library/Snapshots/Duplicati.Library.Snapshots.csproj index f1f1ef261..7801eca99 100644 --- a/Duplicati/Library/Snapshots/Duplicati.Library.Snapshots.csproj +++ b/Duplicati/Library/Snapshots/Duplicati.Library.Snapshots.csproj @@ -64,6 +64,9 @@ ..\..\..\thirdparty\alphavss\Bin\AlphaVSS.Common.dll + + ..\..\..\thirdparty\UnixSupport\UnixSupport.dll + @@ -97,6 +100,11 @@ + + + + + diff --git a/Duplicati/Library/Snapshots/ISnapshotService.cs b/Duplicati/Library/Snapshots/ISnapshotService.cs index 8ee382f4b..2163a278d 100644 --- a/Duplicati/Library/Snapshots/ISnapshotService.cs +++ b/Duplicati/Library/Snapshots/ISnapshotService.cs @@ -31,17 +31,15 @@ namespace Duplicati.Library.Snapshots /// /// Enumerates all files and folders in the snapshot /// - /// The filter to apply when evaluating files and folders /// The callback to invoke with each found path - void EnumerateFilesAndFolders(Duplicati.Library.Utility.FilenameFilter filter, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback); + void EnumerateFilesAndFolders(Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback); /// /// Enumerates all files and folders in the snapshot /// /// The path from which to retrieve files and folders - /// The filter to apply when evaluating files and folders /// The callback to invoke with each found path - void EnumerateFilesAndFolders(string startpath, Duplicati.Library.Utility.FilenameFilter filter, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback); + void EnumerateFilesAndFolders(string startpath, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback); /// /// Gets the last write time of a given file @@ -63,5 +61,19 @@ namespace Duplicati.Library.Snapshots /// The full path to the file in non-snapshot format /// The lenth of the file long GetFileSize(string file); + + /// + /// Returns the symlink target if the entry is a symlink, and null otherwise + /// + /// The file or folder to examine + /// The symlink target + string GetSymlinkTarget(string file); + + /// + /// Gets the attributes for the given file or folder + /// + /// The file attributes + /// The file or folder to examine + System.IO.FileAttributes GetAttributes(string file); } } diff --git a/Duplicati/Library/Snapshots/ISystemIO.cs b/Duplicati/Library/Snapshots/ISystemIO.cs new file mode 100644 index 000000000..2c9a64478 --- /dev/null +++ b/Duplicati/Library/Snapshots/ISystemIO.cs @@ -0,0 +1,47 @@ +// Copyright (C) 2011, Kenneth Skovhede + +// http://www.hexad.dk, opensource@hexad.dk +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +using System; +using System.IO; + +namespace Duplicati.Library.Snapshots +{ + /// + /// Interface for wrapping System.IO operations. + /// + public interface ISystemIO + { + void DirectoryDelete(string path); + void DirectoryDelete(string path, bool recursive); + void DirectoryCreate(string path); + bool DirectoryExists(string path); + void DirectorySetLastWriteTimeUtc(string path, DateTime time); + + void FileMove(string source, string target); + void FileDelete(string path); + void FileSetLastWriteTimeUtc(string path, DateTime time); + bool FileExists(string path); + long FileLength(string path); + Stream FileOpenRead(string path); + Stream FileOpenWrite(string path); + Stream FileCreate(string path); + FileAttributes GetFileAttributes(string path); + void CreateSymlink(string symlinkfile, string target, bool asDir); + string PathGetDirectoryName(string path); + } +} + diff --git a/Duplicati/Library/Snapshots/LinuxSnapshot.cs b/Duplicati/Library/Snapshots/LinuxSnapshot.cs index dbdb00ce7..340892787 100644 --- a/Duplicati/Library/Snapshots/LinuxSnapshot.cs +++ b/Duplicati/Library/Snapshots/LinuxSnapshot.cs @@ -386,12 +386,12 @@ namespace Duplicati.Library.Snapshots /// The path from which to retrieve files and folders /// The filter to apply when evaluating files and folders /// The callback to invoke with each found path - public void EnumerateFilesAndFolders(string startpath, Duplicati.Library.Utility.FilenameFilter filter, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) + public void EnumerateFilesAndFolders(string startpath, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) { foreach (KeyValuePair s in m_entries) if (s.Key.Equals(startpath, Utility.Utility.ClientFilenameStringComparision)) { - Utility.Utility.EnumerateFileSystemEntries(s.Key, filter, callback, this.ListFolders, this.ListFiles); + Utility.Utility.EnumerateFileSystemEntries(s.Key, callback, this.ListFolders, this.ListFiles); return; } @@ -403,10 +403,10 @@ namespace Duplicati.Library.Snapshots /// /// The filter to apply when evaluating files and folders /// The callback to invoke with each found path - public void EnumerateFilesAndFolders(Duplicati.Library.Utility.FilenameFilter filter, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) + public void EnumerateFilesAndFolders(Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) { foreach (KeyValuePair s in m_entries) - Utility.Utility.EnumerateFileSystemEntries(s.Key, filter, callback, this.ListFolders, this.ListFiles); + Utility.Utility.EnumerateFileSystemEntries(s.Key, callback, this.ListFolders, this.ListFiles); } /// @@ -439,6 +439,26 @@ namespace Duplicati.Library.Snapshots return new System.IO.FileInfo(ConvertToSnapshotPath(FindSnapShotByLocalPath(file), file)).Length; } + /// + /// Gets the attributes for the given file or folder + /// + /// The file attributes + /// The file or folder to examine + public System.IO.FileAttributes GetAttributes(string file) + { + return System.IO.File.GetAttributes(file); + } + + /// + /// Returns the symlink target if the entry is a symlink, and null otherwise + /// + /// The file or folder to examine + /// The symlink target + public string GetSymlinkTarget(string file) + { + return UnixSupport.File.GetSymlinkTarget(file); + } + #endregion #region IDisposable Members diff --git a/Duplicati/Library/Snapshots/NoSnapshot.cs b/Duplicati/Library/Snapshots/NoSnapshot.cs index 1ebf9cdc1..a1bf5519a 100644 --- a/Duplicati/Library/Snapshots/NoSnapshot.cs +++ b/Duplicati/Library/Snapshots/NoSnapshot.cs @@ -27,12 +27,17 @@ namespace Duplicati.Library.Snapshots /// A simple implementation of snapshots that only uses the regular System.IO operations, /// used to simplify code that uses snapshots, but also handles a non-snapshot enabled systems /// - public class NoSnapshot : ISnapshotService + public abstract class NoSnapshot : ISnapshotService { /// /// The list of all folders in the snapshot /// - private string[] m_sourcefolders; + protected string[] m_sourcefolders; + + /// + /// A frequently used char-as-string + /// + protected readonly string DIR_SEP = System.IO.Path.DirectorySeparatorChar.ToString(); /// /// Constructs a new backup snapshot, using all the required disks @@ -47,7 +52,7 @@ namespace Duplicati.Library.Snapshots /// Constructs a new backup snapshot, using all the required disks /// /// The folders that are about to be backed up - /// True if the file may be opened with read/write permissions + /// A set of system options public NoSnapshot(string[] folders, Dictionary options) { m_sourcefolders = new string[folders.Length]; @@ -55,6 +60,29 @@ namespace Duplicati.Library.Snapshots m_sourcefolders[i] = Utility.Utility.AppendDirSeparator(folders[i]); } + #region Private Methods + /// + /// Lists all folders in the given folder + /// + /// All folders found in the folder + /// The folder to examinate + protected virtual string[] ListFolders(string folder) + { + return System.IO.Directory.GetDirectories(folder); + } + + + /// + /// Lists all files in the given folder + /// + /// All folders found in the folder + /// The folder to examinate + protected virtual string[] ListFiles(string folder) + { + return System.IO.Directory.GetFiles(folder); + } + #endregion + #region ISnapshotService Members /// @@ -68,14 +96,13 @@ namespace Duplicati.Library.Snapshots /// Enumerates all files and folders in the snapshot /// /// The path from which to retrieve files and folders - /// The filter to apply when evaluating files and folders /// The callback to invoke with each found path - public void EnumerateFilesAndFolders(string startpath, Duplicati.Library.Utility.FilenameFilter filter, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) + public void EnumerateFilesAndFolders(string startpath, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) { foreach (string s in m_sourcefolders) if (s.Equals(startpath, Utility.Utility.ClientFilenameStringComparision)) { - Utility.Utility.EnumerateFileSystemEntries(s, filter, callback); + Utility.Utility.EnumerateFileSystemEntries(s, callback, this.ListFolders, this.ListFiles, this.GetAttributes); return; } @@ -85,12 +112,11 @@ namespace Duplicati.Library.Snapshots /// /// Enumerates all files and folders in the snapshot /// - /// The filter to apply when evaluating files and folders /// The callback to invoke with each found path - public void EnumerateFilesAndFolders(Duplicati.Library.Utility.FilenameFilter filter, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) + public void EnumerateFilesAndFolders(Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) { foreach (string s in m_sourcefolders) - Utility.Utility.EnumerateFileSystemEntries(s, filter, callback); + Utility.Utility.EnumerateFileSystemEntries(s, callback, this.ListFolders, this.ListFiles, this.GetAttributes); } /// @@ -98,7 +124,7 @@ namespace Duplicati.Library.Snapshots /// /// The full path to the file in non-snapshot format /// The last write time of the file - public DateTime GetLastWriteTime(string file) + public virtual DateTime GetLastWriteTime(string file) { return System.IO.File.GetLastWriteTime(file); } @@ -108,7 +134,7 @@ namespace Duplicati.Library.Snapshots /// /// The full path to the file in non-snapshot format /// An open filestream that can be read - public System.IO.Stream OpenRead(string file) + public virtual System.IO.Stream OpenRead(string file) { return System.IO.File.OpenRead(file); } @@ -118,7 +144,7 @@ namespace Duplicati.Library.Snapshots /// /// The full path to the file in non-snapshot format /// An open filestream that can be read - public System.IO.Stream OpenLockedRead(string file) + public virtual System.IO.Stream OpenLockedRead(string file) { return System.IO.File.Open(file, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.ReadWrite); } @@ -128,10 +154,30 @@ namespace Duplicati.Library.Snapshots /// /// The full path to the file in non-snapshot format /// The lenth of the file - public long GetFileSize(string file) + public virtual long GetFileSize(string file) { return new System.IO.FileInfo(file).Length; } + + /// + /// Returns the symlink target if the entry is a symlink, and null otherwise + /// + /// The file or folder to examine + /// The symlink target + public abstract string GetSymlinkTarget(string file); + + /// + /// Gets the attributes for the given file or folder + /// + /// The file attributes + /// The file or folder to examine + public virtual System.IO.FileAttributes GetAttributes(string file) + { + if (file.EndsWith(DIR_SEP)) + return System.IO.File.GetAttributes(file.Substring(0, file.Length - 1)); + else + return System.IO.File.GetAttributes(file); + } #endregion } } diff --git a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs new file mode 100644 index 000000000..5a62ab847 --- /dev/null +++ b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs @@ -0,0 +1,53 @@ +// Copyright (C) 2011, Kenneth Skovhede + +// http://www.hexad.dk, opensource@hexad.dk +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +using System; +using System.Text; +using System.Collections.Generic; + +namespace Duplicati.Library.Snapshots +{ + /// + /// Handler for providing a snapshot like access to files and folders + /// + public class NoSnapshotLinux : NoSnapshot + { + public NoSnapshotLinux(string[] sourcefolders) + : base(sourcefolders) + { + } + + public NoSnapshotLinux(string[] sourcefolders, Dictionary options) + : base(sourcefolders, options) + { + } + + /// + /// Returns the symlink target if the entry is a symlink, and null otherwise + /// + /// The file or folder to examine + /// The symlink target + public override string GetSymlinkTarget(string file) + { + if (file.EndsWith(DIR_SEP)) + return UnixSupport.File.GetSymlinkTarget(file.Substring(0, file.Length - 1)); + else + return UnixSupport.File.GetSymlinkTarget(file); + } + } +} + diff --git a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs new file mode 100644 index 000000000..a8e9dfc6d --- /dev/null +++ b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs @@ -0,0 +1,153 @@ +// Copyright (C) 2011, Kenneth Skovhede + +// http://www.hexad.dk, opensource@hexad.dk +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +using System; +using System.Text; +using System.Collections.Generic; +using Alphaleonis.Win32.Filesystem; + +namespace Duplicati.Library.Snapshots +{ + /// + /// Handler for providing a snapshot like access to files and folders + /// + public class NoSnapshotWindows : NoSnapshot + { + private SystemIOWindows m_sysIO = new SystemIOWindows(); + + public NoSnapshotWindows(string[] sourcefolders) + : base(sourcefolders) + { + } + + public NoSnapshotWindows(string[] sourcefolders, Dictionary options) + : base(sourcefolders, options) + { + } + + /// + /// Returns the symlink target if the entry is a symlink, and null otherwise + /// + /// The file or folder to examine + /// The symlink target + public override string GetSymlinkTarget(string file) + { + return File.GetLinkTargetInfo(SystemIOWindows.PrefixWithUNC(file)).PrintName; + } + + /// + /// Gets the attributes for the given file or folder + /// + /// The file attributes + /// The file or folder to examine + public override System.IO.FileAttributes GetAttributes (string file) + { + return m_sysIO.GetFileAttributes(file); + } + + /// + /// Returns the size of a file + /// + /// The full path to the file in non-snapshot format + /// The lenth of the file + public override long GetFileSize (string file) + { + return m_sysIO.FileLength(file); + } + + /// + /// Gets the last write time of a given file + /// + /// The full path to the file in non-snapshot format + /// The last write time of the file + public override DateTime GetLastWriteTime (string file) + { + if (!SystemIOWindows.IsPathTooLong(file)) + try { return base.GetLastWriteTime(file); } + catch (System.IO.PathTooLongException) { } + + return File.GetLastWriteTime(SystemIOWindows.PrefixWithUNC(file)); + } + + /// + /// Opens a locked file for reading + /// + /// The full path to the file in non-snapshot format + /// An open filestream that can be read + public override System.IO.Stream OpenLockedRead (string file) + { + if (!SystemIOWindows.IsPathTooLong(file)) + try { return base.OpenLockedRead(file); } + catch (System.IO.PathTooLongException) { } + + return File.Open(SystemIOWindows.PrefixWithUNC(file), FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + } + + /// + /// Opens a file for reading + /// + /// The full path to the file in non-snapshot format + /// An open filestream that can be read + public override System.IO.Stream OpenRead (string file) + { + if (!SystemIOWindows.IsPathTooLong(file)) + try { return base.OpenRead(file); } + catch (System.IO.PathTooLongException) { } + + return File.OpenRead(SystemIOWindows.PrefixWithUNC(file)); + } + + /// + /// Lists all files in the given folder + /// + /// All folders found in the folder + /// The folder to examinate + protected override string[] ListFiles (string folder) + { + if (!SystemIOWindows.IsPathTooLong(folder)) + try { return base.ListFiles(folder); } + catch (System.IO.PathTooLongException) { } + + string[] tmp = Directory.GetFiles(SystemIOWindows.PrefixWithUNC(folder)); + string[] res = new string[tmp.Length]; + for(int i = 0; i < tmp.Length; i++) + res[i] = SystemIOWindows.StripUNCPrefix(tmp[i]); + + return res; + } + + /// + /// Lists all folders in the given folder + /// + /// All folders found in the folder + /// The folder to examinate + protected override string[] ListFolders (string folder) + { + if (!SystemIOWindows.IsPathTooLong(folder)) + try { return base.ListFolders(folder); } + catch (System.IO.PathTooLongException) { } + + string[] tmp = Directory.GetDirectories(SystemIOWindows.PrefixWithUNC(folder)); + string[] res = new string[tmp.Length]; + for (int i = 0; i < tmp.Length; i++) + res[i] = SystemIOWindows.StripUNCPrefix(tmp[i]); + + return res; + } + } +} + diff --git a/Duplicati/Library/Snapshots/SystemIOLinux.cs b/Duplicati/Library/Snapshots/SystemIOLinux.cs new file mode 100644 index 000000000..a5a17556b --- /dev/null +++ b/Duplicati/Library/Snapshots/SystemIOLinux.cs @@ -0,0 +1,110 @@ +// Copyright (C) 2011, Kenneth Skovhede + +// http://www.hexad.dk, opensource@hexad.dk +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +using System; +using System.IO; + +namespace Duplicati.Library.Snapshots +{ + public struct SystemIOLinux : ISystemIO + { + #region ISystemIO implementation + public void DirectoryDelete(string path) + { + Directory.Delete(path); + } + + public void DirectoryCreate(string path) + { + Directory.CreateDirectory(path); + } + + public bool DirectoryExists(string path) + { + return Directory.Exists(path); + } + + public void FileDelete(string path) + { + File.Delete(path); + } + + public void FileSetLastWriteTimeUtc(string path, DateTime time) + { + File.SetLastWriteTimeUtc(path, time); + } + + public bool FileExists(string path) + { + return File.Exists(path); + } + + public Stream FileOpenRead(string path) + { + return File.OpenRead(path); + } + + public Stream FileOpenWrite(string path) + { + return File.OpenWrite(path); + } + + public Stream FileCreate(string path) + { + return File.Create(path); + } + + public FileAttributes GetFileAttributes(string path) + { + return File.GetAttributes(path); + } + + public void CreateSymlink(string symlinkfile, string target, bool asDir) + { + UnixSupport.File.CreateSymlink(symlinkfile, target); + } + public string PathGetDirectoryName(string path) + { + return Path.GetDirectoryName(path); + } + + public void DirectorySetLastWriteTimeUtc(string path, DateTime time) + { + Directory.SetLastWriteTimeUtc(path, time); + } + + public void FileMove(string source, string target) + { + File.Move(source, target); + } + + public long FileLength(string path) + { + return new System.IO.FileInfo(path).Length; + } + + public void DirectoryDelete(string path, bool recursive) + { + Directory.Delete(path, recursive); + } + #endregion + + + } + +} + diff --git a/Duplicati/Library/Snapshots/SystemIOWindows.cs b/Duplicati/Library/Snapshots/SystemIOWindows.cs new file mode 100644 index 000000000..fc6fc9b36 --- /dev/null +++ b/Duplicati/Library/Snapshots/SystemIOWindows.cs @@ -0,0 +1,236 @@ +// Copyright (C) 2011, Kenneth Skovhede + +// http://www.hexad.dk, opensource@hexad.dk +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +using System; + + +namespace Duplicati.Library.Snapshots +{ + public struct SystemIOWindows : ISystemIO + { + private const string UNCPREFIX = @"\\?\"; + + public static bool IsPathTooLong(string path) + { + if (path.StartsWith(UNCPREFIX) || path.Length > 260) + return true; + + return false; + } + + public static string PrefixWithUNC(string path) + { + if (!path.StartsWith(UNCPREFIX)) + return UNCPREFIX + path; + else + return path; + } + + public static string StripUNCPrefix(string path) + { + if (path.StartsWith(UNCPREFIX)) + return path.Substring(UNCPREFIX.Length); + else + return path; + } + + #region ISystemIO implementation + public void DirectoryDelete(string path) + { + if (!IsPathTooLong(path)) + try + { + System.IO.Directory.Delete(path); + return; + } + catch (System.IO.PathTooLongException) { } + + Alphaleonis.Win32.Filesystem.Directory.Delete(PrefixWithUNC(path)); + } + + public void DirectoryCreate(string path) + { + if (!IsPathTooLong(path)) + try + { + System.IO.Directory.CreateDirectory(path); + return; + } + catch (System.IO.PathTooLongException) { } + + Alphaleonis.Win32.Filesystem.Directory.CreateDirectory(PrefixWithUNC(path)); + } + + public bool DirectoryExists(string path) + { + if (!IsPathTooLong(path)) + try { return System.IO.Directory.Exists(path); } + catch (System.IO.PathTooLongException) { } + + return Alphaleonis.Win32.Filesystem.Directory.Exists(PrefixWithUNC(path)); + } + + public void FileDelete(string path) + { + if (!IsPathTooLong(path)) + try + { + System.IO.File.Delete(path); + return; + } + catch (System.IO.PathTooLongException) { } + + Alphaleonis.Win32.Filesystem.File.Delete(PrefixWithUNC(path)); + } + + public void FileSetLastWriteTimeUtc(string path, DateTime time) + { + if (!IsPathTooLong(path)) + try + { + System.IO.File.SetLastWriteTimeUtc(path, time); + return; + } + catch (System.IO.PathTooLongException) { } + + Alphaleonis.Win32.Filesystem.File.SetLastWriteTimeUtc(PrefixWithUNC(path), time); + } + + public bool FileExists(string path) + { + if (!IsPathTooLong(path)) + try { return System.IO.File.Exists(path); } + catch (System.IO.PathTooLongException) { } + + return Alphaleonis.Win32.Filesystem.File.Exists(PrefixWithUNC(path)); + } + + public System.IO.Stream FileOpenRead(string path) + { + if (!IsPathTooLong(path)) + try { return System.IO.File.OpenRead(path); } + catch (System.IO.PathTooLongException) { } + + return Alphaleonis.Win32.Filesystem.File.OpenRead(PrefixWithUNC(path)); + } + + public System.IO.Stream FileOpenWrite(string path) + { + if (!IsPathTooLong(path)) + try { return System.IO.File.OpenWrite(path); } + catch (System.IO.PathTooLongException) { } + + if (FileExists(path)) + return Alphaleonis.Win32.Filesystem.File.OpenWrite(PrefixWithUNC(path)); + else + return FileCreate(path); + } + + public System.IO.Stream FileCreate(string path) + { + if (!IsPathTooLong(path)) + try { return System.IO.File.Create(path); } + catch (System.IO.PathTooLongException) { } + + return Alphaleonis.Win32.Filesystem.File.Create(PrefixWithUNC(path)); + } + + public System.IO.FileAttributes GetFileAttributes(string path) + { + if (!IsPathTooLong(path)) + try { return System.IO.File.GetAttributes(path); } + catch (System.IO.PathTooLongException) { } + + return (System.IO.FileAttributes)Alphaleonis.Win32.Filesystem.File.GetAttributes(PrefixWithUNC(path)); + } + + public void CreateSymlink(string symlinkfile, string target, bool asDir) + { + if (FileExists(symlinkfile) || DirectoryExists(symlinkfile)) + throw new System.IO.IOException(string.Format("File already exists: {0}", symlinkfile)); + Alphaleonis.Win32.Filesystem.File.CreateSymbolicLink(target, PrefixWithUNC(symlinkfile), asDir ? Alphaleonis.Win32.Filesystem.SymbolicLinkTarget.Directory : Alphaleonis.Win32.Filesystem.SymbolicLinkTarget.File); + + //Sadly we do not get a notification if the creation fails :( + System.IO.FileAttributes attr = 0; + if ((!asDir && FileExists(symlinkfile)) || (asDir && DirectoryExists(symlinkfile))) + try { attr = GetFileAttributes(symlinkfile); } + catch { } + + if ((attr & System.IO.FileAttributes.ReparsePoint) == 0) + throw new System.IO.IOException(string.Format("Unable to create symlink, check account permissions: {0}", symlinkfile)); + } + + public string PathGetDirectoryName(string path) + { + if (!IsPathTooLong(path)) + try { return System.IO.Path.GetDirectoryName(path); } + catch (System.IO.PathTooLongException) { } + + return StripUNCPrefix(Alphaleonis.Win32.Filesystem.Path.GetDirectoryName(PrefixWithUNC(path))); + } + + public void DirectorySetLastWriteTimeUtc(string path, DateTime time) + { + if (!IsPathTooLong(path)) + try + { + System.IO.Directory.SetLastWriteTimeUtc(path, time); + return; + } + catch (System.IO.PathTooLongException) { } + + Alphaleonis.Win32.Filesystem.File.SetLastWriteTimeUtc(PrefixWithUNC(path), time); + } + + public void FileMove(string source, string target) + { + if (!IsPathTooLong(source) && !IsPathTooLong(target)) + try + { + System.IO.File.Move(source, target); + return; + } + catch (System.IO.PathTooLongException) { } + + Alphaleonis.Win32.Filesystem.File.Move(PrefixWithUNC(source), PrefixWithUNC(target)); + } + + public long FileLength(string path) + { + if (!IsPathTooLong(path)) + try { return new System.IO.FileInfo(path).Length; } + catch (System.IO.PathTooLongException) { } + + return new Alphaleonis.Win32.Filesystem.FileInfo(PrefixWithUNC(path)).Length; + } + + public void DirectoryDelete(string path, bool recursive) + { + if (!IsPathTooLong(path)) + try + { + System.IO.Directory.Delete(path, recursive); + return; + } + catch (System.IO.PathTooLongException) { } + + Alphaleonis.Win32.Filesystem.Directory.Delete(PrefixWithUNC(path), recursive); + } + #endregion + } +} + diff --git a/Duplicati/Library/Snapshots/USNHelper.cs b/Duplicati/Library/Snapshots/USNHelper.cs index dfbd4d47e..cad697193 100644 --- a/Duplicati/Library/Snapshots/USNHelper.cs +++ b/Duplicati/Library/Snapshots/USNHelper.cs @@ -121,9 +121,8 @@ namespace Duplicati.Library.Snapshots /// Helper function to support the Duplicati enumeration callback system /// /// The root path to look in and use as filter base - /// The filter to apply /// The callback function that collects the output - public void EnumerateFilesAndFolders(string rootpath, Duplicati.Library.Utility.FilenameFilter filter, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) + public void EnumerateFilesAndFolders(string rootpath, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) { //Under normal enumeration, the filter will prevent visiting subfolders for excluded folders //But when using USN all files/folders are present in the list, so we have to maintain @@ -141,9 +140,7 @@ namespace Duplicati.Library.Snapshots if (isFolder) { - if (filter == null || filter.ShouldInclude(rootpath, Utility.Utility.AppendDirSeparator(r.Key))) - callback(rootpath, r.Key, Duplicati.Library.Utility.Utility.EnumeratedFileStatus.Folder); - else + if (!callback(rootpath, r.Key, (System.IO.FileAttributes)((int)r.Value.FileAttributes & 0xff))) { if (local_filter.Length != 0) local_filter.Append("|"); @@ -159,8 +156,8 @@ namespace Duplicati.Library.Snapshots } else { - if (filter == null || filter.ShouldInclude(rootpath, r.Key)) - callback(rootpath, r.Key, Duplicati.Library.Utility.Utility.EnumeratedFileStatus.File); + if (local_filter.Length == 0 || !excludedFolders.IsMatch(r.Key)) + callback(rootpath, r.Key, (System.IO.FileAttributes)((int)r.Value.FileAttributes & 0xff)); } } } diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index 204237fd1..9fb814000 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -178,14 +178,16 @@ namespace Duplicati.Library.Snapshots /// The source folder /// The full entry path /// The entry type - public void callback(string rootpath, string path, Utility.Utility.EnumeratedFileStatus status) + public bool callback(string rootpath, string path, FileAttributes attributes) { - if (status == Duplicati.Library.Utility.Utility.EnumeratedFileStatus.File) - files.Add(path); - else if (status == Duplicati.Library.Utility.Utility.EnumeratedFileStatus.Folder) + if ((attributes & Utility.Utility.ATTRIBUTE_ERROR) != 0) + errors.Add(path); + else if ((attributes & FileAttributes.Directory) != 0) folders.Add(path); else - errors.Add(path); + files.Add(path); + + return true; } } #endif @@ -262,12 +264,12 @@ namespace Duplicati.Library.Snapshots /// The path from which to retrieve files and folders /// The filter to apply when evaluating files and folders /// The callback to invoke with each found path - public void EnumerateFilesAndFolders(string startpath, Duplicati.Library.Utility.FilenameFilter filter, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) + public void EnumerateFilesAndFolders(string startpath, Duplicati.Library.Utility.Utility.EnumerationCallbackDelegate callback) { foreach (string s in m_sourcepaths) if (s.Equals(startpath, Utility.Utility.ClientFilenameStringComparision)) { - Utility.Utility.EnumerateFileSystemEntries(s, filter, callback, this.ListFolders, this.ListFiles); + Utility.Utility.EnumerateFileSystemEntries(s, callback, this.ListFolders, this.ListFiles, this.GetAttributes); return; } @@ -279,10 +281,10 @@ namespace Duplicati.Library.Snapshots /// /// The filter to apply when evaluating files and folders /// The callback to invoke with each found path - public void EnumerateFilesAndFolders(Utility.FilenameFilter filter, Utility.Utility.EnumerationCallbackDelegate callback) + public void EnumerateFilesAndFolders(Utility.Utility.EnumerationCallbackDelegate callback) { foreach (string s in m_sourcepaths) - Utility.Utility.EnumerateFileSystemEntries(s, filter, callback, this.ListFolders, this.ListFiles); + Utility.Utility.EnumerateFileSystemEntries(s, callback, this.ListFolders, this.ListFiles, this.GetAttributes); } /// @@ -315,6 +317,25 @@ namespace Duplicati.Library.Snapshots return new Alphaleonis.Win32.Filesystem.FileInfo(GetSnapshotPath(file)).Length; } + /// + /// Gets the attributes for the given file or folder + /// + /// The file attributes + /// The file or folder to examine + public System.IO.FileAttributes GetAttributes(string file) + { + return (System.IO.FileAttributes)Alphaleonis.Win32.Filesystem.File.GetAttributes(GetSnapshotPath(file)); + } + + /// + /// Returns the symlink target if the entry is a symlink, and null otherwise + /// + /// The file or folder to examine + /// The symlink target + public string GetSymlinkTarget(string file) + { + return Alphaleonis.Win32.Filesystem.File.GetLinkTargetInfo(file).PrintName; + } #endregion #region IDisposable Members diff --git a/Duplicati/Library/Utility/BlockingQueue.cs b/Duplicati/Library/Utility/BlockingQueue.cs new file mode 100644 index 000000000..caa9c0431 --- /dev/null +++ b/Duplicati/Library/Utility/BlockingQueue.cs @@ -0,0 +1,229 @@ +#region Disclaimer / License +// Copyright (C) 2011, Kenneth Skovhede +// http://www.hexad.dk, opensource@hexad.dk +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// +#endregion +using System; +using System.Collections.Generic; +using System.Text; + +namespace Duplicati.Library.Utility +{ + /// + /// Class for wrapping a as an + /// + public class BlockingQueueAsEnumerable : IEnumerable + { + private IEnumerator m_enumerator; + + public BlockingQueueAsEnumerable(BlockingQueue queue) + { + m_enumerator = new BlockingQueueEnumerator(queue); + } + + #region IEnumerable implementation + public System.Collections.IEnumerator GetEnumerator() + { + return ((IEnumerable)this).GetEnumerator(); + } + #endregion + + #region IEnumerable implementation + IEnumerator IEnumerable.GetEnumerator() + { + if (m_enumerator == null) + throw new Exception("Can only return a single enumerator for a blocking queue"); + + IEnumerator tmp = m_enumerator; + m_enumerator = null; + return tmp; + } + #endregion + + private class BlockingQueueEnumerator : IEnumerator + { + private BlockingQueue m_queue; + private bool m_first = true; + private T m_current; + + public BlockingQueueEnumerator(BlockingQueue queue) + { + m_queue = queue; + } + + #region IDisposable implementation + public void Dispose() + { + } + #endregion + + #region IEnumerator implementation + public bool MoveNext() + { + m_current = m_queue.Dequeue(); + return !(object.Equals(m_current, default(T)) && m_queue.Completed); + + } + + public void Reset() + { + if (!m_first) + throw new System.NotImplementedException(); + } + + public object Current + { + get + { + return m_current; + } + } + #endregion + + #region IEnumerator implementation + T IEnumerator.Current + { + get + { + return m_current; + } + } + #endregion + } + } + + /// + /// Implementation of a blocking queue that serves as a producer/consumer contact point + /// + /// The type of data in the queue + public class BlockingQueue + { + /// + /// The lock that protects the data structures + /// + private object m_lock = new object(); + /// + /// The queue storing the elements produced + /// + private Queue m_queue = new Queue(); + /// + /// A flag indicating if the queue is now empty forever + /// + private bool m_completed = false; + /// + /// The event that signals that items have been produced (consumers wait for this) + /// + private System.Threading.ManualResetEvent m_itemsProduced = new System.Threading.ManualResetEvent(false); + /// + /// The event that signals that items have been consumed (producers wait for this) + /// + private System.Threading.ManualResetEvent m_itemsConsumed = new System.Threading.ManualResetEvent(false); + /// + /// The maximum capacity of the queue, the producers will be blocked if more elements are put into the queue + /// + private long m_maxCapacity = 100; + + /// + /// Initializes a new instance of the class with the default capacity + /// + public BlockingQueue() + { + } + + /// + /// Gets a value indicating whether this is completed + /// + public bool Completed { get { return m_completed; } } + + /// + /// Initializes a new instance of the class + /// + /// The maximum capacity of the queue, the producers will be blocked if more elements are put into the queue + public BlockingQueue(long maxCapacity) + : this() + { + if (maxCapacity <= 0) + throw new ArgumentException("maxCapacity", "maxCapacity must be a positive number"); + + m_maxCapacity = maxCapacity; + } + + /// + /// Signals that the producers are done and no more elements will be put into the queue + /// + public void SetCompleted() + { + m_completed = true; + m_itemsProduced.Set(); + m_itemsConsumed.Set(); + } + + /// + /// Enqueues the specified item. + /// + /// The item to put into the queue + public void Enqueue(T item) + { + while (true) + { + lock(m_lock) + { + if (m_completed) + throw new InvalidOperationException("Cannot insert items after complete flag has been set"); + + if (m_queue.Count < m_maxCapacity) + { + m_queue.Enqueue(item); + m_itemsProduced.Set(); + return; + } + + m_itemsConsumed.Reset(); + } + m_itemsConsumed.WaitOne(); + } + } + + /// + /// Dequeues an element from the queue + /// + /// The dequed element or default(T) if there are no more elements + public T Dequeue() + { + while(true) + { + lock(m_lock) + { + if (m_queue.Count > 0) + { + m_itemsConsumed.Reset(); + return m_queue.Dequeue(); + } + + if (m_completed) + return default(T); + + m_itemsProduced.Reset(); + } + + m_itemsProduced.WaitOne(); + } + } + + } +} + diff --git a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj index e600e5352..848a4d43f 100644 --- a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj +++ b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj @@ -113,6 +113,7 @@ + diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 8b3ac7dbc..b8efb69a1 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -46,22 +46,18 @@ namespace Duplicati.Library.Utility public static readonly DateTime EPOCH = new DateTime(1970, 1, 1, 0, 0, 0); /// - /// An enumeration used to identify file types when performing file listings + /// The attribute value used to indicate error /// - public enum EnumeratedFileStatus - { - File, - Folder, - Error - }; + public const System.IO.FileAttributes ATTRIBUTE_ERROR = (System.IO.FileAttributes)(1 << 30); /// /// The callback delegate type used to collecting file information /// /// The path that the file enumeration started at /// The current element - /// The type of the element - public delegate void EnumerationCallbackDelegate(string rootpath, string path, EnumeratedFileStatus status); + /// The attributes of the element + /// A value indicating if the folder should be recursed, ignored for other types + public delegate bool EnumerationCallbackDelegate(string rootpath, string path, System.IO.FileAttributes attributes); /// /// Copies the content of one stream into another @@ -92,11 +88,63 @@ namespace Duplicati.Library.Utility target.Write(buf, 0, read); } + /// + /// Returns an IEnumerable with all filesystem entries, recursive + /// + /// The folder to look in + /// An optional filter to apply to the filenames + /// The function to call with the filenames + /// A function to call that lists all folders in the supplied folder + /// A function to call that lists all files in the supplied folder + /// A function to call that obtains the attributes for an element, set to null to skip symlink detection + /// The file system entries + public static IEnumerable EnumerateFileSystemEntriesBlocked(string rootpath, FilenameFilter filter, FileSystemInteraction folderList, FileSystemInteraction fileList, ExtractFileAttributes attributeReader) + { + BlockingQueue queue = new BlockingQueue(); + new BlockedCollector(queue, rootpath, filter, folderList, fileList); + return new BlockingQueueAsEnumerable(queue); + } + + /// + /// Class used to implement blocked collection of files and folders + /// + private class BlockedCollector + { + private BlockingQueue m_queue; + public BlockedCollector(BlockingQueue queue, string rootpath, FilenameFilter filter, FileSystemInteraction folderList, FileSystemInteraction fileList) + { + m_queue = queue; + System.Threading.Thread t = new System.Threading.Thread(BlockedThreadRunner); + t.Start(new object[] { rootpath, filter, folderList, fileList}); + } + + private bool Callback(string rootpath, string path, System.IO.FileAttributes attributes) + { + if ((attributes & ATTRIBUTE_ERROR) == 0) + m_queue.Enqueue(path); + + return true; + } + + private void BlockedThreadRunner(object _args) + { + object[] args = (object[])_args; + string roothpath = (string)args[0]; + FilenameFilter filter = (FilenameFilter)args[1]; + FileSystemInteraction folderList = (FileSystemInteraction)args[2]; + FileSystemInteraction fileList = (FileSystemInteraction)args[3]; + + EnumerateFileSystemEntries(roothpath, Callback, folderList, fileList); + m_queue.SetCompleted(); + } + } + /// /// Returns a list of all files found in the given folder. /// The search is recursive. /// /// The folder to look in + /// The filter to apply. /// A list of the full filenames public static List EnumerateFiles(string basepath) { @@ -114,6 +162,59 @@ namespace Duplicati.Library.Utility return EnumerateFolders(basepath, null); } + /// + /// Returns a list of all files and subfolders found in the given folder. + /// The search is recursive. + /// + /// The folder to look in. + /// A list of the full filenames and foldernames. Foldernames ends with the directoryseparator char + public static List EnumerateFileSystemEntries(string basepath) + { + return EnumerateFileSystemEntries(basepath, (FilenameFilter)null); + } + + /// + /// Returns a list of all files found in the given folder. + /// The search is recursive. + /// + /// The folder to look in + /// The filter to apply. + /// A list of the full filenames + public static List EnumerateFiles(string basepath, FilenameFilter filter) + { + PathCollector c = new PathCollector(false, true, filter); + EnumerateFileSystemEntries(basepath, new EnumerationCallbackDelegate(c.Callback)); + return c.Files; + } + + /// + /// Returns a list of folder names found in the given folder. + /// The search is recursive. + /// + /// The folder to look in + /// The filter to apply. + /// A list of the full paths + public static List EnumerateFolders(string basepath, FilenameFilter filter) + { + PathCollector c = new PathCollector(true, false, filter); + EnumerateFileSystemEntries(basepath, new EnumerationCallbackDelegate(c.Callback)); + return c.Files; + } + + /// + /// Returns a list of all files and subfolders found in the given folder. + /// The search is recursive. + /// + /// The folder to look in. + /// The filter to apply. + /// A list of the full filenames and foldernames. Foldernames ends with the directoryseparator char + public static List EnumerateFileSystemEntries(string basepath, FilenameFilter filter) + { + PathCollector c = new PathCollector(true, true, filter); + EnumerateFileSystemEntries(basepath, new EnumerationCallbackDelegate(c.Callback)); + return c.Files; + } + /// /// A callback delegate used for applying alternate enumeration of filesystems /// @@ -122,17 +223,11 @@ namespace Duplicati.Library.Utility public delegate string[] FileSystemInteraction(string path); /// - /// Returns a list of all files found in the given folder. - /// The search is recursive. + /// A callback delegate used for extracting attributes from a file or folder /// - /// The folder to look in - /// An optional filter to apply to the filenames - /// The function to call with the filenames - /// A list of the full filenames - public static void EnumerateFileSystemEntries(string rootpath, FilenameFilter filter, EnumerationCallbackDelegate callback) - { - EnumerateFileSystemEntries(rootpath, filter, callback, new FileSystemInteraction(System.IO.Directory.GetDirectories), new FileSystemInteraction(System.IO.Directory.GetFiles)); - } + /// The path to return data from + /// Attributes for the file or folder + public delegate System.IO.FileAttributes ExtractFileAttributes(string path); /// /// Returns a list of all files found in the given folder. @@ -141,10 +236,38 @@ namespace Duplicati.Library.Utility /// The folder to look in /// An optional filter to apply to the filenames /// The function to call with the filenames + /// A list of the full filenames + public static void EnumerateFileSystemEntries(string rootpath, EnumerationCallbackDelegate callback) + { + EnumerateFileSystemEntries(rootpath, callback, new FileSystemInteraction(System.IO.Directory.GetDirectories), new FileSystemInteraction(System.IO.Directory.GetFiles)); + } + + /// + /// Returns a list of all files found in the given folder. + /// The search is recursive. + /// + /// The folder to look in + /// The function to call with the filenames /// A function to call that lists all folders in the supplied folder /// A function to call that lists all files in the supplied folder /// A list of the full filenames - public static void EnumerateFileSystemEntries(string rootpath, FilenameFilter filter, EnumerationCallbackDelegate callback, FileSystemInteraction folderList, FileSystemInteraction fileList) + public static void EnumerateFileSystemEntries(string rootpath, EnumerationCallbackDelegate callback, FileSystemInteraction folderList, FileSystemInteraction fileList) + { + + EnumerateFileSystemEntries(rootpath, callback, folderList, fileList, null); + } + + /// + /// Returns a list of all files found in the given folder. + /// The search is recursive. + /// + /// The folder to look in + /// The function to call with the filenames + /// A function to call that lists all folders in the supplied folder + /// A function to call that lists all files in the supplied folder + /// A function to call that obtains the attributes for an element, set to null to avoid reading attributes + /// A list of the full filenames + public static void EnumerateFileSystemEntries(string rootpath, EnumerationCallbackDelegate callback, FileSystemInteraction folderList, FileSystemInteraction fileList, ExtractFileAttributes attributeReader) { if (!System.IO.Directory.Exists(rootpath)) return; @@ -157,11 +280,12 @@ namespace Duplicati.Library.Utility string f = AppendDirSeparator(lst.Dequeue()); try { - foreach (string s in folderList(f)) - if (filter == null || filter.ShouldInclude(rootpath, AppendDirSeparator(s))) - lst.Enqueue(s); + System.IO.FileAttributes attr = attributeReader == null ? System.IO.FileAttributes.Directory : attributeReader(f); + if (!callback(rootpath, f, attr)) + continue; - callback(rootpath, f, EnumeratedFileStatus.Folder); + foreach (string s in folderList(f)) + lst.Enqueue(s); } catch (System.Threading.ThreadAbortException) { @@ -169,14 +293,28 @@ namespace Duplicati.Library.Utility } catch (Exception) { - callback(rootpath, f, EnumeratedFileStatus.Error); + callback(rootpath, f, ATTRIBUTE_ERROR | System.IO.FileAttributes.Directory); } try { - foreach (string s in fileList(f)) - if (filter == null || filter.ShouldInclude(rootpath, s)) - callback(rootpath, s, EnumeratedFileStatus.File); + if (fileList != null) + foreach (string s in fileList(f)) + { + try + { + System.IO.FileAttributes attr = attributeReader == null ? System.IO.FileAttributes.Normal : attributeReader(s); + callback(rootpath, s, attr); + } + catch (System.Threading.ThreadAbortException) + { + throw; + } + catch (Exception) + { + callback(rootpath, s, ATTRIBUTE_ERROR); + } + } } catch (System.Threading.ThreadAbortException) { @@ -184,9 +322,8 @@ namespace Duplicati.Library.Utility } catch (Exception) { - callback(rootpath, f, EnumeratedFileStatus.Error); + callback(rootpath, f, ATTRIBUTE_ERROR); } - } } @@ -196,22 +333,37 @@ namespace Duplicati.Library.Utility private class PathCollector { private List m_files; + private FilenameFilter m_filter; private bool m_includeFolders; private bool m_includeFiles; - public PathCollector(bool includeFolders, bool includeFiles) + public PathCollector(bool includeFolders, bool includeFiles, FilenameFilter filter) { m_files = new List(); + m_filter = filter; m_includeFolders = includeFolders; m_includeFiles = includeFiles; } - public void Callback(string rootpath, string path, EnumeratedFileStatus status) + public bool Callback(string rootpath, string path, System.IO.FileAttributes attributes) { - if (m_includeFolders && status == EnumeratedFileStatus.Folder) - m_files.Add(path); - else if (m_includeFiles && status == EnumeratedFileStatus.File) - m_files.Add(path); + if ((attributes & ATTRIBUTE_ERROR) != 0) + { + //Don't care + } + else + { + if (m_filter != null && !m_filter.ShouldInclude(rootpath, path)) + return false; + + bool isDirectory = (attributes & System.IO.FileAttributes.Directory) != 0; + if (m_includeFolders && isDirectory) + m_files.Add(path); + else if (m_includeFiles && !isDirectory) + m_files.Add(path); + } + + return true; } public List Files { get { return m_files; } } @@ -223,12 +375,23 @@ namespace Duplicati.Library.Utility private class PathSizeCalculator { private long m_size = 0; + private FilenameFilter m_filter; - public void Callback(string rootpath, string path, EnumeratedFileStatus status) + public PathSizeCalculator(FilenameFilter filter) { - if (status == EnumeratedFileStatus.File) + m_filter = filter; + } + + public bool Callback(string rootpath, string path, System.IO.FileAttributes attributes) + { + if (m_filter != null && !m_filter.ShouldInclude(rootpath, path)) + return false; + + if ((attributes & System.IO.FileAttributes.Directory) == 0) try { m_size += new System.IO.FileInfo(path).Length; } catch { } + + return true; } public long Size { get { return m_size; } } @@ -242,64 +405,11 @@ namespace Duplicati.Library.Utility /// The combined size of all files that match the filter public static long GetDirectorySize(string folder, FilenameFilter filter) { - PathSizeCalculator c = new PathSizeCalculator(); - EnumerateFileSystemEntries(folder, filter, new EnumerationCallbackDelegate(c.Callback)); + PathSizeCalculator c = new PathSizeCalculator(filter); + EnumerateFileSystemEntries(folder, new EnumerationCallbackDelegate(c.Callback)); return c.Size; } - /// - /// Returns a list of all files found in the given folder. - /// The search is recursive. - /// - /// The folder to look in - /// An optional filter to apply to the filenames - /// A list of the full filenames - public static List EnumerateFiles(string basepath, FilenameFilter filter) - { - PathCollector c = new PathCollector(false, true); - EnumerateFileSystemEntries(basepath, filter, new EnumerationCallbackDelegate(c.Callback)); - return c.Files; - } - - /// - /// Returns a list of all files and subfolders found in the given folder. - /// The search is recursive. - /// - /// The folder to look in. - /// A list of the full filenames and foldernames. Foldernames ends with the directoryseparator char - public static List EnumerateFileSystemEntries(string basepath) - { - return EnumerateFileSystemEntries(basepath, null); - } - - /// - /// Returns a list of all files and subfolders found in the given folder. - /// The search is recursive. - /// - /// The folder to look in. - /// An optional filter to apply to the filenames - /// A list of the full filenames and foldernames. Foldernames ends with the directoryseparator char - public static List EnumerateFileSystemEntries(string basepath, FilenameFilter filter) - { - PathCollector c = new PathCollector(true, true); - EnumerateFileSystemEntries(basepath, filter, new EnumerationCallbackDelegate(c.Callback)); - return c.Files; - } - - /// - /// Returns a list of folder names found in the given folder. - /// The search is recursive. - /// - /// The folder to look in - /// An optional filter to apply to the folder names - /// A list of the full paths - public static List EnumerateFolders(string basepath, FilenameFilter filter) - { - PathCollector c = new PathCollector(true, false); - EnumerateFileSystemEntries(basepath, filter, new EnumerationCallbackDelegate(c.Callback)); - return c.Files; - } - /// /// A cached instance of the directory separator as a string /// diff --git a/thirdparty/UnixSupport/File.cs b/thirdparty/UnixSupport/File.cs index 46a968283..caa2ae30f 100644 --- a/thirdparty/UnixSupport/File.cs +++ b/thirdparty/UnixSupport/File.cs @@ -9,7 +9,7 @@ namespace UnixSupport /// Opens the file and honors advisory locking. /// /// A open stream that references the file - /// The full path to the file + /// The full path to the file public static System.IO.Stream OpenExclusive(string path, System.IO.FileAccess mode) { return OpenExclusive(path, mode, (int)Mono.Unix.Native.FilePermissions.DEFFILEMODE); @@ -80,6 +80,31 @@ namespace UnixSupport { } } + + /// + /// Gets the symlink target for the given path + /// + /// The path to get the symlink target for + /// The symlink target + public static string GetSymlinkTarget(string path) + { + System.Text.StringBuilder sb = new System.Text.StringBuilder(2048); //2kb, should cover utf16 * 1023 chars + if (Mono.Unix.Native.Syscall.readlink(path, sb, (ulong)sb.Capacity) >= 0) + return sb.ToString(); + + throw new System.IO.FileLoadException(string.Format("Unable to get symlink for \"{0}\", error: {1} ({2})", path, Syscall.GetLastError(), (int)Syscall.GetLastError())); + } + + /// + /// Creates a new symlink + /// + /// The path to create the symbolic link entry + /// The path the symbolic link points to + public static void CreateSymlink(string path, string target) + { + if (Mono.Unix.Native.Syscall.symlink(target, path) != 0) + throw new System.IO.IOException(string.Format("Unable to create symlink from \"{0}\" to \"{1}\", error: {2} ({3})", path, target, Syscall.GetLastError(), (int)Syscall.GetLastError())); + } } } diff --git a/thirdparty/UnixSupport/UnixSupport.dll b/thirdparty/UnixSupport/UnixSupport.dll index 64701577ac8b36c50ef74f683f80406dd398cfae..022fb2b8f2a345772bd25e03f73a9c386b1cd483 100755 GIT binary patch delta 1497 zcmZorY0#L^AvR4>G=KpL9x(7Jax=0qFz`&A_lRo&Hv)p$UOfq@02TAP8PVX`21^5i#6E9%)87}%w0s{vFLoqi40~Z$)8(TgP0|Qr<1}_5xmzX^}0|NtFJ|6=EmsSxY1499m z20sG>R}VJ>10#gR3t|Z{FmM$MGB9veF$f!QGPpur!OWx~#K6F1X~Dq2P?XA0&rl%M z%f-OJs4B*^NL8G9k*Wm543KGyR3#xCkb#Rdgh3{XfK1e4U|^WVRP>mEp@12p0;C+m z0;$yiS*`(6ss*;(4r)15kuxL0Lgq!P;uK{+&-;Rzc9Lk5WLl#`#tz`)?cF}aLIPg|G)5(8`u z3=E)n6J>zJ4igd$VlyywO+L?JUeCpFl;JW17sFSEKMZUPJd8pNYz#t-LJV9CYK(sv zxEOT6q!ptNBNsy)n9Kr`m%yYnlO@=&MsNaPWME`yWE5bS#aPOy$pGRpD+n;mXJlY- zWME`qWdQLuKzVEoZ$L?^o?#lp5k^5qMutBiE|@3D$jq=n5#--#3~v~<8QB|A$RNoWz);9w%oxg0$pCVI zAVU~q5?CaIp%E-HlVJmc6Qdr(Y6dq(B}M~AmIjb>As7@)`3$9#1KE_B%^CJgE@wOG zpIe-qUzC%X!~ltCr_#)vl++@I;L75X)Lgxg)QS>@^wg4g=fr}<HSD9Oyvn>>rXS=BwYBp4zUl30|UTEgI5l$uzQ3Ke500Siu+;qWwNU|>iN zWyoVlWJqGjVMt|AU?^e8XHZ~BXGmozVNhTwW~gMyWyoR3WXNO4W>8>AW5{PHnmn1q zD5Msfwq%APhE#?`h7zz*n3{_i@)>d&6d05ksu>IzY8jN^7J@_!C(q`PoUFqsI60N$ z)?`OcGXsw8RWV_bCJ#E96&N_!8JIcvgjBhi6&Uzf7#IX~*%?^)_+(^cSQHo;`B)j4 zIQS+{Bs4T_?wI zsnjztJdG}jZS`O1KkD;Gq8Gn`s^KH zu4s3%&7f@gv%vS2=c)s9Up%~dyk%uo=jPkroIT1lC-37^_HWa?{XXRP(O1hWw`lK? zE?lVo@Z{T?&9SequkYF`eypoZR_Ecrw@pP?^u@z$g_9Gc?MhE=SbTMJYl}o#=)K2H P(JYe8c(Cj$e60s{jB*W@lnZAPBS+Zfdu z1t#BP?30yHVqmCXV_;x}Y7|joU|?ZjV3?r6z|b&xA#d{JH%u$)Sr{1Ds+fwILDVd! zB1Q&=0%i?v1_rJkP6h@>2#XuU;$dLm(%@xa;L>6cHsEBigPO`zl*qtPAl1vqz`&>~ z#=J;XoTZ1Kfq`+6ssuzc$e2Z{k`NBaq(vHh2%{$NXVJ9!!^ps(T*ttW!^FVw0K^V1 zDay=CF9uoR!@|IDhk=1XIW(9-gO!270VMB~lb^)Ez>vWWO3|tKN8J;q5F}wkjzZlpUxflc( zK_cReAa*^NT*P<)WDtWMm}F#NWN2g*U~p$tVzguc@t73^7JCIUG_Bxy8x(MLC&C44!!<#zqXmmBl5g zxeSnqVQ|YY%1ta`@bq`BNKP#%$;{7#D$w)vXYfzTN=+`A{F9@3@*+-;$>+IlO@pW8=WeNBc^p$0*e1`a zdhOrc2}gSBeGhOK&n^G+_UVn_zwx_@vYSKpzrUwaU=?-y^oP!pRZ$f(2So)L!j>IPaP&x& X4C-6j<;r_aeey?vf1COEKCuD-W0BIP