From 2c1a0ddb4e7f7b511e7de1ceb3b9c9c6ab7afe11 Mon Sep 17 00:00:00 2001 From: Daniel Gehriger Date: Thu, 19 Apr 2018 22:59:28 +0200 Subject: [PATCH] Implemented backup based on changes recorded in NTFS USN journal --- ...ti.Library.Modules.Builtin.UnitTest.csproj | 3 + .../Duplicati.Library.Interface.csproj | 1 + .../Library/Interface/USNJournalDataEntry.cs | 48 + Duplicati/Library/Main/Controller.cs | 3 +- .../Database schema/8. Add volume USN.sql | 10 + .../Main/Database/LocalBackupDatabase.cs | 176 +++- .../Library/Main/Database/LocalDatabase.cs | 1 + .../Main/Database/LocalDeleteDatabase.cs | 1 + .../Main/Duplicati.Library.Main.csproj | 1 + .../Library/Main/Operation/BackupHandler.cs | 833 +++++++++++------- .../Main/Operation/TestFilterHandler.cs | 6 +- Duplicati/Library/Main/Options.cs | 2 +- .../Duplicati.Library.Snapshots.csproj | 8 +- .../Snapshots/IChangeJournalService.cs | 62 ++ .../Library/Snapshots/ISnapshotService.cs | 86 +- Duplicati/Library/Snapshots/LinuxSnapshot.cs | 493 ++++++----- Duplicati/Library/Snapshots/NoSnapshot.cs | 199 ----- .../Library/Snapshots/NoSnapshotLinux.cs | 57 +- .../Library/Snapshots/NoSnapshotWindows.cs | 109 +-- Duplicati/Library/Snapshots/Program.cs | 97 +- Duplicati/Library/Snapshots/SnapshotBase.cs | 199 +++++ .../Library/Snapshots/SnapshotUtility.cs | 11 +- Duplicati/Library/Snapshots/Strings.cs | 5 + Duplicati/Library/Snapshots/SystemIOLinux.cs | 28 +- Duplicati/Library/Snapshots/USNHelper.cs | 518 ----------- Duplicati/Library/Snapshots/USNJournal.cs | 568 ++++++++++++ .../Library/Snapshots/UsnJournalService.cs | 193 ++++ Duplicati/Library/Snapshots/Win32USN.cs | 409 ++++----- .../Library/Snapshots/WindowsSnapshot.cs | 325 +++---- .../Utility/Duplicati.Library.Utility.csproj | 1 + Duplicati/Library/Utility/FilterExpression.cs | 43 +- Duplicati/Library/Utility/IFilter.cs | 6 + .../Library/Utility/JoinedFilterExpression.cs | 11 +- Duplicati/Library/Utility/MD5HashHelper.cs | 59 ++ Duplicati/Library/Utility/Utility.cs | 155 +++- 35 files changed, 2811 insertions(+), 1916 deletions(-) create mode 100644 Duplicati/Library/Interface/USNJournalDataEntry.cs create mode 100644 Duplicati/Library/Main/Database/Database schema/8. Add volume USN.sql create mode 100644 Duplicati/Library/Snapshots/IChangeJournalService.cs delete mode 100644 Duplicati/Library/Snapshots/NoSnapshot.cs create mode 100644 Duplicati/Library/Snapshots/SnapshotBase.cs delete mode 100644 Duplicati/Library/Snapshots/USNHelper.cs create mode 100644 Duplicati/Library/Snapshots/USNJournal.cs create mode 100644 Duplicati/Library/Snapshots/UsnJournalService.cs create mode 100644 Duplicati/Library/Utility/MD5HashHelper.cs diff --git a/Duplicati.Library.Modules.Builtin.UnitTest/Duplicati.Library.Modules.Builtin.UnitTest.csproj b/Duplicati.Library.Modules.Builtin.UnitTest/Duplicati.Library.Modules.Builtin.UnitTest.csproj index 2dca5a99d..9347e553b 100644 --- a/Duplicati.Library.Modules.Builtin.UnitTest/Duplicati.Library.Modules.Builtin.UnitTest.csproj +++ b/Duplicati.Library.Modules.Builtin.UnitTest/Duplicati.Library.Modules.Builtin.UnitTest.csproj @@ -55,6 +55,9 @@ Duplicati.Library.Modules.Builtin + + + \ No newline at end of file diff --git a/Duplicati/Library/Interface/Duplicati.Library.Interface.csproj b/Duplicati/Library/Interface/Duplicati.Library.Interface.csproj index 745e830ff..6c9bee0eb 100644 --- a/Duplicati/Library/Interface/Duplicati.Library.Interface.csproj +++ b/Duplicati/Library/Interface/Duplicati.Library.Interface.csproj @@ -47,6 +47,7 @@ + diff --git a/Duplicati/Library/Interface/USNJournalDataEntry.cs b/Duplicati/Library/Interface/USNJournalDataEntry.cs new file mode 100644 index 000000000..7970b99b5 --- /dev/null +++ b/Duplicati/Library/Interface/USNJournalDataEntry.cs @@ -0,0 +1,48 @@ +#region Disclaimer / License +// Copyright (C) 2015, The Duplicati Team +// http://www.duplicati.com, info@duplicati.com +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// +#endregion + +namespace Duplicati.Library.Interface +{ + /// + /// Data representing the state of an NTFS USN (Change) Journal + /// + public class USNJournalDataEntry + { + /// + /// Volume of journal + /// + public string Volume; + + /// + /// Journal ID, as recorded in the journal + /// + public long JournalId; + + /// + /// USN to start at for next fileset (backup) + /// + public long NextUsn; + + /// + /// A hash value representing the active exclusion filter + /// + public string ConfigHash; + } +} diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index 777260d40..eced1e7dd 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -23,6 +23,7 @@ using System.Linq; using System; using System.Collections.Generic; using System.Text; +using Duplicati.Library.Main.Operation; using Duplicati.Library.Utility; namespace Duplicati.Library.Main @@ -440,7 +441,7 @@ namespace Duplicati.Library.Main m_options.RawOptions["dbpath"] = "INVALID!"; // Redirect all messages from the filter to the message sink - var filtertag = Logging.Log.LogTagFromType(); + var filtertag = Logging.Log.LogTagFromType(); using (Logging.Log.StartScope(m_messageSink.WriteMessage, x => x.FilterTag.Contains(filtertag))) { return RunAction(new TestFilterResults(), ref paths, ref filter, (result) => diff --git a/Duplicati/Library/Main/Database/Database schema/8. Add volume USN.sql b/Duplicati/Library/Main/Database/Database schema/8. Add volume USN.sql new file mode 100644 index 000000000..11be01687 --- /dev/null +++ b/Duplicati/Library/Main/Database/Database schema/8. Add volume USN.sql @@ -0,0 +1,10 @@ +CREATE TABLE "ChangeJournalData" ( + "ID" INTEGER PRIMARY KEY, + "FilesetID" INTEGER NOT NULL, + "VolumeName" TEXT NOT NULL, + "JournalID" INTEGER NOT NULL, + "NextUsn" INTEGER NOT NULL, + "ConfigHash" TEXT NOT NULL +); + +UPDATE "Version" SET "Version" = 8; diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 8c3e20984..0372e202f 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -3,10 +3,12 @@ using System.Collections.Generic; using System.Linq; using System.Text; using System.IO; +using System.Linq.Expressions; namespace Duplicati.Library.Main.Database { + internal class LocalBackupDatabase : LocalDatabase { /// @@ -660,11 +662,26 @@ namespace Duplicati.Library.Main.Database } } - public void AppendFilesFromPreviousSet(System.Data.IDbTransaction transaction, IEnumerable deleted) + /// + /// Populates FilesetEntry table with files from previous fileset, which aren't + /// yet part of the new fileset, and which aren't on the (optional) list of deleted paths. + /// + /// Transaction + /// List of deleted paths, or null + public void AppendFilesFromPreviousSet(System.Data.IDbTransaction transaction, IEnumerable deleted = null) { AppendFilesFromPreviousSet(transaction, deleted, m_filesetId, -1, OperationTimestamp); } + /// + /// Populates FilesetEntry table with files from previous fileset, which aren't + /// yet part of the new fileset, and which aren't on the (optional) list of deleted paths. + /// + /// Transaction + /// List of deleted paths, or null + /// Current file-set ID + /// Source file-set ID + /// If filesetid == -1, used to locate previous file-set public void AppendFilesFromPreviousSet(System.Data.IDbTransaction transaction, IEnumerable deleted, long filesetid, long prevId, DateTime timestamp) { using(var cmd = m_connection.CreateCommand()) @@ -693,7 +710,81 @@ namespace Duplicati.Library.Main.Database tr.Commit(); } } - + + /// + /// Populates FilesetEntry table with files from previous fileset, which aren't + /// yet part of the new fileset, and which aren't excluded by the (optional) exclusion + /// predicate. + /// + /// Transaction + /// Optional exclusion predicate (true = exclude file) + public void AppendFilesFromPreviousSetWithPredicate(System.Data.IDbTransaction transaction, Predicate exclusionPredicate) + { + AppendFilesFromPreviousSetWithPredicate(transaction, exclusionPredicate, m_filesetId, -1, OperationTimestamp); + } + + /// + /// Populates FilesetEntry table with files from previous fileset, which aren't + /// yet part of the new fileset, and which aren't excluded by the (optional) exclusion + /// predicate. + /// + /// Transaction + /// Optional exclusion predicate (true = exclude file) + /// Current fileset ID + /// Source fileset ID + /// If prevFileSetId == -1, used to locate previous fileset + public void AppendFilesFromPreviousSetWithPredicate(System.Data.IDbTransaction transaction, + Predicate exclusionPredicate, long fileSetId, long prevFileSetId, DateTime timestamp) + { + if (exclusionPredicate == null) + { + AppendFilesFromPreviousSet(transaction, null, fileSetId, prevFileSetId, timestamp); + return; + } + + using (var cmd = m_connection.CreateCommand()) + using (var cmdAdd = m_connection.CreateCommand()) + using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) + { + var lastFilesetId = + prevFileSetId < 0 ? GetPreviousFilesetID(cmd, timestamp, fileSetId) : prevFileSetId; + + // prepare command for adding new entries + cmdAdd.Transaction = tr.Parent; + cmdAdd.CommandText = + @"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (?, ?, ?)"; + cmdAdd.AddParameters(3); + cmdAdd.SetParameterValue(0, fileSetId); + + // enumerate files from previous set + cmd.Transaction = tr.Parent; + foreach (var row in cmd.ExecuteReaderEnumerable( + @"SELECT + f.""Path"", fs.""FileID"", fs.""Lastmodified"" + FROM ( SELECT DISTINCT ""FileID"", ""Lastmodified"" + FROM ""FilesetEntry"" + WHERE ""FilesetID"" = ? + AND ""FileID"" NOT IN ( + SELECT ""FileID"" + FROM ""FilesetEntry"" + WHERE ""FilesetID"" = ? + )) AS fs + LEFT JOIN ""File"" AS f ON fs.""FileID"" = f.""ID"";", + lastFilesetId, fileSetId)) + { + if (!exclusionPredicate(row.GetString(0))) + { + cmdAdd.SetParameterValue(1, row.GetInt64(1)); + cmdAdd.SetParameterValue(2, row.GetInt64(2)); + cmdAdd.ExecuteNonQuery(); + } + } + + tr.Commit(); + } + } + + /// /// Creates a timestamped backup operation to correctly associate the fileset with the time it was created. /// @@ -780,5 +871,86 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Retrieves change journal data for file set + /// + /// Fileset-ID + public IEnumerable GetChangeJournalData(long fileSetId) + { + var data = new List(); + + using (var cmd = m_connection.CreateCommand()) + using (var rd = + cmd.ExecuteReader( + @"SELECT ""VolumeName"", ""JournalID"", ""NextUSN"", ""ConfigHash"" FROM ""ChangeJournalData"" WHERE ""FilesetID"" = ?", + fileSetId)) + { + while (rd.Read()) + { + data.Add(new Interface.USNJournalDataEntry + { + Volume = rd.ConvertValueToString(0), + JournalId = rd.ConvertValueToInt64(1), + NextUsn = rd.ConvertValueToInt64(2), + ConfigHash = rd.ConvertValueToString(3) + }); + } + + } + + return data; + } + + /// + /// Adds NTFS change journal data for file set and volume + /// + /// Data to add + /// An optional external transaction + public void CreateChangeJournalData(IEnumerable data, System.Data.IDbTransaction transaction = null) + { + using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) + { + foreach (var entry in data) + { + using(var cmd = m_connection.CreateCommand()) + { + cmd.Transaction = tr.Parent; + var c = cmd.ExecuteNonQuery( + @"INSERT INTO ""ChangeJournalData"" (""FilesetID"", ""VolumeName"", ""JournalID"", ""NextUSN"", ""ConfigHash"") VALUES (?, ?, ?, ?, ?);", + m_filesetId, entry.Volume, entry.JournalId, entry.NextUsn, entry.ConfigHash); + + if (c != 1) + throw new Exception("Unable to add change journal entry"); + } + } + + tr.Commit(); + } + } + + /// + /// Adds NTFS change journal data for file set and volume + /// + /// Data to add + /// Existing file set to update + /// An optional external transaction + public void UpdateChangeJournalData(IEnumerable data, long fileSetId, System.Data.IDbTransaction transaction = null) + { + using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) + { + foreach (var entry in data) + { + using(var cmd = m_connection.CreateCommand()) + { + cmd.Transaction = tr.Parent; + cmd.ExecuteNonQuery( + @"UPDATE ""ChangeJournalData"" SET ""NextUSN"" = ? WHERE ""FilesetID"" = ? AND ""VolumeName"" = ? AND ""JournalID"" = ?;", + entry.NextUsn, fileSetId, entry.Volume, entry.JournalId); + } + } + + tr.Commit(); + } + } } } diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 83f887c43..2808af571 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -400,6 +400,7 @@ namespace Duplicati.Library.Main.Database // If the volume is a fileset, this will remove the fileset, otherwise nothing will happen deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ({0}))", volIdsSubQuery)); + deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""ChangeJournalData"" WHERE ""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ({0}))", volIdsSubQuery)); deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""Fileset"" WHERE ""VolumeID"" IN ({0})", volIdsSubQuery)); var bsIdsSubQuery = string.Format( diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index 3a28024d1..3f958a87a 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -84,6 +84,7 @@ namespace Duplicati.Library.Main.Database //Then we delete anything that is no longer being referenced cmd.ExecuteNonQuery(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" NOT IN (SELECT DISTINCT ""ID"" FROM ""Fileset"")"); + cmd.ExecuteNonQuery(@"DELETE FROM ""ChangeJournalData"" WHERE ""FilesetID"" NOT IN (SELECT DISTINCT ""ID"" FROM ""Fileset"")"); cmd.ExecuteNonQuery(@"DELETE FROM ""File"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FileID"" FROM ""FilesetEntry"") "); cmd.ExecuteNonQuery(@"DELETE FROM ""Metadataset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""MetadataID"" FROM ""File"") "); cmd.ExecuteNonQuery(@"DELETE FROM ""Blockset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""BlocksetID"" FROM ""File"" UNION SELECT DISTINCT ""BlocksetID"" FROM ""Metadataset"") "); diff --git a/Duplicati/Library/Main/Duplicati.Library.Main.csproj b/Duplicati/Library/Main/Duplicati.Library.Main.csproj index 657f584aa..0fb49b543 100644 --- a/Duplicati/Library/Main/Duplicati.Library.Main.csproj +++ b/Duplicati/Library/Main/Duplicati.Library.Main.csproj @@ -180,6 +180,7 @@ + PreserveNewest diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index fdfcf24d7..1daa4828d 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -1,7 +1,28 @@ -using System; +#region Disclaimer / License + +// Copyright (C) 2015, The Duplicati Team +// http://www.duplicati.com, info@duplicati.com +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// + +#endregion +using System; using System.Collections.Generic; +using System.ComponentModel; using System.Linq; -using System.Text; using System.IO; using Duplicati.Library.Main.Database; using Duplicati.Library.Main.Volumes; @@ -18,7 +39,7 @@ namespace Duplicati.Library.Main.Operation private static readonly string LOGTAG = Logging.Log.LogTagFromType(); private readonly Options m_options; - private string m_backendurl; + private readonly string m_backendurl; private byte[] m_blockbuffer; private byte[] m_blocklistbuffer; @@ -30,16 +51,18 @@ namespace Duplicati.Library.Main.Operation private BlockVolumeWriter m_blockvolume; private IndexVolumeWriter m_indexvolume; + // ReSharper disable once InconsistentNaming private readonly IMetahash EMPTY_METADATA; private Library.Utility.IFilter m_filter; private Library.Utility.IFilter m_sourceFilter; - private BackupResults m_result; + private readonly BackupResults m_result; - //To better record what happens with the backend, we flush the log messages regularly + // To better record what happens with the backend, we flush the log messages regularly // We cannot flush immediately, because that would mess up the transaction, as the uploader // is on another thread + // ReSharper disable once InconsistentNaming private readonly TimeSpan FLUSH_TIMESPAN = TimeSpan.FromSeconds(10); private DateTime m_backendLogFlushTimer; @@ -48,7 +71,7 @@ namespace Duplicati.Library.Main.Operation private readonly Options.SymlinkStrategy m_symlinkPolicy; private int m_blocksize; private long m_maxmetadatasize; - private long m_lastfilesetid; + private long m_prevFilesetId; public BackupHandler(string backendurl, Options options, BackupResults results) { @@ -64,228 +87,50 @@ namespace Duplicati.Library.Main.Operation if (options.AllowPassphraseChange) throw new UserInformationException(Strings.Common.PassphraseChangeUnsupported, "PassphraseChangeUnsupported"); } - - - public class FilterHandler - { - private static readonly string FILTER_LOGTAG = Logging.Log.LogTagFromType(); - private Snapshots.ISnapshotService m_snapshot; - private FileAttributes m_attributeFilter; - private Duplicati.Library.Utility.IFilter m_enumeratefilter; - private Duplicati.Library.Utility.IFilter m_emitfilter; - private Options.SymlinkStrategy m_symlinkPolicy; - private Options.HardlinkStrategy m_hardlinkPolicy; - private Dictionary m_hardlinkmap; - private Duplicati.Library.Utility.IFilter m_sourcefilter; - private Queue m_mixinqueue; - - public FilterHandler(Snapshots.ISnapshotService snapshot, FileAttributes attributeFilter, Duplicati.Library.Utility.IFilter sourcefilter, Duplicati.Library.Utility.IFilter filter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy) - { - m_snapshot = snapshot; - m_attributeFilter = attributeFilter; - m_sourcefilter = sourcefilter; - m_emitfilter = filter; - m_symlinkPolicy = symlinkPolicy; - m_hardlinkPolicy = hardlinkPolicy; - m_hardlinkmap = new Dictionary(); - m_mixinqueue = new Queue(); - - bool includes; - bool excludes; - Library.Utility.FilterExpression.AnalyzeFilters(filter, out includes, out excludes); - if (includes && !excludes) - { - m_enumeratefilter = Library.Utility.FilterExpression.Combine(filter, new Duplicati.Library.Utility.FilterExpression("*" + System.IO.Path.DirectorySeparatorChar, true)); - } - else - m_enumeratefilter = m_emitfilter; - } - - public void ReportError(string rootpath, string path, Exception ex) - { - Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FileAccessError", ex, "Error reported while accessing file: {0}", path); - } - - public bool AttributeFilter(string rootpath, string path, FileAttributes attributes) - { - try - { - if (m_snapshot.IsBlockDevice(path)) - { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingBlockDevice", "Excluding block device: {0}", path); - return false; - } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingError", ex, "Failed to process path: {0}", path); - return false; - } - - Duplicati.Library.Utility.IFilter sourcematch; - bool sourcematches; - if (m_sourcefilter.Matches(path, out sourcematches, out sourcematch) && sourcematches) - { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingSourcePath", "Including source path: {0}", path); - - return true; - } - - if (m_hardlinkPolicy != Options.HardlinkStrategy.All) - { - try - { - var id = m_snapshot.HardlinkTargetID(path); - if (id != null) - { - if (m_hardlinkPolicy == Options.HardlinkStrategy.None) - { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingHardlinkByPolicy", "Excluding hardlink: {0} ({1})", path, id); - return false; - } - else if (m_hardlinkPolicy == Options.HardlinkStrategy.First) - { - string prevPath; - if (m_hardlinkmap.TryGetValue(id, out prevPath)) - { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingDuplicateHardlink", "Excluding hardlink ({1}) for: {0}, previous hardlink: {2}", path, id, prevPath); - return false; - } - else - { - m_hardlinkmap.Add(id, path); - } - } - } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingError", ex, "Failed to process path: {0}", path); - return false; - } - } - - if ((m_attributeFilter & attributes) != 0) - { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromAttributes", "Excluding path due to attribute filter: {0}", path); - return false; - } - - Library.Utility.IFilter match; - var filtermatch = false; - if (!Library.Utility.FilterExpression.Matches(m_enumeratefilter, path, out match)) - { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromFilter", "Excluding path due to filter: {0} => {1}", path, match == null ? "null" : match.ToString()); - return false; - } - else if (match != null) - { - filtermatch = true; - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingPathFromFilter", "Including path due to filter: {0} => {1}", path, match.ToString()); - } - - var isSymlink = m_snapshot.IsSymlink(path, attributes); - if (isSymlink && m_symlinkPolicy == Options.SymlinkStrategy.Ignore) - { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludeSymlink", "Excluding symlink: {0}", path); - return false; - } - - if (isSymlink && m_symlinkPolicy == Options.SymlinkStrategy.Store) - { - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "StoreSymlink", "Storing symlink: {0}", path); - - m_mixinqueue.Enqueue(path); - return false; - } - - if (!filtermatch) - Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingPath", "Including path as no filters matched: {0}", path); - - return true; - } - - public IEnumerable EnumerateFilesAndFolders() - { - foreach(var s in m_snapshot.EnumerateFilesAndFolders(this.AttributeFilter, this.ReportError)) - { - while (m_mixinqueue.Count > 0) - yield return m_mixinqueue.Dequeue(); - - Library.Utility.IFilter m; - if (m_emitfilter != m_enumeratefilter && !Library.Utility.FilterExpression.Matches(m_emitfilter, s, out m)) - continue; - - yield return s; - } - - while (m_mixinqueue.Count > 0) - yield return m_mixinqueue.Dequeue(); - } - - public IEnumerable Mixin(IEnumerable list) - { - foreach(var s in list.Where(x => { - var fa = FileAttributes.Normal; - try { fa = m_snapshot.GetAttributes(x); } - catch(Exception ex) { Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", x, ex.Message); } - - return AttributeFilter(null, x, fa); - })) - { - while (m_mixinqueue.Count > 0) - yield return m_mixinqueue.Dequeue(); - - yield return s; - } - - while (m_mixinqueue.Count > 0) - yield return m_mixinqueue.Dequeue(); - } - } - public static Snapshots.ISnapshotService GetSnapshot(string[] sources, Options options) + public static ISnapshotService GetSnapshot(IEnumerable sources, Options options) { try { if (options.SnapShotStrategy != Options.OptimizationStrategy.Off) - return Duplicati.Library.Snapshots.SnapshotUtility.CreateSnapshot(sources, options.RawOptions); + return SnapshotUtility.CreateSnapshot(sources, options.RawOptions); } catch (Exception ex) { - if (options.SnapShotStrategy == Options.OptimizationStrategy.Required) - throw; - else if (options.SnapShotStrategy == Options.OptimizationStrategy.On) - Logging.Log.WriteWarningMessage(LOGTAG, "SnapshotFailed", ex, Strings.Common.SnapshotFailedError(ex.ToString())); + switch (options.SnapShotStrategy) + { + case Options.OptimizationStrategy.Required: + throw; + case Options.OptimizationStrategy.On: + Logging.Log.WriteWarningMessage(LOGTAG, "SnapshotFailed", ex, Strings.Common.SnapshotFailedError(ex.ToString())); + break; + } } - return Library.Utility.Utility.IsClientLinux ? - (Library.Snapshots.ISnapshotService)new Duplicati.Library.Snapshots.NoSnapshotLinux(sources, options.RawOptions) - : - (Library.Snapshots.ISnapshotService)new Duplicati.Library.Snapshots.NoSnapshotWindows(sources, options.RawOptions); + return Library.Utility.Utility.IsClientLinux ? new NoSnapshotLinux() : (ISnapshotService) new NoSnapshotWindows(); } private void CountFilesThread(object state) { + // ReSharper disable once InconsistentNaming var COUNTER_LOGTAG = LOGTAG + ".Counter"; - var snapshot = (Snapshots.ISnapshotService)state; - var updater = m_result.OperationProgressUpdater; + var typedState = (Tuple) state; + var snapshot = typedState.Item1; + var sources = typedState.Item2; var count = 0L; var size = 0L; - var followSymlinks = m_options.SymlinkPolicy != Duplicati.Library.Main.Options.SymlinkStrategy.Follow; + var followSymlinks = m_options.SymlinkPolicy != Options.SymlinkStrategy.Follow; - foreach(var path in new FilterHandler(snapshot, m_attributeFilter, m_sourceFilter, m_filter, m_symlinkPolicy, m_options.HardlinkPolicy).EnumerateFilesAndFolders()) + foreach(var path in new FilterHandler(snapshot, m_attributeFilter, m_sourceFilter, m_filter, m_symlinkPolicy, m_options.HardlinkPolicy).EnumerateFilesAndFolders(sources)) { - var fa = FileAttributes.Normal; - try { fa = snapshot.GetAttributes(path); } - catch (Exception ex) { Logging.Log.WriteVerboseMessage(COUNTER_LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", path, ex.Message); } + var fa = SafeGetFileAttributes(path, snapshot, COUNTER_LOGTAG); if (followSymlinks && snapshot.IsSymlink(path, fa)) continue; - else if ((fa & FileAttributes.Directory) == FileAttributes.Directory) + if (fa.HasFlag(FileAttributes.Directory)) continue; - + count++; try { size += snapshot.GetFileSize(path); } @@ -324,7 +169,6 @@ namespace Duplicati.Library.Main.Operation { var s = 1; var fileTime = incompleteSet.Value + TimeSpan.FromSeconds(s); - var oldFilesetID = incompleteSet.Key; // Probe for an unused filename while (s < 60) @@ -340,14 +184,14 @@ namespace Duplicati.Library.Main.Operation fsw.VolumeID = m_database.RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, m_transaction); if (!string.IsNullOrEmpty(m_options.ControlFiles)) - foreach(var p in m_options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) + foreach(var p in m_options.ControlFiles.Split(new[] { Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) fsw.AddControlFile(p, m_options.GetCompressionHintFromFilename(p)); - var newFilesetID = m_database.CreateFileset(fsw.VolumeID, fileTime, trn); - m_database.LinkFilesetToVolume(newFilesetID, fsw.VolumeID, trn); - m_database.AppendFilesFromPreviousSet(trn, null, newFilesetID, prevId, fileTime); + var newFilesetId = m_database.CreateFileset(fsw.VolumeID, fileTime, trn); + m_database.LinkFilesetToVolume(newFilesetId, fsw.VolumeID, trn); + m_database.AppendFilesFromPreviousSet(trn, null, newFilesetId, prevId, fileTime); - m_database.WriteFileset(fsw, trn, newFilesetID); + m_database.WriteFileset(fsw, trn, newFilesetId); if (m_options.Dryrun) { @@ -367,8 +211,13 @@ namespace Duplicati.Library.Main.Operation finally { if (fsw != null) + { try { fsw.Dispose(); } - catch { fsw = null; } + catch + { + // ignored + } + } } } } @@ -404,10 +253,10 @@ namespace Duplicati.Library.Main.Operation m_database.AddIndexBlockLink(w.VolumeID, volumeid, null); if (m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUpdateIndexFile", "Would upload new index file {0}, with size {1}, previous size {2}", w.RemoteFilename, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(w.Filesize)); + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUpdateIndexFile", "Would upload new index file {0}, with size {1}, previous size {2}", w.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(w.Filesize)); else { - m_database.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null); + m_database.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null); backend.Put(w); } } @@ -446,60 +295,207 @@ namespace Duplicati.Library.Main.Operation } } - private void RunMainOperation(Snapshots.ISnapshotService snapshot, BackendManager backend) + /// + /// Perform actual backup operation + /// + /// Snapshot service to use + /// Storage back-end + /// Sources to back up + private void RunMainOperation(ISnapshotService snapshot, BackendManager backend, IEnumerable sources) { var filterhandler = new FilterHandler(snapshot, m_attributeFilter, m_sourceFilter, m_filter, m_symlinkPolicy, m_options.HardlinkPolicy); using(new Logging.Timer(LOGTAG, "BackupMainOperation", "BackupMainOperation")) { - if (m_options.ChangedFilelist != null && m_options.ChangedFilelist.Length >= 1) + if (m_options.ChangedFilelist != null && m_options.ChangedFilelist.Any()) { - Logging.Log.WriteVerboseMessage(LOGTAG, "UsingChangeList", "Processing supplied change list instead of enumerating filesystem"); - m_result.OperationProgressUpdater.UpdatefileCount(m_options.ChangedFilelist.Length, 0, true); - - foreach(var p in filterhandler.Mixin(m_options.ChangedFilelist)) - { - if (m_result.TaskControlRendevouz() == TaskControlState.Stop) - { - Logging.Log.WriteInformationMessage(LOGTAG, "StopBackup", "Stopping backup operation on request"); - break; - } - - try - { - this.HandleFilesystemEntry(snapshot, backend, p, snapshot.GetAttributes(p)); - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "FailedToProcessEntry", ex, "Failed to process element: {0}, message: {1}", p, ex.Message); - } - } - - m_database.AppendFilesFromPreviousSet(m_transaction, m_options.DeletedFilelist); + BackupUsingChangedFileList(snapshot, backend, m_options.ChangedFilelist, filterhandler); + } + else if (m_options.UsnStrategy != Options.OptimizationStrategy.Off) + { + BackupUsingChangeJournal(snapshot, backend, sources, filterhandler); } else - { - foreach(var path in filterhandler.EnumerateFilesAndFolders()) - { - if (m_result.TaskControlRendevouz() == TaskControlState.Stop) - { - Logging.Log.WriteInformationMessage(LOGTAG, "StopBackup", "Stopping backup operation on request"); - break; - } - - var fa = FileAttributes.Normal; - try { fa = snapshot.GetAttributes(path); } - catch (Exception ex) { Logging.Log.WriteVerboseMessage(LOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", path, ex.Message); } - - this.HandleFilesystemEntry(snapshot, backend, path, fa); - } - + { + BackupUsingFullScan(snapshot, backend, sources, filterhandler); } m_result.OperationProgressUpdater.UpdatefileCount(m_result.ExaminedFiles, m_result.SizeOfExaminedFiles, true); } } + /// + /// Performs backup operation using supplied changed file list + /// + /// Snapshot service to use + /// Storage back-end + /// List of changed file / folders + /// Exclusion filter + private void BackupUsingChangedFileList(ISnapshotService snapshot, BackendManager backend, IReadOnlyCollection changedFileList, FilterHandler filterhandler) + { + var existingSources = changedFileList.Where(snapshot.PathExists); + + Logging.Log.WriteVerboseMessage(LOGTAG, "UsingChangeList", + "Processing supplied change list instead of enumerating filesystem"); + m_result.OperationProgressUpdater.UpdatefileCount(changedFileList.Count, 0, true); + + foreach (var p in filterhandler.Mixin(existingSources)) + { + if (m_result.TaskControlRendevouz() == TaskControlState.Stop) + { + Logging.Log.WriteInformationMessage(LOGTAG, "StopBackup", "Stopping backup operation on request"); + break; + } + + try + { + HandleFilesystemEntry(snapshot, backend, p, snapshot.GetAttributes(p)); + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "FailedToProcessEntry", ex, + "Failed to process element: {0}, message: {1}", p, ex.Message); + } + } + + m_database.AppendFilesFromPreviousSet(m_transaction, m_options.DeletedFilelist); + } + + /// + /// Performs backup operation by scanning supplied source list for changes + /// + /// Snapshot service to use + /// Storage back-end + /// Sources to back up + /// Exclusion filter + private void BackupUsingFullScan(ISnapshotService snapshot, BackendManager backend, IEnumerable sources, + FilterHandler filterhandler) + { + // [DG] I believe that we need to sanitize the sources to ensure that they exist in the *snapshot*. + // Controller.ExpandInputSources() only verifies if sources exist in the non-snapshot filesystem, + // which may or may not correspond to their state in the snapshot. + var existingSources = sources.Where(snapshot.PathExists); + + foreach (var path in filterhandler.EnumerateFilesAndFolders(existingSources)) + { + if (m_result.TaskControlRendevouz() == TaskControlState.Stop) + { + Logging.Log.WriteInformationMessage(LOGTAG, "StopBackup", + "Stopping backup operation on request"); + break; + } + + var fa = SafeGetFileAttributes(path, snapshot, LOGTAG); + HandleFilesystemEntry(snapshot, backend, path, fa); + } + } + + /// + /// Performs backup operation using USN Journal + /// + /// Snapshot service to use + /// Storage back-end + /// Sources to back up + /// Exclusion filter + private void BackupUsingChangeJournal(ISnapshotService snapshot, BackendManager backend, IEnumerable sources, FilterHandler filterhandler) + { + try + { + var journalService = new UsnJournalService(UsnErrorHandler, () => + { + if (m_result.TaskControlRendevouz() == TaskControlState.Stop) + throw new OperationCanceledException(); + }); + + // obtain journal data from previous fileset + var journalData = m_database.GetChangeJournalData(m_prevFilesetId); + + // get filter identifying current source filter / sources configuration + // ReSharper disable once PossibleMultipleEnumeration + var configHash = m_filter.GetFilterHash() + Library.Utility.MD5HashHelper.GetHashString(Library.Utility.MD5HashHelper.GetHash(sources)); + + // filter sources using USN journal, to obtain a sub-set of files / folders that + // may have been modified + // ReSharper disable once PossibleMultipleEnumeration + var modifiedSources = journalService.FilterSources(sources, configHash, journalData); + + // scan modified sources for modifications + if (modifiedSources.Files.Any() || modifiedSources.Folders.Any()) + { + // the modified sources may contain non-existent files / folders, which need to + // be removed before trying to enumerate them + var existingModifiedSources = modifiedSources.Files.Where(snapshot.FileExists) + .Concat(modifiedSources.Folders.Where(snapshot.FolderExists)); + + foreach (var path in filterhandler.EnumerateFilesAndFolders(existingModifiedSources)) + { + if (m_result.TaskControlRendevouz() == TaskControlState.Stop) + throw new OperationCanceledException(); + + var fa = SafeGetFileAttributes(path, snapshot, LOGTAG); + HandleFilesystemEntry(snapshot, backend, path, fa); + } + + // append files from previous fileset, unless part of modifiedSources, which we've just scanned + //TODO: we also need to check the original sources and exclude if not contained in those! + m_database.AppendFilesFromPreviousSetWithPredicate(m_transaction, path => + { + if (modifiedSources.Files.Contains(path)) + return true; // do not append from previous set, already scanned + + foreach (var folder in modifiedSources.Folders) + { + if (path.Equals(folder, Library.Utility.Utility.ClientFilenameStringComparision)) + return true; // do not append from previous set, already scanned + + if (Library.Utility.Utility.IsPathBelowFolder(path, folder)) + return true; // do not append from previous set, already scanned + } + + return false; // append from previous set + }); + } + else + { + // add full previous fileset, as nothing has changed + m_database.AppendFilesFromPreviousSet(m_transaction); + } + + if (modifiedSources.JournalData.Any()) + { + // always record change journal data for current fileset (entry may be dropped later if nothing is uploaded) + m_database.CreateChangeJournalData(modifiedSources.JournalData, m_transaction); + + // update the previous fileset's change journal entry to resume at this point in case nothing was backed up + m_database.UpdateChangeJournalData(modifiedSources.JournalData, m_prevFilesetId, m_transaction); + } + } + catch (OperationCanceledException) + { + Logging.Log.WriteInformationMessage(LOGTAG, "StopBackup", "Stopping backup operation on request"); + } + } + + /// + /// Error callback handler + /// + /// Volume name + /// Not used + /// Exception message + private void UsnErrorHandler(string volume, string path, Exception ex) + { + if (m_options.UsnStrategy == Options.OptimizationStrategy.Auto) + { + Logging.Log.WriteInformationMessage(LOGTAG, "FailedToUseChangeJournal", "Failed to use change journal for volume \"{0}\": {1}", volume, ex.Message); + } + else if (m_options.UsnStrategy == Options.OptimizationStrategy.On) + { + Logging.Log.WriteErrorMessage(LOGTAG, "FailedToUseChangeJournal", ex, "Failed to use change journal for volume \"{0}\": {1}", volume, ex.Message); + } + else + throw ex; + } + private long FinalizeRemoteVolumes(BackendManager backend) { var lastVolumeSize = -1L; @@ -523,7 +519,7 @@ namespace Duplicati.Library.Main.Operation m_blockvolume.Dispose(); m_blockvolume = null; - m_indexvolume.Dispose(); + m_indexvolume?.Dispose(); m_indexvolume = null; } else @@ -564,7 +560,7 @@ namespace Duplicati.Library.Main.Operation using(new Logging.Timer(LOGTAG, "UploadNewFileset", "Uploading a new fileset")) { if (!string.IsNullOrEmpty(m_options.ControlFiles)) - foreach(var p in m_options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) + foreach(var p in m_options.ControlFiles.Split(new[] { Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) filesetvolume.AddControlFile(p, m_options.GetCompressionHintFromFilename(p)); m_database.WriteFileset(filesetvolume, m_transaction); @@ -627,7 +623,7 @@ namespace Duplicati.Library.Main.Operation backend.WaitForComplete(m_database, null); } - if (m_options.BackupTestSampleCount > 0 && m_database.GetRemoteVolumes().Count() > 0) + if (m_options.BackupTestSampleCount > 0 && m_database.GetRemoteVolumes().Any()) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupTest); m_result.TestResults = new TestResults(m_result); @@ -658,9 +654,9 @@ namespace Duplicati.Library.Main.Operation // TODO: If we have a BackendManager, we should query through that using (var backend = DynamicLoader.BackendLoader.GetBackend(m_backendurl, m_options.RawOptions)) { - if (backend is Library.Interface.IQuotaEnabledBackend) + if (backend is IQuotaEnabledBackend enabledBackend) { - Library.Interface.IQuotaInfo quota = ((Library.Interface.IQuotaEnabledBackend)backend).Quota; + var quota = enabledBackend.Quota; if (quota != null) { m_result.BackendWriter.TotalQuotaSpace = quota.TotalQuotaSpace; @@ -670,7 +666,8 @@ namespace Duplicati.Library.Main.Operation } } - m_result.BackendWriter.AssignedQuotaSpace = m_options.QuotaSize; + if (m_result.BackendWriter != null) + m_result.BackendWriter.AssignedQuotaSpace = m_options.QuotaSize; } public void Run(string[] sources, Library.Utility.IFilter filter) @@ -685,9 +682,10 @@ namespace Duplicati.Library.Main.Operation Utility.UpdateOptionsFromDb(m_database, m_options); Utility.VerifyParameters(m_database, m_options); - var probe_path = m_database.GetFirstPath(); - if (probe_path != null && Duplicati.Library.Utility.Utility.GuessDirSeparator(probe_path) != System.IO.Path.DirectorySeparatorChar.ToString()) - throw new UserInformationException(string.Format("The backup contains files that belong to another operating system. Proceeding with a backup would cause the database to contain paths from two different operation systems, which is not supported. To proceed without losing remote data, delete all filesets and make sure the --{0} option is set, then run the backup again to re-use the existing data on the remote store.", "no-auto-compact"), "CrossOsDatabaseReuseNotSupported"); + var probePath = m_database.GetFirstPath(); + if (probePath != null && Library.Utility.Utility.GuessDirSeparator(probePath) != Path.DirectorySeparatorChar.ToString()) + throw new UserInformationException( + "The backup contains files that belong to another operating system. Proceeding with a backup would cause the database to contain paths from two different operation systems, which is not supported. To proceed without losing remote data, delete all filesets and make sure the --no-auto-compact option is set, then run the backup again to re-use the existing data on the remote store.", "CrossOsDatabaseReuseNotSupported"); if (m_database.PartiallyRecreated) throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated"); @@ -718,7 +716,7 @@ namespace Duplicati.Library.Main.Operation // If there is no filter, we set an empty filter to simplify the code // If there is a filter, we make sure that the sources are included m_filter = filter ?? new Library.Utility.FilterExpression(); - m_sourceFilter = new Library.Utility.FilterExpression(sources, true); + m_sourceFilter = new Library.Utility.FilterExpression(sources); m_backendLogFlushTimer = DateTime.Now.Add(FLUSH_TIMESPAN); System.Threading.Thread parallelScanner = null; @@ -745,7 +743,7 @@ namespace Duplicati.Library.Main.Operation Name = "Read ahead file counter", IsBackground = true }; - parallelScanner.Start(snapshot); + parallelScanner.Start(Tuple.Create(snapshot, sources)); } } @@ -791,9 +789,10 @@ namespace Duplicati.Library.Main.Operation var prevfileset = m_database.FilesetTimes.FirstOrDefault(); if (prevfileset.Value.ToUniversalTime() > m_database.OperationTimestamp.ToUniversalTime()) - throw new UserInformationException(string.Format("The previous backup has time {0}, but this backup has time {1}. Something is wrong with the clock.", prevfileset.Value.ToLocalTime(), m_database.OperationTimestamp.ToLocalTime()), "FilesetIntervalClockIssue"); + throw new UserInformationException( + $"The previous backup has time {prevfileset.Value.ToLocalTime()}, but this backup has time {m_database.OperationTimestamp.ToLocalTime()}. Something is wrong with the clock.", "FilesetIntervalClockIssue"); - m_lastfilesetid = prevfileset.Value.Ticks == 0 ? -1 : prevfileset.Key; + m_prevFilesetId = prevfileset.Value.Ticks == 0 ? -1 : prevfileset.Key; // Rebuild any index files that are missing RecreateMissingIndexFiles(backend); @@ -812,7 +811,7 @@ namespace Duplicati.Library.Main.Operation var filesetvolumeid = m_database.RegisterRemoteVolume(filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, m_transaction); m_database.CreateFileset(filesetvolumeid, VolumeBase.ParseFilename(filesetvolume.RemoteFilename).Time, m_transaction); - RunMainOperation(snapshot, backend); + RunMainOperation(snapshot, backend, sources); //If the scanner is still running for some reason, make sure we kill it now if (parallelScanner != null && parallelScanner.IsAlive) @@ -872,7 +871,6 @@ namespace Duplicati.Library.Main.Operation m_database.Vacuum(); } m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Complete); - return; } } catch (Exception ex) @@ -897,18 +895,18 @@ namespace Duplicati.Library.Main.Operation } } - private Dictionary GenerateMetadata(Snapshots.ISnapshotService snapshot, string path, System.IO.FileAttributes attributes) + private Dictionary GenerateMetadata(ISnapshotService snapshot, string path, FileAttributes attributes) { - string METALOGTAG = LOGTAG + ".Metadata"; + // ReSharper disable once InconsistentNaming + var METALOGTAG = LOGTAG + ".Metadata"; try { Dictionary metadata; if (m_options.StoreMetadata) { - metadata = snapshot.GetMetadata(path, attributes.HasFlag(System.IO.FileAttributes.ReparsePoint), m_symlinkPolicy == Options.SymlinkStrategy.Follow); - if (metadata == null) - metadata = new Dictionary(); + metadata = snapshot.GetMetadata(path, attributes.HasFlag(FileAttributes.ReparsePoint), m_symlinkPolicy == Options.SymlinkStrategy.Follow) ?? + new Dictionary(); if (!metadata.ContainsKey("CoreAttributes")) metadata["CoreAttributes"] = attributes.ToString(); @@ -951,9 +949,17 @@ namespace Duplicati.Library.Main.Operation } } - private bool HandleFilesystemEntry(Snapshots.ISnapshotService snapshot, BackendManager backend, string path, System.IO.FileAttributes attributes) + /// + /// Add path to backup if it has been modified, and isn't included by exclusion filters + /// + /// Snapshot handler + /// Backend + /// File / folder path to add (non-recursively!) + /// File attributes + private void HandleFilesystemEntry(ISnapshotService snapshot, BackendManager backend, string path, FileAttributes attributes) { - string FILELOGTAG = LOGTAG + ".FileEntry"; + // ReSharper disable once InconsistentNaming + var FILELOGTAG = LOGTAG + ".FileEntry"; // If we lost the connection, there is no point in keeping on processing if (backend.HasDied) @@ -969,7 +975,7 @@ namespace Duplicati.Library.Main.Operation backend.FlushDbMessages(m_database, null); } - DateTime lastwrite = new DateTime(0, DateTimeKind.Utc); + var lastwrite = new DateTime(0, DateTimeKind.Utc); try { lastwrite = snapshot.GetLastWriteTimeUtc(path); @@ -979,7 +985,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteWarningMessage(FILELOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", path); } - if ((attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint) + if (attributes.HasFlag(FileAttributes.ReparsePoint)) { // Not all reparse points are symlinks. // For example, on Windows 10 Fall Creator's Update, the OneDrive folder (and all subfolders) @@ -991,7 +997,7 @@ namespace Duplicati.Library.Main.Operation if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Ignore) { Logging.Log.WriteVerboseMessage(FILELOGTAG, "IgnoreSymlink", "Ignoring symlink {0}", path); - return false; + return; } if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Store) @@ -1008,7 +1014,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteVerboseMessage(FILELOGTAG, "StoreSymlink", "Stored symlink {0}", path); //Do not recurse symlinks - return false; + return; } } else @@ -1017,22 +1023,13 @@ namespace Duplicati.Library.Main.Operation } } - if ((attributes & FileAttributes.Directory) == FileAttributes.Directory) + if (attributes.HasFlag(FileAttributes.Directory)) { - IMetahash metahash; - - if (m_options.StoreMetadata) - { - metahash = Utility.WrapMetadata(GenerateMetadata(snapshot, path, attributes), m_options); - } - else - { - metahash = EMPTY_METADATA; - } + var metahash = m_options.StoreMetadata ? Utility.WrapMetadata(GenerateMetadata(snapshot, path, attributes), m_options) : EMPTY_METADATA; Logging.Log.WriteVerboseMessage(FILELOGTAG, "AddDirectory", "Adding directory {0}", path); AddFolderToOutput(backend, path, lastwrite, metahash); - return true; + return; } m_result.OperationProgressUpdater.UpdatefilesProcessed(++m_result.ExaminedFiles, m_result.SizeOfExaminedFiles); @@ -1057,11 +1054,11 @@ namespace Duplicati.Library.Main.Operation lastFileSize = filestatsize; oldMetahash = metahashandsize.FileHash; oldMetasize = metahashandsize.Blob.Length; - oldId = m_database.GetFileLastModified(path, m_lastfilesetid, out oldModified); + oldId = m_database.GetFileLastModified(path, m_prevFilesetId, out oldModified); } else { - oldId = m_database.GetFileEntry(path, m_lastfilesetid, out oldModified, out lastFileSize, out oldMetahash, out oldMetasize); + oldId = m_database.GetFileEntry(path, m_prevFilesetId, out oldModified, out lastFileSize, out oldMetahash, out oldMetasize); } var timestampChanged = lastwrite != oldModified || lastwrite.Ticks == 0 || oldModified.Ticks == 0; @@ -1074,8 +1071,6 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteVerboseMessage(FILELOGTAG, "CheckFileForChanges", "Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", path, oldId <= 0, timestampChanged, filesizeChanged, metadatachanged, lastwrite, oldModified); m_result.OpenedFiles++; - - long filesize = 0; var hint = m_options.GetCompressionHintFromFilename(path); var oldHash = oldId < 0 ? null : m_database.GetFileHash(oldId); @@ -1083,13 +1078,13 @@ namespace Duplicati.Library.Main.Operation using (var blocklisthashes = new Library.Utility.FileBackedStringList()) using (var hashcollector = new Library.Utility.FileBackedStringList()) { + long filesize; using (var fs = snapshot.OpenRead(path)) { try { m_result.OperationProgressUpdater.StartFile(path, fs.Length); } catch (Exception ex) { Logging.Log.WriteWarningMessage(FILELOGTAG, "FileLengthFailure", ex, "Failed to read file length for file {0}", path); } - if ((filesize = ProcessStream(fs, hint, backend, blocklisthashes, hashcollector, false)) < 0) - return false; + if ((filesize = ProcessStream(fs, hint, backend, blocklisthashes, hashcollector, false)) < 0) return; } m_result.SizeOfOpenedFiles += filesize; @@ -1130,7 +1125,6 @@ namespace Duplicati.Library.Main.Operation else { // When we write the file to output, update the last modified time - oldModified = lastwrite; Logging.Log.WriteVerboseMessage(FILELOGTAG, "NoFileChanges", "File has not changed {0}", path); } } @@ -1159,16 +1153,16 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteWarningMessage(FILELOGTAG, "PathProcessingFailed", ex, "Failed to process path: {0}", path); m_result.FilesWithError++; } - - return true; } - private long ProcessStream(System.IO.Stream stream, Library.Interface.CompressionHint hint, BackendManager backend, Library.Utility.FileBackedStringList blocklisthashes, Library.Utility.FileBackedStringList hashcollector, bool skipfilehash) + private long ProcessStream(Stream stream, CompressionHint hint, BackendManager backend, + Library.Utility.FileBackedStringList blocklisthashes, Library.Utility.FileBackedStringList hashcollector, + bool skipfilehash) { int blocklistoffset = 0; long filesize = 0; - using(var fs = new Blockprocessor(stream, m_blockbuffer)) + using (var fs = new Blockprocessor(stream, m_blockbuffer)) { m_filehasher.Initialize(); @@ -1181,13 +1175,15 @@ namespace Duplicati.Library.Main.Operation if (!skipfilehash) m_filehasher.TransformBlock(m_blockbuffer, offset, size, m_blockbuffer, offset); - + var blockkey = m_blockhasher.ComputeHash(m_blockbuffer, offset, size); if (m_blocklistbuffer.Length - blocklistoffset < blockkey.Length) { - var blkey = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset)); + var blkey = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, + blocklistoffset)); blocklisthashes.Add(blkey); - AddBlockToOutput(backend, blkey, m_blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true); + AddBlockToOutput(backend, blkey, m_blocklistbuffer, 0, blocklistoffset, + CompressionHint.Noncompressible, true); blocklistoffset = 0; } @@ -1201,7 +1197,7 @@ namespace Duplicati.Library.Main.Operation if (!skipfilehash) { - m_result.OperationProgressUpdater.UpdateFileProgress(filesize); + m_result.OperationProgressUpdater.UpdateFileProgress(filesize); if (m_result.TaskControlRendevouz() == TaskControlState.Stop) return -1; } @@ -1220,9 +1216,11 @@ namespace Duplicati.Library.Main.Operation //If all fits in a single block, don't bother with blocklists if (hashcollector.Count > 1) { - var blkeyfinal = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset)); + var blkeyfinal = + Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset)); blocklisthashes.Add(blkeyfinal); - AddBlockToOutput(backend, blkeyfinal, m_blocklistbuffer, 0, blocklistoffset, CompressionHint.Noncompressible, true); + AddBlockToOutput(backend, blkeyfinal, m_blocklistbuffer, 0, blocklistoffset, + CompressionHint.Noncompressible, true); } } @@ -1236,13 +1234,15 @@ namespace Duplicati.Library.Main.Operation /// /// Adds the found file data to the output unless the block already exists /// + /// /// The block hash /// The data matching the hash - /// The size of the data /// The offset into the data + /// The size of the data /// Hint for compression module /// Indicates if the block is list data - private bool AddBlockToOutput(BackendManager backend, string key, byte[] data, int offset, int len, CompressionHint hint, bool isBlocklistData) + private void AddBlockToOutput(BackendManager backend, string key, byte[] data, int offset, int len, + CompressionHint hint, bool isBlocklistData) { if (m_blockvolume == null) { @@ -1311,11 +1311,7 @@ namespace Duplicati.Library.Main.Operation } } - - return true; } - - return false; } private void AddUnmodifiedFile(long oldId, DateTime lastModified) @@ -1354,6 +1350,7 @@ namespace Duplicati.Library.Main.Operation /// /// Adds a file to the output, /// + /// /// The name of the file to record /// The value of the lastModified timestamp /// A lookup table with various metadata values describing the file @@ -1366,6 +1363,7 @@ namespace Duplicati.Library.Main.Operation /// /// Adds a file to the output, /// + /// Storage backend manager /// The name of the file to record /// The value of the lastModified timestamp /// A lookup table with various metadata values describing the file @@ -1378,17 +1376,18 @@ namespace Duplicati.Library.Main.Operation /// /// Adds a file to the output, /// + /// Storage backend manager /// The name of the file to record /// The value of the lastModified timestamp /// The list of hashes that make up the file /// The size of the file /// A lookup table with various metadata values describing the file + /// File hash + /// private void AddFileToOutput(BackendManager backend, string filename, long size, DateTime lastmodified, IMetahash metadata, IEnumerable hashlist, string filehash, IEnumerable blocklisthashes) { - long blocksetid; - var metadataid = AddMetadataToOutput(backend, metadata); - m_database.AddBlockset(filehash, size, m_blocksize, hashlist, blocklisthashes, out blocksetid, m_transaction); + m_database.AddBlockset(filehash, size, m_blocksize, hashlist, blocklisthashes, out var blocksetid, m_transaction); m_database.AddFile(filename, lastmodified, blocksetid, metadataid, m_transaction); } @@ -1407,6 +1406,22 @@ namespace Duplicati.Library.Main.Operation } } + internal static FileAttributes SafeGetFileAttributes(string path, ISnapshotService snapshot, string tag) + { + var fa = FileAttributes.Normal; + try + { + fa = snapshot.GetAttributes(path); + } + catch (Exception ex) + { + Logging.Log.WriteVerboseMessage(tag, "FailedAttributeRead", + "Failed to read attributes from {0}: {1}", path, ex.Message); + } + + return fa; + } + public void Dispose() { if (m_blockvolume != null) @@ -1427,4 +1442,188 @@ namespace Duplicati.Library.Main.Operation m_result.EndTime = DateTime.UtcNow; } } + + internal sealed class FilterHandler + { + private static readonly string FILTER_LOGTAG = Logging.Log.LogTagFromType(); + private readonly ISnapshotService m_snapshot; + private readonly FileAttributes m_attributeFilter; + private readonly Library.Utility.IFilter m_enumeratefilter; + private readonly Library.Utility.IFilter m_emitfilter; + private readonly Options.SymlinkStrategy m_symlinkPolicy; + private readonly Options.HardlinkStrategy m_hardlinkPolicy; + private readonly Dictionary m_hardlinkmap; + private readonly Library.Utility.IFilter m_sourcefilter; + private readonly Queue m_mixinqueue; + + public FilterHandler(ISnapshotService snapshot, FileAttributes attributeFilter, + Library.Utility.IFilter sourcefilter, Library.Utility.IFilter filter, + Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy) + { + m_snapshot = snapshot; + m_attributeFilter = attributeFilter; + m_sourcefilter = sourcefilter; + m_emitfilter = filter; + m_symlinkPolicy = symlinkPolicy; + m_hardlinkPolicy = hardlinkPolicy; + m_hardlinkmap = new Dictionary(); + m_mixinqueue = new Queue(); + + Library.Utility.FilterExpression.AnalyzeFilters(filter, out var includes, out var excludes); + if (includes && !excludes) + { + m_enumeratefilter = Library.Utility.FilterExpression.Combine(filter, + new Library.Utility.FilterExpression("*" + Path.DirectorySeparatorChar)); + } + else + { + m_enumeratefilter = m_emitfilter; + } + } + + public void ReportError(string rootpath, string path, Exception ex) + { + Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FileAccessError", ex, "Error reported while accessing file: {0}", path); + } + + public bool AttributeFilter(string rootpath, string path, FileAttributes attributes) + { + try + { + if (m_snapshot.IsBlockDevice(path)) + { + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingBlockDevice", "Excluding block device: {0}", path); + return false; + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingError", ex, "Failed to process path: {0}", path); + return false; + } + + if (m_sourcefilter.Matches(path, out var sourcematches, out _) && sourcematches) + { + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingSourcePath", "Including source path: {0}", path); + + return true; + } + + if (m_hardlinkPolicy != Options.HardlinkStrategy.All) + { + try + { + var id = m_snapshot.HardlinkTargetID(path); + if (id != null) + { + switch (m_hardlinkPolicy) + { + case Options.HardlinkStrategy.None: + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingHardlinkByPolicy", "Excluding hardlink: {0} ({1})", path, id); + return false; + + case Options.HardlinkStrategy.First: + if (m_hardlinkmap.TryGetValue(id, out var prevPath)) + { + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingDuplicateHardlink", "Excluding hardlink ({1}) for: {0}, previous hardlink: {2}", path, id, prevPath); + return false; + } + else + { + m_hardlinkmap.Add(id, path); + } + + break; + } + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingError", ex, "Failed to process path: {0}", path); + return false; + } + } + + if ((m_attributeFilter & attributes) != 0) + { + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromAttributes", "Excluding path due to attribute filter: {0}", path); + return false; + } + + var filtermatch = false; + if (!Library.Utility.FilterExpression.Matches(m_enumeratefilter, path, out var match)) + { + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludingPathFromFilter", "Excluding path due to filter: {0} => {1}", path, match == null ? "null" : match.ToString()); + return false; + } + + if (match != null) + { + filtermatch = true; + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingPathFromFilter", "Including path due to filter: {0} => {1}", path, match.ToString()); + } + + var isSymlink = m_snapshot.IsSymlink(path, attributes); + if (isSymlink && m_symlinkPolicy == Options.SymlinkStrategy.Ignore) + { + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "ExcludeSymlink", "Excluding symlink: {0}", path); + return false; + } + + if (isSymlink && m_symlinkPolicy == Options.SymlinkStrategy.Store) + { + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "StoreSymlink", "Storing symlink: {0}", path); + + m_mixinqueue.Enqueue(path); + return false; + } + + if (!filtermatch) + Logging.Log.WriteVerboseMessage(FILTER_LOGTAG, "IncludingPath", "Including path as no filters matched: {0}", path); + + return true; + } + + /// + /// Enumerate files and folders (recursively) in sources, according to active attribute and exclusion filters. + /// + /// Sources to enumerate + /// + public IEnumerable EnumerateFilesAndFolders(IEnumerable sources) + { + foreach(var s in m_snapshot.EnumerateFilesAndFolders(sources, AttributeFilter, ReportError)) + { + // emit any paths that were added by the enumeration itself (symlink targets, ...) + while (m_mixinqueue.Any()) + yield return m_mixinqueue.Dequeue(); + + if (m_emitfilter != m_enumeratefilter && !Library.Utility.FilterExpression.Matches(m_emitfilter, s, out _)) + continue; + + yield return s; + } + + // emit any paths that were added by the enumeration itself (symlink targets, ...) + while (m_mixinqueue.Any()) + yield return m_mixinqueue.Dequeue(); + } + + public IEnumerable Mixin(IEnumerable list) + { + foreach (var s in list.Where(x => + { + var fa = BackupHandler.SafeGetFileAttributes(x, m_snapshot, FILTER_LOGTAG); + return AttributeFilter(null, x, fa); + })) + { + while (m_mixinqueue.Count > 0) + yield return m_mixinqueue.Dequeue(); + + yield return s; + } + + while (m_mixinqueue.Count > 0) + yield return m_mixinqueue.Dequeue(); + } + } } diff --git a/Duplicati/Library/Main/Operation/TestFilterHandler.cs b/Duplicati/Library/Main/Operation/TestFilterHandler.cs index 82d204563..48391fdc2 100644 --- a/Duplicati/Library/Main/Operation/TestFilterHandler.cs +++ b/Duplicati/Library/Main/Operation/TestFilterHandler.cs @@ -29,7 +29,7 @@ namespace Duplicati.Library.Main.Operation private static readonly string LOGTAG = Logging.Log.LogTagFromType(); private readonly Options m_options; - private TestFilterResults m_result; + private readonly TestFilterResults m_result; public TestFilterHandler(Options options, TestFilterResults results) { @@ -39,12 +39,12 @@ namespace Duplicati.Library.Main.Operation public void Run(string[] sources, Library.Utility.IFilter filter) { - var storeSymlinks = m_options.SymlinkPolicy == Duplicati.Library.Main.Options.SymlinkStrategy.Store; + var storeSymlinks = m_options.SymlinkPolicy == Options.SymlinkStrategy.Store; var sourcefilter = new Library.Utility.FilterExpression(sources, true); using(var snapshot = BackupHandler.GetSnapshot(sources, m_options)) { - foreach(var path in new BackupHandler.FilterHandler(snapshot, m_options.FileAttributeFilter, sourcefilter, filter, m_options.SymlinkPolicy, m_options.HardlinkPolicy).EnumerateFilesAndFolders()) + foreach(var path in new FilterHandler(snapshot, m_options.FileAttributeFilter, sourcefilter, filter, m_options.SymlinkPolicy, m_options.HardlinkPolicy).EnumerateFilesAndFolders(sources)) { var fa = FileAttributes.Normal; try { fa = snapshot.GetAttributes(path); } diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index 460eb3dc6..1e9f0eea4 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -1149,7 +1149,7 @@ namespace Duplicati.Library.Main } } /// - /// Gets the snapshot strategy to use + /// Gets the update sequence number (USN) strategy to use /// public OptimizationStrategy UsnStrategy { diff --git a/Duplicati/Library/Snapshots/Duplicati.Library.Snapshots.csproj b/Duplicati/Library/Snapshots/Duplicati.Library.Snapshots.csproj index c8aeceac0..689d3822b 100644 --- a/Duplicati/Library/Snapshots/Duplicati.Library.Snapshots.csproj +++ b/Duplicati/Library/Snapshots/Duplicati.Library.Snapshots.csproj @@ -5,7 +5,7 @@ AnyCPU {D63E53E4-A458-4C2F-914D-92F715F58ACE} Exe - False + False Properties Duplicati.Library.Snapshots Duplicati.Library.Snapshots @@ -60,15 +60,17 @@ + + - + - + diff --git a/Duplicati/Library/Snapshots/IChangeJournalService.cs b/Duplicati/Library/Snapshots/IChangeJournalService.cs new file mode 100644 index 000000000..83d90935d --- /dev/null +++ b/Duplicati/Library/Snapshots/IChangeJournalService.cs @@ -0,0 +1,62 @@ +#region Disclaimer / License + +// Copyright (C) 2015, The Duplicati Team +// http://www.duplicati.com, info@duplicati.com +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// + +#endregion + +using System.Collections.Generic; +using Duplicati.Library.Interface; + +namespace Duplicati.Library.Snapshots +{ + public interface IChangeJournalService + { + /// + /// Filters sources, returning sub-set having been modified since last + /// change, as specified by journalData. + /// + /// Sources to filter + /// A hash value representing current exclusion filter. A full scan is triggered if hash has changed. + /// The journal data of the previous backup (fileset) + /// Filtered sources + FilterData FilterSources(IEnumerable sources, string filterHash, IEnumerable journalData); + } + + /// + /// Filtered sources + /// + public struct FilterData + { + /// + /// Set of potentially modified files + /// + public HashSet Files; + + /// + /// Set of folders that are potentially modified, or whose children + /// are potentially modified + /// + public List Folders; + + /// + /// Journal data to use for next backup + /// + public List JournalData; + } +} diff --git a/Duplicati/Library/Snapshots/ISnapshotService.cs b/Duplicati/Library/Snapshots/ISnapshotService.cs index db43b7d47..42844ab5a 100644 --- a/Duplicati/Library/Snapshots/ISnapshotService.cs +++ b/Duplicati/Library/Snapshots/ISnapshotService.cs @@ -29,75 +29,111 @@ namespace Duplicati.Library.Snapshots public interface ISnapshotService : IDisposable { /// - /// Enumerates all files and folders in the snapshot + /// Enumerates all files and folders in the snapshot, restricted to sources /// + /// Sources to enumerate /// The callback to invoke with each found path /// The callback used to report errors - IEnumerable EnumerateFilesAndFolders(Duplicati.Library.Utility.Utility.EnumerationFilterDelegate callback, Duplicati.Library.Utility.Utility.ReportAccessError errorCallback); - - /// - /// Gets the last write time of a given file in UTC - /// - /// The full path to the file in non-snapshot format - /// The last write time of the file - DateTime GetLastWriteTimeUtc(string file); + IEnumerable EnumerateFilesAndFolders(IEnumerable sources, Utility.Utility.EnumerationFilterDelegate callback, Utility.Utility.ReportAccessError errorCallback); /// /// Gets the last write time of a given file in UTC /// - /// The full path to the file in non-snapshot format + /// The full path to the file in non-snapshot format /// The last write time of the file - DateTime GetCreationTimeUtc(string file); + DateTime GetLastWriteTimeUtc(string localPath); + + /// + /// Gets the last write time of a given file in UTC + /// + /// The full path to the file in non-snapshot format + /// The last write time of the file + DateTime GetCreationTimeUtc(string localPath); /// /// Opens a file for reading /// - /// The full path to the file in non-snapshot format + /// The full path to the file in non-snapshot format /// An open filestream that can be read and seeked - Stream OpenRead(string file); + Stream OpenRead(string localPath); /// /// Returns the size of a file /// - /// The full path to the file in non-snapshot format + /// The full path to the file in non-snapshot format /// The lenth of the file - long GetFileSize(string file); + long GetFileSize(string localPath); /// /// Returns the symlink target if the entry is a symlink, and null otherwise /// - /// The file or folder to examine + /// The file or folder to examine /// The symlink target - string GetSymlinkTarget(string file); + string GetSymlinkTarget(string localPath); /// /// Gets the attributes for the given file or folder /// /// The file attributes - /// The file or folder to examine - System.IO.FileAttributes GetAttributes(string file); + /// The file or folder to examine + FileAttributes GetAttributes(string localPath); /// /// Gets the metadata for the given file or folder /// /// The metadata for the given file or folder - /// The file or folder to examine + /// The file or folder to examine /// A flag indicating if the target is a symlink /// A flag indicating if a symlink should be followed - Dictionary GetMetadata(string file, bool isSymlink, bool followSymlink); + Dictionary GetMetadata(string localPath, bool isSymlink, bool followSymlink); /// /// Gets a value indicating if the path points to a block device /// /// true if this instance is a block device; otherwise, false. - /// The file or folder to examine - bool IsBlockDevice(string file); + /// The file or folder to examine + bool IsBlockDevice(string localPath); /// /// Gets a unique hardlink target ID /// /// The hardlink ID - /// The file or folder to examine - string HardlinkTargetID(string path); + /// The file or folder to examine + string HardlinkTargetID(string localPath); + + /// + /// Converts a snapshot path to a local path + /// + /// The snapshot-local path + /// The local path + string ConvertToLocalPath(string snapshotPath); + + /// + /// Converts a local path to a snapshot-local path + /// + /// The local path + /// The snapshot path + string ConvertToSnapshotPath(string localPath); + + /// + /// Tests if a file exists in the snapshot + /// + /// The local path + /// True if file exists, false otherwise + bool FileExists(string localFilePath); + + /// + /// Tests if a folder exists in the snapshot + /// + /// The local path + /// True if folder exists, false otherwise + bool FolderExists(string localFolderPath); + + /// + /// Tests if a file or folder exists in the snapshot + /// + /// The local path + /// True if file or folder exists, false otherwise + bool PathExists(string localFileOrFolderPath); } } diff --git a/Duplicati/Library/Snapshots/LinuxSnapshot.cs b/Duplicati/Library/Snapshots/LinuxSnapshot.cs index ef9632dc3..96311b1cf 100644 --- a/Duplicati/Library/Snapshots/LinuxSnapshot.cs +++ b/Duplicati/Library/Snapshots/LinuxSnapshot.cs @@ -16,12 +16,11 @@ // License along with this library; if not, write to the Free Software // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA // -using System.Linq; - - #endregion + using System; using System.Collections.Generic; +using System.Diagnostics; using System.Text; namespace Duplicati.Library.Snapshots @@ -33,35 +32,106 @@ namespace Duplicati.Library.Snapshots /// The class presents all files and folders with their regular filenames to the caller, /// and internally handles the conversion to the snapshot path. /// - public class LinuxSnapshot : ISnapshotService + public sealed class LinuxSnapshot : SnapshotBase { - protected readonly SystemIOLinux _sysIO = new SystemIOLinux(); - + private static SystemIOLinux SYS_IO = new SystemIOLinux(); + + /// + /// This is a lookup, mapping each source folder to the corresponding snapshot + /// + private readonly List> m_entries; + + /// + /// This is the list of the snapshots we have created, which must be disposed + /// + private List m_snapShots; + + /// + /// Constructs a new snapshot module using LVM + /// + /// The list of folders to create snapshots for + public LinuxSnapshot(IEnumerable sources) + { + try + { + m_entries = new List>(); + + // Make sure we do not create more snapshots than we have to + var snaps = new Dictionary(); + foreach (var path in sources) + { + var tmp = new SnapShot(path); + if (!snaps.TryGetValue(tmp.DeviceName, out var snap)) + { + snaps.Add(tmp.DeviceName, tmp); + snap = tmp; + } + + m_entries.Add(new KeyValuePair(path, snap)); + } + + m_snapShots = new List(snaps.Values); + + // We have all the snapshots that we need, lets activate them + foreach (var snap in m_snapShots) + { + snap.CreateSnapshotVolume(); + } + } + catch + { + // If something goes wrong, try to clean up + try + { + Dispose(); + } + catch + { + // ignored + } + + throw; + } + } + + #region IDisposable Members + + /// + /// Releases any held resources + /// + public override void Dispose() + { + if (m_snapShots != null) + { + Exception exs = null; + + //Attempt to clean out as many as possible + foreach(var s in m_snapShots) + { + try { s.Dispose(); } + catch (Exception ex) { exs = ex; } + } + + //Don't try this again + m_snapShots = null; + + //Report errors, if any + if (exs != null) + throw exs; + } + } + + #endregion + /// /// Internal helper class for keeping track of a single snapshot volume /// - private class SnapShot : IDisposable + private sealed class SnapShot : IDisposable { /// /// The unique id of the snapshot /// - private string m_name; - /// - /// The directory that the snapshot represents - /// - private string m_realDir; - /// - /// The temporary folder that the snapshot is mounted to - /// - private string m_tmpDir; - /// - /// The temporary folder that corresponds to the original folder - /// - private string m_mountPoint; - /// - /// The device name for the entry, used to prevent duplicate entries - /// - private string m_device; + private readonly string m_name; /// /// Constructs a new snapshot for the given folder @@ -69,55 +139,75 @@ namespace Duplicati.Library.Snapshots /// public SnapShot(string path) { - m_name = string.Format("duplicati-{0}", Guid.NewGuid().ToString()); - m_realDir = System.IO.Directory.Exists(path) ? Utility.Utility.AppendDirSeparator(path) : path; - GetVolumeName(m_realDir); + m_name = $"duplicati-{Guid.NewGuid().ToString()}"; + LocalPath = System.IO.Directory.Exists(path) ? Utility.Utility.AppendDirSeparator(path) : path; + Initialize(LocalPath); } /// /// Gets the path of the folder that this snapshot represents /// - public string LocalPath { get { return m_realDir; } } + public string LocalPath { get; } /// /// Gets a value representing the volume on which the folder resides /// - public string DeviceName { get { return m_device; } } + public string DeviceName { get; private set; } /// /// Gets the path where the snapshot is mounted /// - public string SnapshotPath { get { return m_tmpDir; } } + public string SnapshotPath { get; private set; } /// /// Gets the path the source disk is originally mounted /// - public string MountPoint { get { return m_mountPoint; } } + public string MountPoint { get; private set; } + + #region IDisposable Members /// - /// Converts a snapshot path to a local path + /// Cleanup any used resources /// - /// The snapshot path - /// The local path - public string ConvertToLocalPath(string path) + public void Dispose() { - if (!path.StartsWith(m_mountPoint, StringComparison.Ordinal)) - throw new InvalidOperationException(); + if (SnapshotPath != null && System.IO.Directory.Exists(SnapshotPath)) + { + var output = ExecuteCommand("remove-lvm-snapshot.sh", $"\"{m_name}\" \"{DeviceName}\" \"{SnapshotPath}\"", 0); + if (System.IO.Directory.Exists(SnapshotPath)) + throw new Exception(Strings.LinuxSnapshot.MountFolderNotRemovedError(SnapshotPath, output)); - return m_tmpDir + path.Substring(m_mountPoint.Length); + SnapshotPath = null; + DeviceName = null; + } } + #endregion + /// /// Converts a local path to a snapshot path /// - /// The local path + /// The local path /// The snapshot path - public string ConvertToSnapshotPath(string path) + public string ConvertToSnapshotPath(string localPath) { - if (!path.StartsWith(m_tmpDir, StringComparison.Ordinal)) + if (!localPath.StartsWith(MountPoint, StringComparison.Ordinal)) throw new InvalidOperationException(); - return m_mountPoint + path.Substring(m_tmpDir.Length); + return SnapshotPath + localPath.Substring(MountPoint.Length); + } + + /// + /// Converts a snapshot path to a local path + /// + /// The snapshot path + /// The local path + public string ConvertToLocalPath(string snapshotPath) + { + if (!snapshotPath.StartsWith(SnapshotPath, StringComparison.Ordinal)) + throw new InvalidOperationException(); + + return MountPoint + snapshotPath.Substring(SnapshotPath.Length); } /// @@ -129,34 +219,37 @@ namespace Duplicati.Library.Snapshots /// A string with the combined output of the stdout and stderr private static string ExecuteCommand(string program, string commandline, int expectedExitCode) { - program = System.IO.Path.Combine(System.IO.Path.Combine(Duplicati.Library.AutoUpdater.UpdaterManager.InstalledBaseDir, "lvm-scripts"), program); - System.Diagnostics.ProcessStartInfo inf = new System.Diagnostics.ProcessStartInfo(program, commandline); - inf.CreateNoWindow = true; - inf.RedirectStandardError = true; - inf.RedirectStandardOutput = true; - inf.WindowStyle = System.Diagnostics.ProcessWindowStyle.Hidden; - inf.UseShellExecute = false; + program = System.IO.Path.Combine(System.IO.Path.Combine(AutoUpdater.UpdaterManager.InstalledBaseDir, "lvm-scripts"), program); + var inf = new ProcessStartInfo(program, commandline) + { + CreateNoWindow = true, + RedirectStandardError = true, + RedirectStandardOutput = true, + WindowStyle = ProcessWindowStyle.Hidden, + UseShellExecute = false + }; try { - System.Diagnostics.Process p = System.Diagnostics.Process.Start(inf); + var p = Process.Start(inf); //Allow up 20 seconds for the execution + Debug.Assert(p != null, nameof(p) + " != null"); if (!p.WaitForExit(30 * 1000)) { //Attempt to close down semi-nicely p.Kill(); p.WaitForExit(5 * 1000); //This should work, and if it does, prevents a race with any cleanup invocations - throw new Duplicati.Library.Interface.UserInformationException(Strings.LinuxSnapshot.ExternalProgramTimeoutError(program, commandline), "LvmScriptTimeout"); + throw new Interface.UserInformationException(Strings.LinuxSnapshot.ExternalProgramTimeoutError(program, commandline), "LvmScriptTimeout"); } //Build the output string. Since the process has exited, these cannot block - string output = string.Format("Exit code: {1}{0}{2}{0}{3}", Environment.NewLine, p.ExitCode, p.StandardOutput.ReadToEnd(), p.StandardError.ReadToEnd()); + var output = string.Format("Exit code: {1}{0}{2}{0}{3}", Environment.NewLine, p.ExitCode, p.StandardOutput.ReadToEnd(), p.StandardError.ReadToEnd()); //Throw an exception if something went wrong if (p.ExitCode != expectedExitCode) - throw new Duplicati.Library.Interface.UserInformationException(Strings.LinuxSnapshot.ScriptExitCodeError(p.ExitCode, expectedExitCode, output), "LvmScriptWrongExitCode"); + throw new Interface.UserInformationException(Strings.LinuxSnapshot.ScriptExitCodeError(p.ExitCode, expectedExitCode, output), "LvmScriptWrongExitCode"); return output; } @@ -169,20 +262,20 @@ namespace Duplicati.Library.Snapshots /// /// Finds the LVM id of the volume id where the folder is placed /// - private void GetVolumeName(string folder) + private void Initialize(string folder) { //Figure out what logical volume the path is located on - string output = ExecuteCommand("find-volume.sh", string.Format("\"{0}\"", folder), 0); + var output = ExecuteCommand("find-volume.sh", $"\"{folder}\"", 0); - System.Text.RegularExpressions.Regex rex = new System.Text.RegularExpressions.Regex("device=\"(?[^\"]+)\""); - System.Text.RegularExpressions.Match m = rex.Match(output); + var rex = new System.Text.RegularExpressions.Regex("device=\"(?[^\"]+)\""); + var m = rex.Match(output); if (!m.Success) throw new Exception(Strings.LinuxSnapshot.ScriptOutputError("device", output)); - m_device = rex.Match(output).Groups["device"].Value; + DeviceName = rex.Match(output).Groups["device"].Value; - if (string.IsNullOrEmpty(m_device) || m_device.Trim().Length == 0) + if (string.IsNullOrEmpty(DeviceName) || DeviceName.Trim().Length == 0) throw new Exception(Strings.LinuxSnapshot.ScriptOutputError("device", output)); rex = new System.Text.RegularExpressions.Regex("mountpoint=\"(?[^\"]+)\""); @@ -191,12 +284,12 @@ namespace Duplicati.Library.Snapshots if (!m.Success) throw new Exception(Strings.LinuxSnapshot.ScriptOutputError("mountpoint", output)); - m_mountPoint = rex.Match(output).Groups["mountpoint"].Value; + MountPoint = rex.Match(output).Groups["mountpoint"].Value; - if (string.IsNullOrEmpty(m_mountPoint) || m_mountPoint.Trim().Length == 0) + if (string.IsNullOrEmpty(MountPoint) || MountPoint.Trim().Length == 0) throw new Exception(Strings.LinuxSnapshot.ScriptOutputError("mountpoint", output)); - m_mountPoint = Utility.Utility.AppendDirSeparator(m_mountPoint); + MountPoint = Utility.Utility.AppendDirSeparator(MountPoint); } /// @@ -205,94 +298,26 @@ namespace Duplicati.Library.Snapshots /// public void CreateSnapshotVolume() { - if (m_device == null) + if (DeviceName == null) throw new InvalidOperationException(); - if (m_tmpDir != null) + if (SnapshotPath != null) throw new InvalidOperationException(); //Create the snapshot volume - string output = ExecuteCommand("create-lvm-snapshot.sh", string.Format("\"{0}\" \"{1}\" \"{2}\"", m_name, m_device, Utility.Utility.AppendDirSeparator(Utility.TempFolder.SystemTempPath)), 0); + var output = ExecuteCommand("create-lvm-snapshot.sh", $"\"{m_name}\" \"{DeviceName}\" \"{Utility.Utility.AppendDirSeparator(Utility.TempFolder.SystemTempPath)}\"", 0); - System.Text.RegularExpressions.Regex rex = new System.Text.RegularExpressions.Regex("tmpdir=\"(?[^\"]+)\""); - System.Text.RegularExpressions.Match m = rex.Match(output); + var rex = new System.Text.RegularExpressions.Regex("tmpdir=\"(?[^\"]+)\""); + var m = rex.Match(output); if (!m.Success) throw new Exception(Strings.LinuxSnapshot.ScriptOutputError("tmpdir", output)); - m_tmpDir = rex.Match(output).Groups["tmpdir"].Value; + SnapshotPath = rex.Match(output).Groups["tmpdir"].Value; - if (!System.IO.Directory.Exists(m_tmpDir)) - throw new Exception(Strings.LinuxSnapshot.MountFolderMissingError(m_tmpDir, output)); + if (!System.IO.Directory.Exists(SnapshotPath)) + throw new Exception(Strings.LinuxSnapshot.MountFolderMissingError(SnapshotPath, output)); - m_tmpDir = Utility.Utility.AppendDirSeparator(m_tmpDir); - } - - #region IDisposable Members - - /// - /// Cleanup any used resources - /// - public void Dispose() - { - if (m_tmpDir != null && System.IO.Directory.Exists(m_tmpDir)) - { - string output = ExecuteCommand("remove-lvm-snapshot.sh", string.Format("\"{0}\" \"{1}\" \"{2}\"", m_name, m_device, m_tmpDir), 0); - if (System.IO.Directory.Exists(m_tmpDir)) - throw new Exception(Strings.LinuxSnapshot.MountFolderNotRemovedError(m_tmpDir, output)); - - m_tmpDir = null; - m_device = null; - } - } - - #endregion - } - - /// - /// This is the list of the snapshots we have created, which must be disposed - /// - private List m_activeSnapShots; - - /// - /// This is a looup, mapping each source folder to the corresponding snapshot - /// - private List> m_entries; - - /// - /// Constructs a new snapshot module using LVM - /// - /// The list of folders to create snapshots for - /// A set of commandline options - public LinuxSnapshot(string[] sources, Dictionary options) - { - try - { - m_entries = new List>(); - - //Make sure we do not create more snapshots than we have to - Dictionary snaps = new Dictionary(); - foreach (string s in sources) - { - SnapShot sn = new SnapShot(s); - if (!snaps.ContainsKey(sn.DeviceName)) - snaps.Add(sn.DeviceName, sn); - - m_entries.Add(new KeyValuePair(s, snaps[sn.DeviceName])); - } - - m_activeSnapShots = new List(snaps.Values); - - //We have all the snapshots that we need, lets activate them - foreach (SnapShot s in m_activeSnapShots) - s.CreateSnapshotVolume(); - } - catch - { - //If something goes wrong, try to clean up - try { Dispose(); } - catch { } - - throw; + SnapshotPath = Utility.Utility.AppendDirSeparator(SnapshotPath); } } @@ -302,14 +327,14 @@ namespace Duplicati.Library.Snapshots /// A callback function that takes a non-snapshot path to a folder, /// and returns all folders found in a non-snapshot path format. /// - /// The non-snapshot path of the folder to list + /// The non-snapshot path of the folder to list /// A list of non-snapshot paths - private string[] ListFolders(string folder) + protected override string[] ListFolders(string localFolderPath) { - KeyValuePair snap = FindSnapShotByLocalPath(folder); + var snap = FindSnapshotByLocalPath(localFolderPath); - string[] tmp = System.IO.Directory.GetDirectories(ConvertToSnapshotPath(snap, folder)); - for (int i = 0; i < tmp.Length; i++) + var tmp = System.IO.Directory.GetDirectories(ConvertToSnapshotPath(snap, localFolderPath)); + for (var i = 0; i < tmp.Length; i++) tmp[i] = ConvertToLocalPath(snap, tmp[i]); return tmp; @@ -320,14 +345,14 @@ namespace Duplicati.Library.Snapshots /// A callback function that takes a non-snapshot path to a folder, /// and returns all files found in a non-snapshot path format. /// - /// The non-snapshot path of the folder to list + /// The non-snapshot path of the folder to list /// A list of non-snapshot paths - private string[] ListFiles(string folder) + protected override string[] ListFiles(string localFolderPath) { - KeyValuePair snap = FindSnapShotByLocalPath(folder); + var snap = FindSnapshotByLocalPath(localFolderPath); - string[] tmp = System.IO.Directory.GetFiles(ConvertToSnapshotPath(snap, folder)); - for (int i = 0; i < tmp.Length; i++) + var tmp = System.IO.Directory.GetFiles(ConvertToSnapshotPath(snap, localFolderPath)); + for (var i = 0; i < tmp.Length; i++) tmp[i] = ConvertToLocalPath(snap, tmp[i]); return tmp; } @@ -335,150 +360,155 @@ namespace Duplicati.Library.Snapshots /// /// Locates the snapshot instance that maps the path /// - /// The file or folder name to match + /// The file or folder name to match /// The matching snapshot - private KeyValuePair FindSnapShotByLocalPath(string name) + private SnapShot FindSnapshotByLocalPath(string localPath) { KeyValuePair? best = null; - - foreach (KeyValuePair s in m_entries) - if (name.StartsWith(s.Key, StringComparison.Ordinal) && (best == null || s.Key.Length > best.Value.Key.Length)) - best = s; - - if (best == null) + foreach (var s in m_entries) { - StringBuilder sb = new StringBuilder(); - sb.Append(Environment.NewLine); - - foreach (KeyValuePair s in m_entries) - sb.AppendFormat("{0} ({1} -> {2}){3}", s.Key, s.Value.MountPoint, s.Value.SnapshotPath, Environment.NewLine); - - throw new InvalidOperationException(Strings.LinuxSnapshot.InvalidFilePathError(name, sb.ToString())); + if (localPath.StartsWith(s.Key, StringComparison.Ordinal) && (best == null || s.Key.Length > best.Value.Key.Length)) + { + best = s; + } } - return best.Value; + if (best != null) + return best.Value.Value; + + var sb = new StringBuilder(); + sb.Append(Environment.NewLine); + + foreach (var s in m_entries) + { + sb.Append($"{s.Key} ({s.Value.MountPoint} -> {s.Value.SnapshotPath}){Environment.NewLine}"); + } + + throw new InvalidOperationException(Strings.LinuxSnapshot.InvalidFilePathError(localPath, sb.ToString())); + } + + /// + /// Locates the snapshot containing the snapshot path + /// + /// + /// Snapshot containing snapshotPath + private SnapShot FindSnapshotBySnapshotPath(string snapshotPath) + { + foreach (var snap in m_snapShots) + { + if (snapshotPath.StartsWith(snap.SnapshotPath, StringComparison.Ordinal)) + return snap; + } + + throw new InvalidOperationException(); } /// /// Converts a local path to a snapshot path /// /// The snapshot that represents the mapping - /// The filename to convert + /// The filename to convert /// The converted path - private string ConvertToSnapshotPath(KeyValuePair snap, string file) + private string ConvertToSnapshotPath(SnapShot snap, string localPath) { - return snap.Value.SnapshotPath + file.Substring(snap.Value.MountPoint.Length); + return snap.ConvertToSnapshotPath(localPath); } /// /// Converts a snapshot path to a local path /// /// The snapshot that represents the mapping - /// The filename to convert + /// The filename to convert /// The converted path - private string ConvertToLocalPath(KeyValuePair snap, string file) + private string ConvertToLocalPath(SnapShot snap, string snapshotPath) { - return snap.Value.MountPoint + file.Substring(snap.Value.SnapshotPath.Length); + return snap.ConvertToLocalPath(snapshotPath); } #endregion #region ISnapshotService Members - /// - /// Enumerates all files and folders in the snapshot - /// - /// The callback to invoke with each found path - /// The callback used to report errors - public IEnumerable EnumerateFilesAndFolders(Duplicati.Library.Utility.Utility.EnumerationFilterDelegate callback, Duplicati.Library.Utility.Utility.ReportAccessError errorCallback) - { - return m_entries.SelectMany( - s => Utility.Utility.EnumerateFileSystemEntries(s.Key, callback, this.ListFolders, this.ListFiles, this.GetAttributes, errorCallback) - ); - } - /// /// Gets the last write time of a given file in UTC /// - /// The full path to the file in non-snapshot format + /// The full path to the file in non-snapshot format /// The last write time of the file - public DateTime GetLastWriteTimeUtc(string file) + public override DateTime GetLastWriteTimeUtc(string localPath) { - return System.IO.File.GetLastWriteTimeUtc(ConvertToSnapshotPath(FindSnapShotByLocalPath(file), file)); + return System.IO.File.GetLastWriteTimeUtc(ConvertToSnapshotPath(localPath)); } /// /// Gets the creation time of a given file in UTC /// - /// The full path to the file in non-snapshot format + /// The full path to the file in non-snapshot format /// The last write time of the file - public DateTime GetCreationTimeUtc(string file) + public override DateTime GetCreationTimeUtc(string localPath) { - return System.IO.File.GetLastWriteTimeUtc(ConvertToSnapshotPath(FindSnapShotByLocalPath(file), file)); + return System.IO.File.GetLastWriteTimeUtc(ConvertToSnapshotPath(localPath)); } /// /// Opens a file for reading /// - /// The full path to the file in non-snapshot format + /// The full path to the file in non-snapshot format /// An open filestream that can be read - public System.IO.Stream OpenRead(string file) + public override System.IO.Stream OpenRead(string localPath) { - return System.IO.File.OpenRead(ConvertToSnapshotPath(FindSnapShotByLocalPath(file), file)); + return System.IO.File.OpenRead(ConvertToSnapshotPath(localPath)); } /// /// Returns the size of a file /// - /// The full path to the file in non-snapshot format + /// The full path to the file in non-snapshot format /// The lenth of the file - public long GetFileSize(string file) + public override long GetFileSize(string localPath) { - return new System.IO.FileInfo(ConvertToSnapshotPath(FindSnapShotByLocalPath(file), file)).Length; + return new System.IO.FileInfo(ConvertToSnapshotPath(localPath)).Length; } /// /// Gets the attributes for the given file or folder /// /// The file attributes - /// The file or folder to examine - public System.IO.FileAttributes GetAttributes(string file) + /// The file or folder to examine + public override System.IO.FileAttributes GetAttributes(string localPath) { - return System.IO.File.GetAttributes(ConvertToSnapshotPath(FindSnapShotByLocalPath(file), file)); + return System.IO.File.GetAttributes(ConvertToSnapshotPath(localPath)); } /// /// Returns the symlink target if the entry is a symlink, and null otherwise /// - /// The file or folder to examine + /// The file or folder to examine /// The symlink target - public string GetSymlinkTarget(string file) + public override string GetSymlinkTarget(string localPath) { - var local = ConvertToSnapshotPath(FindSnapShotByLocalPath(file), file); - return _sysIO.GetSymlinkTarget(local); + return SYS_IO.GetSymlinkTarget(ConvertToSnapshotPath(localPath)); } /// /// Gets the metadata for the given file or folder /// /// The metadata for the given file or folder - /// The file or folder to examine + /// The file or folder to examine /// A flag indicating if the target is a symlink /// A flag indicating if a symlink should be followed - public Dictionary GetMetadata(string file, bool isSymlink, bool followSymlink) + public override Dictionary GetMetadata(string localPath, bool isSymlink, bool followSymlink) { - var local = ConvertToSnapshotPath(FindSnapShotByLocalPath(file), file); - return _sysIO.GetMetadata(local, isSymlink, followSymlink); + return SYS_IO.GetMetadata(ConvertToSnapshotPath(localPath), isSymlink, followSymlink); } - + /// /// Gets a value indicating if the path points to a block device /// /// true if this instance is a block device; otherwise, false. - /// The file or folder to examine - public bool IsBlockDevice(string file) + /// The file or folder to examine + public override bool IsBlockDevice(string localPath) { - var n = UnixSupport.File.GetFileType(NoSnapshot.NormalizePath(file)); + var n = UnixSupport.File.GetFileType(NormalizePath(localPath)); switch (n) { case UnixSupport.File.FileType.Directory: @@ -488,48 +518,33 @@ namespace Duplicati.Library.Snapshots default: return true; } - } + } /// /// Gets a unique hardlink target ID /// /// The hardlink ID - /// The file or folder to examine - public string HardlinkTargetID(string path) + /// The file or folder to examine + public override string HardlinkTargetID(string localPath) { - var local = ConvertToSnapshotPath(FindSnapShotByLocalPath(path), path); - local = NoSnapshot.NormalizePath(local); + var snapshotPath = NormalizePath(ConvertToSnapshotPath(localPath)); - if (UnixSupport.File.GetHardlinkCount(local) <= 1) + if (UnixSupport.File.GetHardlinkCount(snapshotPath) <= 1) return null; - return UnixSupport.File.GetInodeTargetID(local); + return UnixSupport.File.GetInodeTargetID(snapshotPath); } - #endregion - #region IDisposable Members - - /// - /// Releases any held resources - /// - public void Dispose() + /// + public override string ConvertToLocalPath(string snapshotPath) { - if (m_activeSnapShots != null) - { - Exception exs = null; + return ConvertToLocalPath(FindSnapshotBySnapshotPath(snapshotPath), snapshotPath); + } - //Attempt to clean out as many as possible - foreach(SnapShot s in m_activeSnapShots) - try { s.Dispose(); } - catch (Exception ex) { exs = ex; } - - //Don't try this again - m_activeSnapShots = null; - - //Report errors, if any - if (exs != null) - throw exs; - } + /// + public override string ConvertToSnapshotPath(string localPath) + { + return ConvertToSnapshotPath(FindSnapshotByLocalPath(localPath), localPath); } #endregion diff --git a/Duplicati/Library/Snapshots/NoSnapshot.cs b/Duplicati/Library/Snapshots/NoSnapshot.cs deleted file mode 100644 index dd32d8da8..000000000 --- a/Duplicati/Library/Snapshots/NoSnapshot.cs +++ /dev/null @@ -1,199 +0,0 @@ -#region Disclaimer / License -// Copyright (C) 2015, The Duplicati Team -// http://www.duplicati.com, info@duplicati.com -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public -// License along with this library; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -// -using System.Linq; - - -#endregion -using System; -using System.Collections.Generic; -using System.Text; - -namespace Duplicati.Library.Snapshots -{ - /// - /// A simple implementation of snapshots that only uses the regular System.IO operations, - /// used to simplify code that uses snapshots, but also handles a non-snapshot enabled systems - /// - public abstract class NoSnapshot : ISnapshotService - { - /// - /// The list of all folders in the snapshot - /// - protected string[] m_sources; - - /// - /// Constructs a new backup snapshot, using all the required disks - /// - /// The folders that are about to be backed up - protected NoSnapshot(string[] sources) - : this(sources, new Dictionary()) - { - } - - /// - /// Constructs a new backup snapshot, using all the required disks - /// - /// The folders that are about to be backed up - /// A set of system options - protected NoSnapshot(string[] sources, Dictionary options) - { - m_sources = new string[sources.Length]; - for (int i = 0; i < m_sources.Length; i++) - m_sources[i] = System.IO.Directory.Exists(sources[i]) ? Utility.Utility.AppendDirSeparator(sources[i]) : sources[i]; - } - - #region Private Methods - /// - /// Normalizes a path before calling system methods - /// - /// The path to normalize. - /// The normalized path. - public static string NormalizePath(string path) - { - var p = System.IO.Path.GetFullPath(path); - - // This should not be required, but some versions of Mono apperently do not strip the trailing slash - return (p.Length > 1 && p[p.Length - 1] == System.IO.Path.DirectorySeparatorChar) ? p.Substring(0, p.Length - 1) : p; - } - - /// - /// Lists all folders in the given folder - /// - /// All folders found in the folder - /// The folder to examinate - protected virtual string[] ListFolders(string folder) - { - return System.IO.Directory.GetDirectories(folder); - } - - - /// - /// Lists all files in the given folder - /// - /// All folders found in the folder - /// The folder to examinate - protected virtual string[] ListFiles(string folder) - { - return System.IO.Directory.GetFiles(folder); - } - #endregion - - #region ISnapshotService Members - - /// - /// Cleans up any resources and closes the backup set - /// - public void Dispose() - { - } - - /// - /// Enumerates all files and folders in the snapshot - /// - /// The callback to invoke with each found path - /// The callback used to report errors - public IEnumerable EnumerateFilesAndFolders(Duplicati.Library.Utility.Utility.EnumerationFilterDelegate callback, Duplicati.Library.Utility.Utility.ReportAccessError errorCallback) - { - return m_sources.SelectMany( - s => Utility.Utility.EnumerateFileSystemEntries(s, callback, this.ListFolders, this.ListFiles, this.GetAttributes, errorCallback) - ); - } - - /// - /// Gets the last write time of a given file in UTC - /// - /// The full path to the file in non-snapshot format - /// The last write time of the file - public virtual DateTime GetLastWriteTimeUtc(string file) - { - return System.IO.File.GetLastWriteTimeUtc(file); - } - - /// - /// Gets the last write time of a given file in UTC - /// - /// The full path to the file in non-snapshot format - /// The last write time of the file - public virtual DateTime GetCreationTimeUtc(string file) - { - return System.IO.File.GetCreationTimeUtc(file); - } - - /// - /// Opens a file for reading - /// - /// The full path to the file in non-snapshot format - /// An open filestream that can be read - public virtual System.IO.Stream OpenRead(string file) - { - return System.IO.File.Open(file, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.ReadWrite); - } - - /// - /// Returns the size of a file - /// - /// The full path to the file in non-snapshot format - /// The lenth of the file - public virtual long GetFileSize(string file) - { - return new System.IO.FileInfo(file).Length; - } - - /// - /// Returns the symlink target if the entry is a symlink, and null otherwise - /// - /// The file or folder to examine - /// The symlink target - public abstract string GetSymlinkTarget(string file); - - /// - /// Gets the attributes for the given file or folder - /// - /// The file attributes - /// The file or folder to examine - public virtual System.IO.FileAttributes GetAttributes(string file) - { - return System.IO.File.GetAttributes(NormalizePath(file)); - } - - /// - /// Gets the metadata for the given file or folder - /// - /// The metadata for the given file or folder - /// The file or folder to examine - /// A flag indicating if the target is a symlink - /// A flag indicating if a symlink should be followed - public abstract Dictionary GetMetadata(string file, bool isSymlink, bool followSymlink); - - /// - /// Gets a value indicating if the path points to a block device - /// - /// true if this instance is a block device; otherwise, false. - /// The file or folder to examine - public abstract bool IsBlockDevice(string file); - - /// - /// Gets a unique hardlink target ID - /// - /// The hardlink ID - /// The file or folder to examine - public abstract string HardlinkTargetID(string path); - #endregion - } -} diff --git a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs index 3aa6f8343..db9c404cf 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs @@ -15,8 +15,7 @@ // You should have received a copy of the GNU Lesser General Public // License along with this library; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -using System; -using System.Text; + using System.Collections.Generic; namespace Duplicati.Library.Snapshots @@ -24,50 +23,40 @@ namespace Duplicati.Library.Snapshots /// /// Handler for providing a snapshot like access to files and folders /// - public class NoSnapshotLinux : NoSnapshot + public sealed class NoSnapshotLinux : SnapshotBase { - private readonly SystemIOLinux _sysIO = new SystemIOLinux(); - - public NoSnapshotLinux(string[] sourcefolders) - : base(sourcefolders) - { - } - - public NoSnapshotLinux(string[] sourcefolders, Dictionary options) - : base(sourcefolders, options) - { - } + private static SystemIOLinux SYS_IO = new SystemIOLinux(); /// /// Returns the symlink target if the entry is a symlink, and null otherwise /// - /// The file or folder to examine + /// The file or folder to examine /// The symlink target - public override string GetSymlinkTarget(string file) + public override string GetSymlinkTarget(string localPath) { - return _sysIO.GetSymlinkTarget(file); + return SYS_IO.GetSymlinkTarget(localPath); } /// /// Gets the metadata for the given file or folder /// /// The metadata for the given file or folder - /// The file or folder to examine + /// The file or folder to examine /// A flag indicating if the target is a symlink /// A flag indicating if a symlink should be followed - public override Dictionary GetMetadata(string file, bool isSymlink, bool followSymlink) + public override Dictionary GetMetadata(string localPath, bool isSymlink, bool followSymlink) { - return _sysIO.GetMetadata(file, isSymlink, followSymlink); + return SYS_IO.GetMetadata(localPath, isSymlink, followSymlink); } /// /// Gets a value indicating if the path points to a block device /// /// true if this instance is a block device; otherwise, false. - /// The file or folder to examine - public override bool IsBlockDevice(string file) + /// The file or folder to examine + public override bool IsBlockDevice(string localPath) { - var n = UnixSupport.File.GetFileType(NormalizePath(file)); + var n = UnixSupport.File.GetFileType(NormalizePath(localPath)); switch (n) { case UnixSupport.File.FileType.Directory: @@ -83,14 +72,26 @@ namespace Duplicati.Library.Snapshots /// Gets a unique hardlink target ID /// /// The hardlink ID - /// The file or folder to examine - public override string HardlinkTargetID(string path) + /// The file or folder to examine + public override string HardlinkTargetID(string localPath) { - path = NormalizePath(path); - if (UnixSupport.File.GetHardlinkCount(path) <= 1) + localPath = NormalizePath(localPath); + if (UnixSupport.File.GetHardlinkCount(localPath) <= 1) return null; - return UnixSupport.File.GetInodeTargetID(path); + return UnixSupport.File.GetInodeTargetID(localPath); + } + + /// + public override string ConvertToLocalPath(string snapshotPath) + { + return snapshotPath; + } + + /// + public override string ConvertToSnapshotPath(string localPath) + { + return localPath; } } } diff --git a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs index d4a692ee8..740039ad3 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs @@ -16,7 +16,6 @@ // License along with this library; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA using System; -using System.Text; using System.Collections.Generic; using Alphaleonis.Win32.Filesystem; @@ -25,92 +24,82 @@ namespace Duplicati.Library.Snapshots /// /// Handler for providing a snapshot like access to files and folders /// - public class NoSnapshotWindows : NoSnapshot + public sealed class NoSnapshotWindows : SnapshotBase { - private SystemIOWindows m_sysIO = new SystemIOWindows(); - - public NoSnapshotWindows(string[] sources) - : base(sources) - { - } - - public NoSnapshotWindows(string[] sources, Dictionary options) - : base(sources, options) - { - } + private static SystemIOWindows IO_WIN = new SystemIOWindows(); /// /// Returns the symlink target if the entry is a symlink, and null otherwise /// - /// The file or folder to examine + /// The file or folder to examine /// The symlink target - public override string GetSymlinkTarget(string file) + public override string GetSymlinkTarget(string localPath) { - return m_sysIO.GetSymlinkTarget(file); + return IO_WIN.GetSymlinkTarget(localPath); } /// /// Gets the attributes for the given file or folder /// /// The file attributes - /// The file or folder to examine - public override System.IO.FileAttributes GetAttributes (string file) + /// The file or folder to examine + public override System.IO.FileAttributes GetAttributes (string localPath) { - return m_sysIO.GetFileAttributes(file); + return IO_WIN.GetFileAttributes(localPath); } /// /// Returns the size of a file /// - /// The full path to the file in non-snapshot format + /// The full path to the file in non-snapshot format /// The lenth of the file - public override long GetFileSize (string file) + public override long GetFileSize (string localPath) { - return m_sysIO.FileLength(file); + return IO_WIN.FileLength(localPath); } /// /// Gets the last write time of a given file in UTC /// - /// The full path to the file in non-snapshot format + /// The full path to the file in non-snapshot format /// The last write time of the file - public override DateTime GetLastWriteTimeUtc (string file) + public override DateTime GetLastWriteTimeUtc (string localPath) { - return m_sysIO.FileGetLastWriteTimeUtc(file); + return IO_WIN.FileGetLastWriteTimeUtc(localPath); } /// /// Gets the last write time of a given file in UTC /// - /// The full path to the file in non-snapshot format + /// The full path to the file in non-snapshot format /// The last write time of the file - public override DateTime GetCreationTimeUtc (string file) + public override DateTime GetCreationTimeUtc (string localPath) { - return m_sysIO.FileGetCreationTimeUtc(file); + return IO_WIN.FileGetCreationTimeUtc(localPath); } /// /// Opens a file for reading /// - /// The full path to the file in non-snapshot format + /// The full path to the file in non-snapshot format /// An open filestream that can be read - public override System.IO.Stream OpenRead (string file) + public override System.IO.Stream OpenRead (string localPath) { - return m_sysIO.FileOpenRead(file); + return IO_WIN.FileOpenRead(localPath); } /// /// Lists all files in the given folder /// /// All folders found in the folder - /// The folder to examinate - protected override string[] ListFiles (string folder) + /// The folder to examinate + protected override string[] ListFiles (string localFolderPath) { - if (!SystemIOWindows.IsPathTooLong(folder)) - try { return base.ListFiles(folder); } + if (!SystemIOWindows.IsPathTooLong(localFolderPath)) + try { return base.ListFiles(localFolderPath); } catch (System.IO.PathTooLongException) { } - string[] tmp = Directory.GetFiles(SystemIOWindows.PrefixWithUNC(folder)); + string[] tmp = Directory.GetFiles(SystemIOWindows.PrefixWithUNC(localFolderPath)); string[] res = new string[tmp.Length]; for(int i = 0; i < tmp.Length; i++) res[i] = SystemIOWindows.StripUNCPrefix(tmp[i]); @@ -118,18 +107,30 @@ namespace Duplicati.Library.Snapshots return res; } + /// + public override string ConvertToLocalPath(string snapshotPath) + { + return snapshotPath; + } + + /// + public override string ConvertToSnapshotPath(string localPath) + { + return localPath; + } + /// /// Lists all folders in the given folder /// /// All folders found in the folder - /// The folder to examinate - protected override string[] ListFolders (string folder) + /// The folder to examinate + protected override string[] ListFolders (string localFolderPath) { - if (!SystemIOWindows.IsPathTooLong(folder)) - try { return base.ListFolders(folder); } + if (!SystemIOWindows.IsPathTooLong(localFolderPath)) + try { return base.ListFolders(localFolderPath); } catch (System.IO.PathTooLongException) { } - string[] tmp = Directory.GetDirectories(SystemIOWindows.PrefixWithUNC(folder)); + string[] tmp = Directory.GetDirectories(SystemIOWindows.PrefixWithUNC(localFolderPath)); string[] res = new string[tmp.Length]; for (int i = 0; i < tmp.Length; i++) res[i] = SystemIOWindows.StripUNCPrefix(tmp[i]); @@ -141,32 +142,12 @@ namespace Duplicati.Library.Snapshots /// Gets the metadata for the given file or folder /// /// The metadata for the given file or folder - /// The file or folder to examine + /// The file or folder to examine /// A flag indicating if the target is a symlink /// A flag indicating if a symlink should be followed - public override Dictionary GetMetadata(string file, bool isSymlink, bool followSymlink) + public override Dictionary GetMetadata(string localPath, bool isSymlink, bool followSymlink) { - return m_sysIO.GetMetadata(file, isSymlink, followSymlink); - } - - /// - /// Gets a value indicating if the path points to a block device - /// - /// true if this instance is a block device; otherwise, false. - /// The file or folder to examine - public override bool IsBlockDevice(string file) - { - return false; - } - - /// - /// Gets a unique hardlink target ID - /// - /// The hardlink ID - /// The file or folder to examine - public override string HardlinkTargetID(string path) - { - return null; + return IO_WIN.GetMetadata(localPath, isSymlink, followSymlink); } } } diff --git a/Duplicati/Library/Snapshots/Program.cs b/Duplicati/Library/Snapshots/Program.cs index 1919dbd20..e139d2ff8 100644 --- a/Duplicati/Library/Snapshots/Program.cs +++ b/Duplicati/Library/Snapshots/Program.cs @@ -78,88 +78,53 @@ Where is the folder where files will be locked/created etc"); return; } - if (Library.Utility.Utility.ParseBoolOption(options, "usn-test")) + if (!System.IO.Directory.Exists(args[0])) + System.IO.Directory.CreateDirectory(args[0]); + + string filename = System.IO.Path.Combine(args[0], "testfile.bin"); + + Console.WriteLine(string.Format("Creating file {0}", filename)); + + using (System.IO.FileStream fs = new System.IO.FileStream(filename, System.IO.FileMode.Create, System.IO.FileAccess.ReadWrite, System.IO.FileShare.None)) { - using (USNHelper usn = new USNHelper(args[0])) + Console.WriteLine(string.Format("Attempting to read locked file {0}", filename)); + + try { - Console.WriteLine("Current USN JournalID: " + usn.JournalID); - Console.WriteLine("Current USN no: " + usn.USN); + using (System.IO.FileStream fs2 = new System.IO.FileStream(filename, System.IO.FileMode.Create, System.IO.FileAccess.ReadWrite, System.IO.FileShare.None)) + { } - if (options.ContainsKey("last-usn")) - { - Console.WriteLine("Changed entries ..."); - foreach (string s in usn.GetChangedFileSystemEntries(args[0], long.Parse(options["last-usn"]))) - Console.WriteLine(s + ", " + usn.GetChangeFlags(s)); - - Console.WriteLine(); - Console.WriteLine("Renamed entries ..."); - foreach (string s in usn.GetRenamedFileSystemEntries(args[0], long.Parse(options["last-usn"]))) - Console.WriteLine(s); - } - else - { - Console.WriteLine("Files ..."); - foreach (string s in usn.Files) - Console.WriteLine(s); - - Console.WriteLine(); - Console.WriteLine("Folders ..."); - - foreach (string s in usn.Folders) - Console.WriteLine(s); - } + Console.WriteLine(string.Format("Could open locked file {0}, cannot test", filename)); + Console.WriteLine("* Test failed"); + return; } - } - else - { - if (!System.IO.Directory.Exists(args[0])) - System.IO.Directory.CreateDirectory(args[0]); - - string filename = System.IO.Path.Combine(args[0], "testfile.bin"); - - Console.WriteLine(string.Format("Creating file {0}", filename)); - - using (System.IO.FileStream fs = new System.IO.FileStream(filename, System.IO.FileMode.Create, System.IO.FileAccess.ReadWrite, System.IO.FileShare.None)) + catch (Exception ex) { - Console.WriteLine(string.Format("Attempting to read locked file {0}", filename)); + Console.WriteLine(string.Format("The file {0} was correctly locked, message: {1}", filename, ex.Message)); + } + Console.WriteLine("Creating snapshot for folder: {0}", args[0]); + Console.WriteLine("If this fails, try to run as " + (Utility.Utility.IsClientLinux ? "root" : "Administrator")); + using (ISnapshotService snapshot = SnapshotUtility.CreateSnapshot(new[] { args[0] }, options)) + { + Console.WriteLine("Attempting to read locked file via snapshot"); try { - using (System.IO.FileStream fs2 = new System.IO.FileStream(filename, System.IO.FileMode.Create, System.IO.FileAccess.ReadWrite, System.IO.FileShare.None)) + using (System.IO.Stream s = snapshot.OpenRead(filename)) { } - Console.WriteLine(string.Format("Could open locked file {0}, cannot test", filename)); - Console.WriteLine("* Test failed"); - return; + Console.WriteLine(string.Format("Could open locked file {0}, through snapshot", filename)); } catch (Exception ex) { - Console.WriteLine(string.Format("The file {0} was correctly locked, message: {1}", filename, ex.Message)); - } - - Console.WriteLine("Creating snapshot for folder: {0}", args[0]); - Console.WriteLine("If this fails, try to run as " + (Utility.Utility.IsClientLinux ? "root" : "Administrator")); - using (ISnapshotService snapshot = SnapshotUtility.CreateSnapshot(new string[] { args[0] }, options)) - { - Console.WriteLine("Attempting to read locked file via snapshot"); - try - { - using (System.IO.Stream s = snapshot.OpenRead(filename)) - { } - - Console.WriteLine(string.Format("Could open locked file {0}, through snapshot", filename)); - } - catch (Exception ex) - { - Console.WriteLine(string.Format("The file {0} was locked even through snapshot, message: {1}", filename, ex)); - Console.WriteLine("* Test failed"); - return; - } + Console.WriteLine(string.Format("The file {0} was locked even through snapshot, message: {1}", filename, ex)); + Console.WriteLine("* Test failed"); + return; } } - - Console.WriteLine("* Test passed"); } + + Console.WriteLine("* Test passed"); } catch (Exception ex) { diff --git a/Duplicati/Library/Snapshots/SnapshotBase.cs b/Duplicati/Library/Snapshots/SnapshotBase.cs new file mode 100644 index 000000000..da32263f2 --- /dev/null +++ b/Duplicati/Library/Snapshots/SnapshotBase.cs @@ -0,0 +1,199 @@ +#region Disclaimer / License +// Copyright (C) 2015, The Duplicati Team +// http://www.duplicati.com, info@duplicati.com +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// +using System.Linq; + + +#endregion +using System; +using System.Collections.Generic; +using System.IO; + +namespace Duplicati.Library.Snapshots +{ + public abstract class SnapshotBase : ISnapshotService + { + #region ISnapshotService + + /// + /// Enumerates all files and folders in the shadow copy + /// + /// Source paths to enumerate + /// The callback to invoke with each found path + /// The callback used to report errors + public IEnumerable EnumerateFilesAndFolders(IEnumerable sources, Utility.Utility.EnumerationFilterDelegate callback, Utility.Utility.ReportAccessError errorCallback) + { + // Add trailing slashes to folders + var sanitizedSources = sources.Select(x => Directory.Exists(x) ? Utility.Utility.AppendDirSeparator(x) : x).ToList(); + + return sanitizedSources.SelectMany( + s => Utility.Utility.EnumerateFileSystemEntries(s, callback, ListFolders, ListFiles, GetAttributes, errorCallback) + ); + } + + /// + /// Gets the last write time of a given file in UTC + /// + /// The full path to the file in non-snapshot format + /// The last write time of the file + public virtual DateTime GetLastWriteTimeUtc(string localPath) + { + return File.GetLastWriteTimeUtc(localPath); + } + + /// + /// Gets the last write time of a given file in UTC + /// + /// The full path to the file in non-snapshot format + /// The last write time of the file + public virtual DateTime GetCreationTimeUtc(string localPath) + { + return File.GetCreationTimeUtc(localPath); + } + + /// + /// Opens a file for reading + /// + /// The full path to the file in non-snapshot format + /// An open filestream that can be read + public virtual Stream OpenRead(string localPath) + { + return File.Open(localPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + } + + /// + /// Returns the size of a file + /// + /// The full path to the file in non-snapshot format + /// The lenth of the file + public virtual long GetFileSize(string localPath) + { + return new FileInfo(localPath).Length; + } + + /// + /// Returns the symlink target if the entry is a symlink, and null otherwise + /// + /// The file or folder to examine + /// The symlink target + public abstract string GetSymlinkTarget(string localPath); + + /// + /// Gets the attributes for the given file or folder + /// + /// The file attributes + /// The file or folder to examine + public virtual FileAttributes GetAttributes(string localPath) + { + return File.GetAttributes(NormalizePath(localPath)); + } + + /// + /// Gets the metadata for the given file or folder + /// + /// The metadata for the given file or folder + /// The file or folder to examine + /// A flag indicating if the target is a symlink + /// A flag indicating if a symlink should be followed + public abstract Dictionary GetMetadata(string localPath, bool isSymlink, bool followSymlink); + + /// + /// Gets a value indicating if the path points to a block device + /// + /// true if this instance is a block device; otherwise, false. + /// The file or folder to examine + public virtual bool IsBlockDevice(string localPath) + { + return false; + } + + /// + /// Gets a unique hardlink target ID + /// + /// The hardlink ID + /// The file or folder to examine + public virtual string HardlinkTargetID(string localPath) + { + return null; + } + + /// + public abstract string ConvertToLocalPath(string snapshotPath); + + /// + public abstract string ConvertToSnapshotPath(string localPath); + + /// + public bool FileExists(string localFilePath) + { + return new FileInfo(ConvertToSnapshotPath(localFilePath)).Exists; + } + + /// + public bool FolderExists(string localFolderPath) + { + return new DirectoryInfo(ConvertToSnapshotPath(localFolderPath)).Exists; + } + + /// + public bool PathExists(string localFileOrFolderPath) + { + return FileExists(localFileOrFolderPath) || FolderExists(localFileOrFolderPath); + } + + #endregion + + /// + /// Normalizes a path before calling system methods + /// + /// The path to normalize. + /// The normalized path. + public static string NormalizePath(string path) + { + var p = Path.GetFullPath(path); + + // This should not be required, but some versions of Mono apperently do not strip the trailing slash + return p.Length > 1 && p[p.Length - 1] == Path.DirectorySeparatorChar ? p.Substring(0, p.Length - 1) : p; + } + + /// + /// Lists all folders in the given folder + /// + /// All folders found in the folder + /// The folder to examinate + protected virtual string[] ListFolders(string localFolderPath) + { + return Directory.GetDirectories(localFolderPath); + } + + /// + /// Lists all files in the given folder + /// + /// All folders found in the folder + /// The folder to examinate + protected virtual string[] ListFiles(string localFolderPath) + { + return Directory.GetFiles(localFolderPath); + } + + public virtual void Dispose() + { + + } + } +} \ No newline at end of file diff --git a/Duplicati/Library/Snapshots/SnapshotUtility.cs b/Duplicati/Library/Snapshots/SnapshotUtility.cs index 4875b8239..8a6185171 100644 --- a/Duplicati/Library/Snapshots/SnapshotUtility.cs +++ b/Duplicati/Library/Snapshots/SnapshotUtility.cs @@ -17,10 +17,9 @@ // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA // #endregion -using System; + using System.Collections.Generic; using System.IO; -using System.Text; namespace Duplicati.Library.Snapshots { @@ -35,7 +34,7 @@ namespace Duplicati.Library.Snapshots /// The list of folders to create snapshots of /// A set of commandline options /// The ISnapshotService implementation - public static ISnapshotService CreateSnapshot(string[] folders, Dictionary options) + public static ISnapshotService CreateSnapshot(IEnumerable folders, Dictionary options) { return Utility.Utility.IsClientLinux @@ -55,9 +54,9 @@ namespace Duplicati.Library.Snapshots /// A set of commandline options /// The ISnapshotService implementation [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] - private static ISnapshotService CreateLinuxSnapshot(string[] folders, Dictionary options) + private static ISnapshotService CreateLinuxSnapshot(IEnumerable folders, Dictionary options) { - return new LinuxSnapshot(folders, options); + return new LinuxSnapshot(folders); } /// @@ -67,7 +66,7 @@ namespace Duplicati.Library.Snapshots /// A set of commandline options /// The ISnapshotService implementation [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] - private static ISnapshotService CreateWindowsSnapshot(string[] folders, Dictionary options) + private static ISnapshotService CreateWindowsSnapshot(IEnumerable folders, Dictionary options) { return new WindowsSnapshot(folders, options); } diff --git a/Duplicati/Library/Snapshots/Strings.cs b/Duplicati/Library/Snapshots/Strings.cs index 7c1f6208c..2524a4947 100644 --- a/Duplicati/Library/Snapshots/Strings.cs +++ b/Duplicati/Library/Snapshots/Strings.cs @@ -12,9 +12,14 @@ Command: {1} {2}", message, executable, arguments); } public static string ScriptOutputError(string parameter, string message) { return LC.L(@"Script returned successfully, but the output was missing the {0} parameter: {1}", parameter, message); } } internal static class USNHelper { + public static string PathResolveError { get { return LC.L(@"Unable to determine full file path for USN entry"); } } + public static string JournalEntriesDeleted { get { return LC.L(@"USN journal entries were purged since last scan"); } } public static string EmptyResponseError { get { return LC.L(@"Unexpected empty response while enumerating"); } } public static string LinuxNotSupportedError { get { return LC.L(@"USN is not supported on Linux"); } } public static string SafeGuardError { get { return LC.L(@"The number of files returned by USN was zero. This is likely an error. To remedy this, USN has been disabled."); } } + public static string UnexpectedPathFormat { get { return LC.L(@"Unexpected path format encountered"); } } + public static string UnsupportedUsnVersion { get { return LC.L(@"Unsupported USN journal version."); } } + } internal static class WinNativeMethod { public static string MissingBackupPrivilegeError { get { return LC.L(@"Calling process does not have the backup privilege"); } } diff --git a/Duplicati/Library/Snapshots/SystemIOLinux.cs b/Duplicati/Library/Snapshots/SystemIOLinux.cs index 56ea72b4b..f994910aa 100644 --- a/Duplicati/Library/Snapshots/SystemIOLinux.cs +++ b/Duplicati/Library/Snapshots/SystemIOLinux.cs @@ -27,17 +27,17 @@ namespace Duplicati.Library.Snapshots #region ISystemIO implementation public void DirectoryDelete(string path) { - Directory.Delete(NoSnapshot.NormalizePath(path)); + Directory.Delete(SnapshotBase.NormalizePath(path)); } public void DirectoryCreate(string path) { - Directory.CreateDirectory(NoSnapshot.NormalizePath(path)); + Directory.CreateDirectory(SnapshotBase.NormalizePath(path)); } public bool DirectoryExists(string path) { - return Directory.Exists(NoSnapshot.NormalizePath(path)); + return Directory.Exists(SnapshotBase.NormalizePath(path)); } public void FileDelete(string path) @@ -92,7 +92,7 @@ namespace Duplicati.Library.Snapshots public FileAttributes GetFileAttributes(string path) { - return File.GetAttributes(NoSnapshot.NormalizePath(path)); + return File.GetAttributes(SnapshotBase.NormalizePath(path)); } public void SetFileAttributes(string path, FileAttributes attributes) @@ -107,12 +107,12 @@ namespace Duplicati.Library.Snapshots public string GetSymlinkTarget(string path) { - return UnixSupport.File.GetSymlinkTarget(NoSnapshot.NormalizePath(path)); + return UnixSupport.File.GetSymlinkTarget(SnapshotBase.NormalizePath(path)); } public string PathGetDirectoryName(string path) { - return Path.GetDirectoryName(NoSnapshot.NormalizePath(path)); + return Path.GetDirectoryName(SnapshotBase.NormalizePath(path)); } public IEnumerable EnumerateFileSystemEntries(string path) @@ -137,22 +137,22 @@ namespace Duplicati.Library.Snapshots public void DirectorySetLastWriteTimeUtc(string path, DateTime time) { - Directory.SetLastWriteTimeUtc(NoSnapshot.NormalizePath(path), time); + Directory.SetLastWriteTimeUtc(SnapshotBase.NormalizePath(path), time); } public void DirectorySetCreationTimeUtc(string path, DateTime time) { - Directory.SetCreationTimeUtc(NoSnapshot.NormalizePath(path), time); + Directory.SetCreationTimeUtc(SnapshotBase.NormalizePath(path), time); } public DateTime DirectoryGetLastWriteTimeUtc(string path) { - return Directory.GetLastWriteTimeUtc(NoSnapshot.NormalizePath(path)); + return Directory.GetLastWriteTimeUtc(SnapshotBase.NormalizePath(path)); } public DateTime DirectoryGetCreationTimeUtc(string path) { - return Directory.GetCreationTimeUtc(NoSnapshot.NormalizePath(path)); + return Directory.GetCreationTimeUtc(SnapshotBase.NormalizePath(path)); } public void FileMove(string source, string target) @@ -162,17 +162,17 @@ namespace Duplicati.Library.Snapshots public long FileLength(string path) { - return new System.IO.FileInfo(path).Length; + return new FileInfo(path).Length; } public void DirectoryDelete(string path, bool recursive) { - Directory.Delete(NoSnapshot.NormalizePath(path), recursive); + Directory.Delete(SnapshotBase.NormalizePath(path), recursive); } public Dictionary GetMetadata(string file, bool isSymlink, bool followSymlink) { - var f = NoSnapshot.NormalizePath(file); + var f = SnapshotBase.NormalizePath(file); var dict = new Dictionary(); var n = UnixSupport.File.GetExtendedAttributes(f, isSymlink, followSymlink); @@ -199,7 +199,7 @@ namespace Duplicati.Library.Snapshots if (data == null) return; - var f = NoSnapshot.NormalizePath(file); + var f = SnapshotBase.NormalizePath(file); foreach(var x in data.Where(x => x.Key.StartsWith("unix-ext:", StringComparison.Ordinal)).Select(x => new KeyValuePair(x.Key.Substring("unix-ext:".Length), Convert.FromBase64String(x.Value)))) UnixSupport.File.SetExtendedAttribute(f, x.Key, x.Value); diff --git a/Duplicati/Library/Snapshots/USNHelper.cs b/Duplicati/Library/Snapshots/USNHelper.cs deleted file mode 100644 index d21703af4..000000000 --- a/Duplicati/Library/Snapshots/USNHelper.cs +++ /dev/null @@ -1,518 +0,0 @@ -#region Disclaimer / License -// Copyright (C) 2015, The Duplicati Team -// http://www.duplicati.com, info@duplicati.com -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public -// License along with this library; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -// -#endregion -using System; -using System.Collections.Generic; -using System.Text; -using Microsoft.Win32.SafeHandles; -using System.Runtime.InteropServices; -using System.ComponentModel; - -namespace Duplicati.Library.Snapshots -{ - /// - /// Class that encapsulates USN access to a single volume - /// - public class USNHelper : IDisposable - { - /// - /// The path this USN points to - /// - private string m_path; - - /// - /// The safe filehandle - /// - private SafeFileHandle m_volumeHandle; - - /// - /// The current USN journal from the device - /// - private Win32USN.USN_JOURNAL_DATA m_journal; - - /// - /// The FileNameReferenceNumber for the root folder - /// - private ulong m_volumeRootFileNameReferenceNumber; - - /// - /// This is a cache of all the filesystem entries found on the drive - /// - private List> m_entryList; - - /// - /// Constructs a new USN helper instance - /// - /// The path to the folder to perform USN services - public USNHelper(string path) - : this(path, null) - { - } - - - /// - /// Constructs a new USN helper instance - /// - /// The path to the folder to perform USN services - /// The root volume where the USN lookup is performed - internal USNHelper(string path, string volumeRoot) - { - if (Utility.Utility.IsClientLinux) - throw new Duplicati.Library.Interface.UserInformationException(Strings.USNHelper.LinuxNotSupportedError, "UsnOnLinuxNotSupported"); - - if (!System.IO.Path.IsPathRooted(path)) - throw new Exception(string.Format("Path {0} is not rooted", path)); - - m_path = Utility.Utility.AppendDirSeparator(path); - - try - { - string devicename = @"\\.\" + System.IO.Path.GetPathRoot(path).TrimEnd('\\'); - if (volumeRoot != null) - volumeRoot = volumeRoot.TrimEnd('\\'); - - m_volumeHandle = Win32USN.CreateFile(volumeRoot == null ? devicename : volumeRoot, Win32USN.EFileAccess.GenericRead, Win32USN.EFileShare.ReadWrite, IntPtr.Zero, Win32USN.ECreationDisposition.OpenExisting, Win32USN.EFileAttributes.BackupSemantics, IntPtr.Zero); - if (m_volumeHandle == null || m_volumeHandle.IsInvalid) - throw new Win32Exception(Marshal.GetLastWin32Error()); - - uint bytesReturned = 0; - if (!Win32USN.DeviceIoControl(m_volumeHandle, Win32USN.EIOControlCode.FsctlQueryUsnJournal, null, 0, out m_journal, (uint)Marshal.SizeOf(typeof(Win32USN.USN_JOURNAL_DATA)), ref bytesReturned, IntPtr.Zero)) - throw new Win32Exception(Marshal.GetLastWin32Error()); - - Win32USN.BY_HANDLE_FILE_INFORMATION fileInfo; - using (SafeFileHandle driveHandle = Win32USN.CreateFile(System.IO.Path.GetPathRoot(path), Win32USN.EFileAccess.GenericRead, Win32USN.EFileShare.ReadWrite, IntPtr.Zero, Win32USN.ECreationDisposition.OpenExisting, Win32USN.EFileAttributes.BackupSemantics, IntPtr.Zero)) - if (!Win32USN.GetFileInformationByHandle(driveHandle, out fileInfo)) - throw new Win32Exception(Marshal.GetLastWin32Error()); - - m_volumeRootFileNameReferenceNumber = ((ulong)fileInfo.FileIndexHigh << 32) | ((ulong)fileInfo.FileIndexLow); - } - catch - { - if (m_volumeHandle != null) - { - m_volumeHandle.Dispose(); - m_volumeHandle = null; - } - - throw; - } - - if (this.FileSystemEntries.Count == 0) - throw new Exception(Strings.USNHelper.SafeGuardError); - } - - /// - /// Helper function to support the Duplicati enumeration callback system - /// - /// The root path to look in and use as filter base - /// The callback function that collects the output - public void EnumerateFilesAndFolders(string rootpath, Duplicati.Library.Utility.Utility.EnumerationFilterDelegate callback) - { - //Under normal enumeration, the filter will prevent visiting subfolders for excluded folders - //But when using USN all files/folders are present in the list, so we have to maintain - // a list of subfolders that are excluded from the set - System.Text.StringBuilder local_filter = new StringBuilder(); - System.Text.RegularExpressions.Regex excludedFolders = null; - - foreach (KeyValuePair r in this.Records) - if (r.Key.StartsWith(rootpath, Utility.Utility.ClientFilenameStringComparision)) - { - bool isFolder = (r.Value.FileAttributes & Win32USN.EFileAttributes.Directory) != 0; - - if (excludedFolders != null && excludedFolders.Match(r.Key).Success) - continue; - - if (isFolder) - { - if (!callback(rootpath, r.Key, (System.IO.FileAttributes)((int)r.Value.FileAttributes & 0xff))) - { - if (local_filter.Length != 0) - local_filter.Append("|"); - local_filter.Append("("); - local_filter.Append(Utility.Utility.ConvertGlobbingToRegExp(r.Key + "*")); - local_filter.Append(")"); - - if (Utility.Utility.IsFSCaseSensitive) - excludedFolders = new System.Text.RegularExpressions.Regex(local_filter.ToString()); - else - excludedFolders = new System.Text.RegularExpressions.Regex(local_filter.ToString(), System.Text.RegularExpressions.RegexOptions.IgnoreCase); - } - } - else - { - if (local_filter.Length == 0 || !excludedFolders.IsMatch(r.Key)) - callback(rootpath, r.Key, (System.IO.FileAttributes)((int)r.Value.FileAttributes & 0xff)); - } - } - } - - /// - /// Gets the USN JournalID for the current volume - /// - public long JournalID - { - get - { - return m_journal.UsnJournalID; - } - } - - /// - /// Gets the next USN number for the volume - /// - public long USN - { - get - { - return m_journal.NextUsn; - } - } - - - /// - /// Internal property to access the cached version of the file entry list - /// - private List> Records - { - get - { - if (m_entryList == null) - m_entryList = BuildUSNTable(0); - return m_entryList; - } - } - - /// - /// Returns a list of files or folders that have changed since the recorded USN - /// - /// The folder to find entries for - /// The last known USN - /// A list of changed files and folders - public List GetChangedFileSystemEntries(string sourceFolder, long lastUsn) - { - List result = new List(); - foreach (KeyValuePair r in this.Records) - if (r.Value.Usn >= lastUsn && r.Key.StartsWith(sourceFolder, Utility.Utility.ClientFilenameStringComparision)) - result.Add(r.Key); - - return result; - } - - public string GetChangeFlags(string entry) - { - foreach (KeyValuePair r in this.Records) - if (r.Key.Equals(entry, Utility.Utility.ClientFilenameStringComparision)) - return r.Value.Reason.ToString(); - - return ""; - - } - - public List GetRenamedFileSystemEntries(string sourceFolder, long lastUsn) - { - List result = new List(); - foreach (KeyValuePair r in this.Records) - if (r.Value.Usn >= lastUsn && ((r.Value.Reason & Win32USN.USNReason.USN_REASON_RENAME_NEW_NAME) != 0) && r.Key.StartsWith(sourceFolder, Utility.Utility.ClientFilenameStringComparision)) - result.Add(r.Key); - - return result; - } - - - /// - /// Returns a list of all files and folders in a given subfolder - /// - /// The folder to enumerat - /// A list of folders - public List GetFileSystemEntries(string sourceFolder) - { - List result = new List(); - foreach (KeyValuePair r in this.Records) - if (r.Key.StartsWith(sourceFolder, Utility.Utility.ClientFilenameStringComparision)) - result.Add(r.Key); - - return result; - } - - /// - /// Returns a list of files and folders found in the folder - /// - public List FileSystemEntries - { - get - { - List result = new List(); - foreach (KeyValuePair r in this.Records) - if (r.Key.StartsWith(m_path, Utility.Utility.ClientFilenameStringComparision)) - result.Add(r.Key); - - return result; - } - } - - /// - /// Returns a list of files found in the folder - /// - public List Files - { - get - { - List result = new List(); - foreach (KeyValuePair r in this.Records) - if (r.Key.StartsWith(m_path, Utility.Utility.ClientFilenameStringComparision) && (r.Value.FileAttributes & Win32USN.EFileAttributes.Directory) == 0) - result.Add(r.Key); - - return result; - } - } - - /// - /// Returns a list of folders found in the folder - /// - public List Folders - { - get - { - List result = new List(); - foreach (KeyValuePair r in this.Records) - if (r.Key.StartsWith(m_path, Utility.Utility.ClientFilenameStringComparision) && (r.Value.FileAttributes & Win32USN.EFileAttributes.Directory) != 0) - result.Add(r.Key); - - return result; - } - } - - /// - /// Cached copy of the USN_RECORD size - /// - static readonly int USN_RECORD_SIZE = Marshal.SizeOf(typeof(Win32USN.USN_RECORD)); - - /// - /// Helper function to work with unmanaged memory - /// - /// The number of bytes in buffer - /// The pointer to the start of the buffer - /// The list of records to update with results - /// A flag indicating if the end of the list is reached - /// The next USN or FRN number, depending on what pointer was passed - private long ExtractUsnEntries(uint bytesRead, IntPtr startOffset, List> records, out bool moreEntries) - { - int bytesLeft = (int)bytesRead; - IntPtr curPtr = startOffset; - - if (bytesLeft < 8) - throw new Exception(Strings.USNHelper.EmptyResponseError); - - long nextUsn = Marshal.ReadInt64(curPtr); - curPtr = new IntPtr(curPtr.ToInt64() + 8); - bytesLeft -= 8; - - moreEntries = (bytesLeft > USN_RECORD_SIZE); - - while (bytesLeft > USN_RECORD_SIZE) - { - Win32USN.USN_RECORD r = (Win32USN.USN_RECORD)Marshal.PtrToStructure(curPtr, typeof(Win32USN.USN_RECORD)); - - //According to MSDN, apps should not handle data less than 2.0, http://msdn.microsoft.com/en-us/library/aa365722.aspx - if (r.MajorVersion >= 2) - { - string filename = Marshal.PtrToStringUni(new IntPtr(curPtr.ToInt64() + r.FileNameOffset), r.FileNameLength / sizeof(char)); - - if (r.Usn > m_journal.NextUsn) - { - moreEntries = false; - break; - } - - records.Add(new KeyValuePair(filename, r)); - } - bytesLeft -= (int)r.RecordLength; - curPtr = new IntPtr(curPtr.ToInt64() + r.RecordLength); - } - - return nextUsn; - } - - /// - /// Returns a list of all files and folders changed since the USN - /// - /// The USN number to start the list from, set to zero to get all - /// A list of files and folders changed since the USN - private List> BuildUSNTable(long startUsn) - { - const int ALLOCATED_MEMORY = 64 * 1024; - - IntPtr allocatedMemory = IntPtr.Zero; - List> records = new List>(); - - try - { - uint bytesRead = 0; - bool more = true; - allocatedMemory = Marshal.AllocHGlobal(ALLOCATED_MEMORY); - - Win32USN.MFT_ENUM_DATA startParams = new Win32USN.MFT_ENUM_DATA(); - startParams.StartFileReferenceNumber = 0; - startParams.LowUsn = Math.Max(startUsn, m_journal.LowestValidUsn); - startParams.HighUsn = m_journal.NextUsn; - - while (more) - { - if (!Win32USN.DeviceIoControl(m_volumeHandle, Win32USN.EIOControlCode.FsctlEnumUsnData, - ref startParams, (uint)Marshal.SizeOf(typeof(Win32USN.MFT_ENUM_DATA)), - allocatedMemory, ALLOCATED_MEMORY, - ref bytesRead, IntPtr.Zero)) - { - int errorCode = Marshal.GetLastWin32Error(); - - //If we get no error or EOF the enumeration is completed - if (errorCode == Win32USN.ERROR_HANDLE_EOF || errorCode == Win32USN.ERROR_SUCCESS) - break; - else - throw new Win32Exception(errorCode); - } - - startParams.StartFileReferenceNumber = (ulong)ExtractUsnEntries(bytesRead, allocatedMemory, records, out more); - } - - return ParseRecordList(records); - } - finally - { - if (allocatedMemory != IntPtr.Zero) - { - Marshal.FreeHGlobal(allocatedMemory); - allocatedMemory = IntPtr.Zero; - } - } - } - - /// - /// Calculates the full path of each entry in the USN table - /// - /// The list of records with local names - /// A list of USN entries with full path - private List> ParseRecordList(List> records) - { - string rootDir = System.IO.Path.GetPathRoot(m_path); - - List> result = new List>(); - Dictionary folderLookup = new Dictionary(); - - for (int i = 0; i < records.Count; i++) - if ((records[i].Value.FileAttributes & Win32USN.EFileAttributes.Directory) != 0) - folderLookup.Add(records[i].Value.FileReferenceNumber, i); - - //Loop through each record - foreach (KeyValuePair r in records) - { - //Build the path, starting at the file/folder entry - List items = new List(); - items.Add(r.Key + (((r.Value.FileAttributes & Win32USN.EFileAttributes.Directory) == 0) ? "" : "\\")); - - KeyValuePair cur = r; - - //Walk back up the chain as far as we can go - while (folderLookup.ContainsKey(cur.Value.ParentFileReferenceNumber)) - { - cur = records[folderLookup[cur.Value.ParentFileReferenceNumber]]; - items.Insert(0, cur.Key); - } - - //If the parent entry is indeed the drive, record this entry - if (cur.Value.ParentFileReferenceNumber == m_volumeRootFileNameReferenceNumber) - result.Add(new KeyValuePair(System.IO.Path.Combine(rootDir, string.Join(System.IO.Path.DirectorySeparatorChar.ToString(), items.ToArray())), r.Value)); - } - - return result; - } - - /// - /// Unused internal function that can be used to read all USN records - /// - /// The USN number to start from - private void GetChangedItems(long lastUsn) - { - const int ALLOCATED_MEMORY = 64 * 1024; - - IntPtr allocatedMemory = IntPtr.Zero; - List> records = new List>(); - - try - { - uint bytesRead = 0; - bool more = true; - allocatedMemory = Marshal.AllocHGlobal(ALLOCATED_MEMORY); - - Win32USN.READ_USN_JOURNAL_DATA startParams = new Win32USN.READ_USN_JOURNAL_DATA(); - startParams.UsnJournalID = m_journal.UsnJournalID; - startParams.StartUsn = lastUsn; - startParams.ReasonMask = Win32USN.USNReason.USN_REASON_ANY; - startParams.ReturnOnlyOnClose = 0; - startParams.Timeout = 0; - startParams.BytesToWaitFor = 0; - - while (more) - { - if (!Win32USN.DeviceIoControl(m_volumeHandle, Win32USN.EIOControlCode.FsctlReadUsnJournal, - startParams, (uint)Marshal.SizeOf(typeof(Win32USN.READ_USN_JOURNAL_DATA)), - allocatedMemory, ALLOCATED_MEMORY, - ref bytesRead, IntPtr.Zero)) - { - int errorCode = Marshal.GetLastWin32Error(); - - //If we get no error or EOF the enumeration is completed - if (errorCode == Win32USN.ERROR_HANDLE_EOF || errorCode == Win32USN.ERROR_SUCCESS) - break; - else - throw new Win32Exception(errorCode); - } - - startParams.StartUsn = ExtractUsnEntries(bytesRead, allocatedMemory, records, out more); - } - - //Records now contains all Usn entries - } - finally - { - if (allocatedMemory != IntPtr.Zero) - { - Marshal.FreeHGlobal(allocatedMemory); - allocatedMemory = IntPtr.Zero; - } - } - } - - #region IDisposable Members - - /// - /// Cleans up any resources held, including the volume handle - /// - public void Dispose() - { - if (m_volumeHandle != null) - { - m_volumeHandle.Dispose(); - m_volumeHandle = null; - } - } - - #endregion - } -} diff --git a/Duplicati/Library/Snapshots/USNJournal.cs b/Duplicati/Library/Snapshots/USNJournal.cs new file mode 100644 index 000000000..e7c184d68 --- /dev/null +++ b/Duplicati/Library/Snapshots/USNJournal.cs @@ -0,0 +1,568 @@ +#region Disclaimer / License +// Copyright (C) 2015, The Duplicati Team +// http://www.duplicati.com, info@duplicati.com +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// +#endregion + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics; +using System.Linq; +using System.Runtime.InteropServices; +using System.Runtime.Serialization; +using Microsoft.Win32.SafeHandles; + +namespace Duplicati.Library.Snapshots +{ + /// + /// Class that encapsulates USN journal access to a single volume + /// + public sealed class USNJournal : IDisposable + { + private readonly Action m_cancelHandler; + + [Flags] + public enum ChangeReason + { + Modified = 1, + Created = 2, + Deleted = 4, + RenamedFrom = 8, + RenamedTo = 16, + Any = Modified|Created|Deleted|RenamedFrom|RenamedTo + } + + [Flags] + public enum EntryType + { + Directory = 1, + File = 2, + Any = Directory|File + } + + /// + /// The volume this USN points to + /// + private readonly string m_volume; + + /// + /// The FileNameReferenceNumber for the root folder + /// + private readonly ulong m_volumeRootRefNumber; + + /// + /// This is a cache of all the filesystem entries found on the drive + /// + private IReadOnlyCollection m_entryList; + + /// + /// The current USN journal from the device + /// + private readonly Win32USN.USN_JOURNAL_DATA_V0 m_journal; + + /// + /// Start USN of current cached m_entryList + /// + private long m_minUsn; + + /// + /// The safe filehandle + /// + private SafeFileHandle m_volumeHandle; + + /// + /// Constructs a new USN helper instance + /// + /// The root volume where the USN lookup is performed + /// Callback for aborting lengthy operations. Throw OperationCancelledException() to abort. + internal USNJournal(string volumeRoot, Action cancelHandler) + { + if (Utility.Utility.IsClientLinux) + throw new Interface.UserInformationException(Strings.USNHelper.LinuxNotSupportedError, "UsnOnLinuxNotSupported"); + + m_cancelHandler = cancelHandler; + m_volume = Utility.Utility.AppendDirSeparator(volumeRoot); + + try + { + var device = GetDeviceNameFromPath(m_volume); + + m_volumeHandle = Win32USN.CreateFile(device, Win32USN.FileAccess.GenericRead, + Win32USN.FileShare.ReadWrite, IntPtr.Zero, Win32USN.CreationDisposition.OpenExisting, + Win32USN.FileAttributes.BackupSemantics, IntPtr.Zero); + + if (m_volumeHandle == null || m_volumeHandle.IsInvalid) + throw new Win32Exception(Marshal.GetLastWin32Error()); + + Win32USN.ControlWithOutput(m_volumeHandle, Win32USN.FsCtl.QueryUSNJournal, ref m_journal); + + m_volumeRootRefNumber = GetFileRefNumber(volumeRoot); + } + catch + { + if (m_volumeHandle != null) + { + m_volumeHandle.Dispose(); + m_volumeHandle = null; + } + + throw; + } + } + + /// + /// Gets the USN JournalID for the current volume + /// + public long JournalId => m_journal.UsnJournalID; + + /// + /// Gets the next USN number for the volume + /// + public long NextUsn => m_journal.NextUsn; + + /// + /// Returns a list of files or folders that have changed since the recorded USN + /// + /// The folder to find entries for + /// Minimum USN of entry + /// Filter expression for change reason + /// A list of tuples with changed files and folders and their type + public IEnumerable> GetChangedFileSystemEntries(string sourceFolder, long minUsn, ChangeReason reason = ChangeReason.Any) + { + foreach (var r in GetRecords(minUsn)) + { + if (r.UsnRecord.Usn >= minUsn + && (reason == ChangeReason.Any || (MapChangeReason(r.UsnRecord.Reason) & reason) != 0) + && (string.IsNullOrEmpty(sourceFolder) || r.FullPath.Equals(sourceFolder, + Utility.Utility.ClientFilenameStringComparision) + || Utility.Utility.IsPathBelowFolder(r.FullPath, sourceFolder))) + { + yield return Tuple.Create(r.FullPath, + r.UsnRecord.FileAttributes.HasFlag(Win32USN.FileAttributes.Directory) + ? EntryType.Directory + : EntryType.File); + } + } + } + + public static string GetVolumeRootFromPath(string path) + { + if (path == null) + throw new Exception(Strings.USNHelper.UnexpectedPathFormat); + + return new System.IO.DirectoryInfo(path).Root.FullName; + } + + public static string GetDeviceNameFromPath(string path) + { + return @"\\.\" + GetVolumeRootFromPath(path).TrimEnd('\\'); + } + + /// + /// Internal method to initially create and then access the cached version of the file entry list + /// + private IEnumerable GetRecords(long minUsn) + { + if (m_entryList == null || m_minUsn != minUsn) + { + m_minUsn = minUsn; + m_entryList = ResolveFullPaths(GetRawRecords(minUsn)); + } + + return m_entryList; + } + + /// + /// Get NTFS file reference number (FRN) from path + /// + /// Input path + /// NTFS file reference number + private static ulong GetFileRefNumber(string filePath) + { + Win32USN.BY_HANDLE_FILE_INFORMATION fileInfo; + using (var driveHandle = Win32USN.CreateFile(filePath, Win32USN.FileAccess.GenericRead, + Win32USN.FileShare.ReadWrite, IntPtr.Zero, Win32USN.CreationDisposition.OpenExisting, + Win32USN.FileAttributes.BackupSemantics, IntPtr.Zero)) + { + if (!Win32USN.GetFileInformationByHandle(driveHandle, out fileInfo)) + throw new Win32Exception(Marshal.GetLastWin32Error()); + } + + return ((ulong) fileInfo.FileIndexHigh << 32) | fileInfo.FileIndexLow; + } + + + /// + /// Extract USN_RECORD_V2 from buffer + /// + /// + /// + /// Entry filename + /// + private static Win32USN.USN_RECORD_V2 GetBufferedEntry(IntPtr bufferPointer, long offset, out string fileName) + { + var entryPointer = new IntPtr(bufferPointer.ToInt64() + offset); + var nativeEntry = (Win32USN.USN_RECORD_V2) Marshal.PtrToStructure(entryPointer, typeof(Win32USN.USN_RECORD_V2)); + + //TODO: add support for V3 records + if (nativeEntry.MajorVersion != 2) + throw new Exception(Strings.USNHelper.UnsupportedUsnVersion); + + var filenamePointer = new IntPtr(bufferPointer.ToInt64() + offset + nativeEntry.FileNameOffset); + fileName = Marshal.PtrToStringUni(filenamePointer, nativeEntry.FileNameLength / sizeof(char)); + return nativeEntry; + } + + /// + /// Enumerates all USN records in a raw data block + /// + /// Data block + /// Entries + private static IEnumerable EnumerateRecords(IReadOnlyCollection entryData) + { + if (entryData.Count <= sizeof(long)) + yield break; + + var bufferHandle = GCHandle.Alloc(entryData, GCHandleType.Pinned); + try + { + var bufferPointer = bufferHandle.AddrOfPinnedObject(); + + long offset = sizeof(long); + while (offset < entryData.Count) + { + var entry = GetBufferedEntry(bufferPointer, offset, out var fileName); + + yield return new Record(entry, fileName); + + offset += entry.RecordLength; + } + } + finally + { + bufferHandle.Free(); + } + } + + /// + /// Returns collection of USN records, starting at startUSN + /// + /// The USN number to start the list from, set to zero to get all + /// A list of files and folders changed since the USN + private ICollection GetRawRecords(long startUsn) + { + var records = new List(); + + var readData = new Win32USN.READ_USN_JOURNAL_DATA_V0 + { + StartUsn = Math.Max(startUsn, m_journal.LowestValidUsn), + ReasonMask = Win32USN.USNReason.USN_REASON_BASIC_INFO_CHANGE | + Win32USN.USNReason.USN_REASON_DATA_EXTEND | + Win32USN.USNReason.USN_REASON_DATA_OVERWRITE | + Win32USN.USNReason.USN_REASON_DATA_TRUNCATION | + Win32USN.USNReason.USN_REASON_EA_CHANGE | + Win32USN.USNReason.USN_REASON_FILE_CREATE | + Win32USN.USNReason.USN_REASON_FILE_DELETE | + Win32USN.USNReason.USN_REASON_HARD_LINK_CHANGE | + Win32USN.USNReason.USN_REASON_NAMED_DATA_EXTEND | + Win32USN.USNReason.USN_REASON_NAMED_DATA_OVERWRITE | + Win32USN.USNReason.USN_REASON_NAMED_DATA_TRUNCATION | + Win32USN.USNReason.USN_REASON_RENAME_NEW_NAME | + Win32USN.USNReason.USN_REASON_RENAME_OLD_NAME | + Win32USN.USNReason.USN_REASON_REPARSE_POINT_CHANGE | + Win32USN.USNReason.USN_REASON_SECURITY_CHANGE | + Win32USN.USNReason.USN_REASON_STREAM_CHANGE, + ReturnOnlyOnClose = 0, + Timeout = 0, + BytesToWaitFor = 0, + UsnJournalID = m_journal.UsnJournalID + }; + + var bufferSize = 4096; // larger buffer returns more record, but pervents user from cancelling operation for a longer time + while (readData.StartUsn < m_journal.NextUsn) + { + // permit cancellation + m_cancelHandler(); + + if (!Win32USN.ControlWithInput(m_volumeHandle, Win32USN.FsCtl.ReadUSNJournal, + ref readData, bufferSize, out var entryData)) + { + var e = Marshal.GetLastWin32Error(); + if (e == Win32USN.ERROR_HANDLE_EOF || e == Win32USN.ERROR_SUCCESS) + break; + + if (e == Win32USN.ERROR_INSUFFICIENT_BUFFER) + { + bufferSize = bufferSize * 2; + continue; + } + + if (e == Win32USN.ERROR_JOURNAL_ENTRY_DELETED) + throw new UsnJournalSoftFailureException(Strings.USNHelper.JournalEntriesDeleted, new Win32Exception(e)); + + throw new Win32Exception(e); + } + + records.AddRange(EnumerateRecords(entryData).TakeWhile(rec => rec.UsnRecord.Usn >= startUsn && rec.UsnRecord.Usn < m_journal.NextUsn)); + readData.StartUsn = Marshal.ReadInt64(entryData, 0); + } + + return records; + } + + /// + /// Retrieves a USN_RECORD_V2 by file reference number (FRN, not USN!) + /// + /// File reference number + /// Returned entry if successful; null otherwise + private Record GetRecordByFileRef(ulong frn) + { + var enumData = new Win32USN.MFT_ENUM_DATA + { + StartFileReferenceNumber = frn, + LowUsn = 0, + HighUsn = m_journal.NextUsn + }; + + var bufferSize = 512; + byte[] entryData; + while (!Win32USN.ControlWithInput(m_volumeHandle, Win32USN.FsCtl.EnumUSNData, + ref enumData, bufferSize, out entryData)) + { + var e = Marshal.GetLastWin32Error(); + if (e != Win32USN.ERROR_INSUFFICIENT_BUFFER) + return null; + + // retry, increasing buffer size + bufferSize = bufferSize * 2; + } + + // not really a foreach: we only check the first record + foreach (var rec in EnumerateRecords(entryData)) + if (rec.UsnRecord.FileReferenceNumber == frn) + return rec; + + return null; + } + + /// + /// Calculates the full path of each entry in the USN table + /// + /// The list of records with local names + /// A list of USN entries with full path + private List ResolveFullPaths(ICollection records) + { + // initialize file ref-nr (FRN) to path/parent-FRN look-up table + var cache = new Dictionary(); + foreach (var rec in records) + { + if (rec.UsnRecord.FileAttributes.HasFlag(Win32USN.FileAttributes.Directory)) + { + if (!cache.TryGetValue(rec.UsnRecord.FileReferenceNumber, out var e)) + { + e = new SortedRecords(); + cache.Add(rec.UsnRecord.FileReferenceNumber, e); + } + + e.Add(rec); + } + } + + // iterate through USN records + var result = new List(); + foreach (var rec in records) + { + var pathList = new LinkedList(); + pathList.AddFirst(rec); + + // walk back up the chain as far as we can go + var cur = rec; + while (true) + { + var parentRefNr = cur.UsnRecord.ParentFileReferenceNumber; + if (parentRefNr == m_volumeRootRefNumber) + break; // done + + if (!cache.TryGetValue(parentRefNr, out var parents)) + { + // parent FRN not found in look-up table, fetch it from change journal + var parentRecord = GetRecordByFileRef(parentRefNr); + parents = new SortedRecords(new List { parentRecord }); + cache.Add(parentRefNr, parents); + } + + // take parent entry having next smaller USN + var parent = parents.GetParentOf(cur); + if (parent == null) + throw new UsnJournalSoftFailureException(Strings.USNHelper.PathResolveError); + + pathList.AddFirst(parent); + + cur = parent; + } + + // generate full path + Debug.Assert(m_volume != null, nameof(m_volume) + " != null"); + var path = m_volume; + foreach (var r in pathList) + { + path = System.IO.Path.Combine(path, r.FileName); + } + + if (rec.UsnRecord.FileAttributes.HasFlag(Win32USN.FileAttributes.Directory)) + { + path = Utility.Utility.AppendDirSeparator(path); + } + + // set resolved path + rec.FullPath = path; + result.Add(rec); + } + + return result; + } + + private static ChangeReason MapChangeReason(Win32USN.USNReason reason) + { + if (reason.HasFlag(Win32USN.USNReason.USN_REASON_FILE_CREATE)) + return ChangeReason.Created; + + if (reason.HasFlag(Win32USN.USNReason.USN_REASON_FILE_DELETE)) + return ChangeReason.Deleted; + + if (reason.HasFlag(Win32USN.USNReason.USN_REASON_RENAME_OLD_NAME)) + return ChangeReason.RenamedFrom; + + if (reason.HasFlag(Win32USN.USNReason.USN_REASON_RENAME_NEW_NAME)) + return ChangeReason.RenamedTo; + + return ChangeReason.Modified; + } + + #region IDisposable Members + + /// + /// Cleans up any resources held, including the volume handle + /// + public void Dispose() + { + if (m_volumeHandle != null) + { + m_volumeHandle.Dispose(); + m_volumeHandle = null; + } + } + + #endregion + + private class SortedRecords + { + private bool m_isSorted; + private readonly List m_records; + + public SortedRecords(List recs = null) + { + m_records = recs ?? new List(); + m_isSorted = false; + } + + public void Add(Record rec) + { + m_records.Add(rec); + m_isSorted = false; + } + + private void Sort() + { + if (!m_isSorted) + { + m_records.Sort((lhs, rhs) => lhs.UsnRecord.Usn.CompareTo(rhs.UsnRecord.Usn)); + m_isSorted = true; + } + } + + public Record GetParentOf(Record usnRecord) + { + Sort(); + + // perform binary search + int index = m_records.BinarySearch(usnRecord, + Comparer.Create( + (left, right) => left.UsnRecord.Usn.CompareTo(right.UsnRecord.Usn))); + + if (index >= 0) + { + if (usnRecord.UsnRecord.Usn == 0) + return m_records[index]; + + throw new ArgumentException(nameof(usnRecord)); // exact match not possible unless dummy USN + } + + // obtain (MSDN) "the index of the next element that is larger than item" + index = ~index; + + if (index > 0) + return m_records[index - 1]; // return next smaller record + + if (index < m_records.Count && !m_records[index].UsnRecord.Reason + .HasFlag(Win32USN.USNReason.USN_REASON_RENAME_NEW_NAME)) + return m_records[index]; // return next larger record, unless it's a filename change + + if (index == m_records.Count) + return null; //TODO: possibly use other means to find record + + return null; + } + } + + private class Record + { + public Record(Win32USN.USN_RECORD_V2 record, string fileName) + { + UsnRecord = record; + FileName = fileName; + FullPath = null; + } + + public Win32USN.USN_RECORD_V2 UsnRecord { get; } + + public string FileName { get; } + + public string FullPath { get; set; } + } + } + + public class UsnJournalSoftFailureException : Exception + { + public UsnJournalSoftFailureException() + { + } + + public UsnJournalSoftFailureException(string message) : base(message) + { + } + + public UsnJournalSoftFailureException(string message, Exception innerException) : base(message, innerException) + { + } + + protected UsnJournalSoftFailureException(SerializationInfo info, StreamingContext context) : base(info, context) + { + } + } +} diff --git a/Duplicati/Library/Snapshots/UsnJournalService.cs b/Duplicati/Library/Snapshots/UsnJournalService.cs new file mode 100644 index 000000000..1d80c8871 --- /dev/null +++ b/Duplicati/Library/Snapshots/UsnJournalService.cs @@ -0,0 +1,193 @@ +#region Disclaimer / License + +// Copyright (C) 2015, The Duplicati Team +// http://www.duplicati.com, info@duplicati.com +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// + +#endregion + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using Duplicati.Library.Interface; +using Duplicati.Library.Logging; + +namespace Duplicati.Library.Snapshots +{ + public class UsnJournalService : IChangeJournalService + { + /// + /// The tag used for logging + /// + private static readonly string LOGTAG = Log.LogTagFromType(); + + private readonly Utility.Utility.ReportAccessError m_errorCallback; + private readonly Action m_cancelHandler; + + /// + /// Constructor. + /// + /// Callback for handling errors + /// Callback for aborting operation. Throw OperationCancelledException() to abort. + public UsnJournalService(Utility.Utility.ReportAccessError errorCallback, Action cancelHandler) + { + m_errorCallback = errorCallback; + m_cancelHandler = cancelHandler; + } + + #region IChangeJournalService Members + + /// + public FilterData FilterSources(IEnumerable sources, string filterHash, IEnumerable journalData) + { + // create lookup for journal data + var journalDataDict = journalData.ToDictionary(data => data.Volume); + + // prepare result + var result = new FilterData + { + Files = new HashSet(Utility.Utility.ClientFilenameStringComparer), + Folders = new List(), + JournalData = new List() + }; + + // iterate over volumes + foreach (var sourcesPerVolume in SortByVolume(sources)) + { + var volume = sourcesPerVolume.Key; + + try + { + // prepare journal data entry to store with current fileset + var journal = new USNJournal(volume, m_cancelHandler); + var nextData = new USNJournalDataEntry + { + Volume = volume, + JournalId = journal.JournalId, + NextUsn = journal.NextUsn, + ConfigHash = filterHash + }; + + result.JournalData.Add(nextData); + + // only use change journal if: + // - journal ID hasn't changed + // - nextUsn isn't zero (we use this as magic value to force a rescan) + // - the exclude filter hash hasn't changed + if (!journalDataDict.TryGetValue(volume, out var data) + || data.JournalId != nextData.JournalId + || data.NextUsn == 0 + || data.ConfigHash != nextData.ConfigHash) + { + ScheduleFullScan(sourcesPerVolume, volume, result); + } + else + { + var changedFiles = new HashSet(Utility.Utility.ClientFilenameStringComparer); + var changedFolders = new HashSet(Utility.Utility.ClientFilenameStringComparer); + + // iterate over sources and retrieve *renamed* directories + foreach (var source in sourcesPerVolume.Value) + { + foreach (var entry in journal.GetChangedFileSystemEntries(source, data.NextUsn)) + { + if (entry.Item2.HasFlag(USNJournal.EntryType.File)) + { + changedFiles.Add(entry.Item1); + } + else + { + changedFolders.Add(Utility.Utility.AppendDirSeparator(entry.Item1)); + } + } + } + + var reducedFolderList = Utility.Utility.SimplifyFolderList(changedFolders).ToList(); + var reducedFileList = Utility.Utility.GetFilesNotInFolders(changedFiles, reducedFolderList); + result.Folders.AddRange(reducedFolderList); + result.Files.UnionWith(reducedFileList); + } + } + catch (UsnJournalSoftFailureException) + { + // journal is fine, but we cannot recover gapless changes since last time + // => schedule full scan + ScheduleFullScan(sourcesPerVolume, volume, result); + } + catch (Exception e) + { + m_errorCallback(volume, volume, e); + ScheduleFullScan(sourcesPerVolume, volume, result); + } + } + + return result; + } + + /// + /// Add ALL sources for volume to result set + /// + /// Sources + /// Volume + /// Result set + private static void ScheduleFullScan(KeyValuePair> sourcesPerVolume, string volume, FilterData result) + { + foreach (var src in sourcesPerVolume.Value) + { + if (src.EndsWith(Utility.Utility.DirectorySeparatorString, StringComparison.Ordinal)) + { + result.Folders.Add(src); + } + else + { + result.Files.Add(src); + } + } + + Log.WriteInformationMessage(LOGTAG, "SkipUsnForVolume", $"Performing full scan for volume \"{volume}\""); + } + + /// + /// Sort sources by root volume + /// + /// List of sources + /// Dictionary of volumes, with list of sources as values + private static Dictionary> SortByVolume(IEnumerable sources) + { + var sourcesByVolume = new Dictionary>(); + foreach (var path in sources) + { + // get NTFS volume root + var root = USNJournal.GetVolumeRootFromPath(path); + + if (!sourcesByVolume.TryGetValue(root, out var list)) + { + list = new List(); + sourcesByVolume.Add(root, list); + } + + list.Add(path); + } + + return sourcesByVolume; + } + + #endregion + + } +} diff --git a/Duplicati/Library/Snapshots/Win32USN.cs b/Duplicati/Library/Snapshots/Win32USN.cs index 19479e0fb..5b59ca13d 100644 --- a/Duplicati/Library/Snapshots/Win32USN.cs +++ b/Duplicati/Library/Snapshots/Win32USN.cs @@ -18,8 +18,6 @@ // #endregion using System; -using System.Collections.Generic; -using System.Text; using System.Runtime.InteropServices; using Microsoft.Win32.SafeHandles; @@ -33,24 +31,64 @@ namespace Duplicati.Library.Snapshots internal static class Win32USN { public const int ERROR_HANDLE_EOF = 38; + public const int ERROR_INSUFFICIENT_BUFFER = 122; public const int ERROR_SUCCESS = 0; + public const int ERROR_JOURNAL_ENTRY_DELETED = 1181; #region Structures [StructLayout(LayoutKind.Sequential)] - public struct USN_JOURNAL_DATA + public struct USN_JOURNAL_DATA_V0 { - public long UsnJournalID; //DWORDLONG UsnJournalID - public long FirstUsn; //USN FirstUsn - public long NextUsn; //USN NextUsn - public long LowestValidUsn; //USN LowestValidUsn - public long MaxUsn; //USN MaxUsn - public long MaximumSize; //DWORDLONG MaximumSize + /// + /// The current journal identifier. A journal is assigned a new identifier on creation and can be + /// stamped with a new identifier in the course of its existence. + /// The NTFS file system uses this identifier for an integrity check. + /// + public long UsnJournalID; + + /// + /// The number of first record that can be read from the journal. + /// + public long FirstUsn; + + /// + /// The number of next record to be written to the journal. + /// + public long NextUsn; + + /// + /// The first record that was written into the journal for this journal instance. + /// Enumerating the files or directories on a volume can return a USN lower than this value + /// (in other words, a FirstUsn member value less than the LowestValidUsn member value). + /// If it does, the journal has been stamped with a new identifier since the last USN was written. + /// In this case, LowestValidUsn may indicate a discontinuity in the journal, in which changes + /// to some or all files or directories on the volume may have occurred that are not recorded in the change journal. + /// + public long LowestValidUsn; + + /// + /// The largest USN that the change journal supports. An administrator must delete + /// the change journal as the value of NextUsn approaches this value. + /// + public long MaxUsn; + + /// + /// The target maximum size for the change journal, in bytes. The change journal can grow larger + /// than this value, but it is then truncated at the next NTFS file system checkpoint to less than this value. + /// + public long MaximumSize; + + /// + /// The number of bytes of disk memory added to the end and removed from the beginning of the change + /// journal each time memory is allocated or deallocated. In other words, allocation and deallocation + /// take place in units of this size. An integer multiple of a cluster size is a reasonable value for this member. + /// public long AllocationDelta; //DWORDLONG AllocationDelta } [StructLayout(LayoutKind.Sequential)] - public struct READ_USN_JOURNAL_DATA + public struct READ_USN_JOURNAL_DATA_V0 { public long StartUsn; public USNReason ReasonMask; @@ -61,7 +99,7 @@ namespace Duplicati.Library.Snapshots } [StructLayout(LayoutKind.Sequential)] - public struct USN_RECORD + public struct USN_RECORD_V2 { public uint RecordLength; public ushort MajorVersion; @@ -73,7 +111,7 @@ namespace Duplicati.Library.Snapshots public USNReason Reason; public uint SourceInfo; public uint SecurityId; - public EFileAttributes FileAttributes; + public FileAttributes FileAttributes; public ushort FileNameLength; public ushort FileNameOffset; // immediately after the FileNameOffset comes an array of WCHARs containing the FileName @@ -114,217 +152,29 @@ namespace Duplicati.Library.Snapshots } [Flags] - public enum EFileDevice : uint + public enum FsCtl : uint { - Beep = 0x00000001, - CDRom = 0x00000002, - CDRomFileSytem = 0x00000003, - Controller = 0x00000004, - Datalink = 0x00000005, - Dfs = 0x00000006, - Disk = 0x00000007, - DiskFileSystem = 0x00000008, - FileSystem = 0x00000009, - InPortPort = 0x0000000a, - Keyboard = 0x0000000b, - Mailslot = 0x0000000c, - MidiIn = 0x0000000d, - MidiOut = 0x0000000e, - Mouse = 0x0000000f, - MultiUncProvider = 0x00000010, - NamedPipe = 0x00000011, - Network = 0x00000012, - NetworkBrowser = 0x00000013, - NetworkFileSystem = 0x00000014, - Null = 0x00000015, - ParellelPort = 0x00000016, - PhysicalNetcard = 0x00000017, - Printer = 0x00000018, - Scanner = 0x00000019, - SerialMousePort = 0x0000001a, - SerialPort = 0x0000001b, - Screen = 0x0000001c, - Sound = 0x0000001d, - Streams = 0x0000001e, - Tape = 0x0000001f, - TapeFileSystem = 0x00000020, - Transport = 0x00000021, - Unknown = 0x00000022, - Video = 0x00000023, - VirtualDisk = 0x00000024, - WaveIn = 0x00000025, - WaveOut = 0x00000026, - Port8042 = 0x00000027, - NetworkRedirector = 0x00000028, - Battery = 0x00000029, - BusExtender = 0x0000002a, - Modem = 0x0000002b, - Vdm = 0x0000002c, - MassStorage = 0x0000002d, - Smb = 0x0000002e, - Ks = 0x0000002f, - Changer = 0x00000030, - Smartcard = 0x00000031, - Acpi = 0x00000032, - Dvd = 0x00000033, - FullscreenVideo = 0x00000034, - DfsFileSystem = 0x00000035, - DfsVolume = 0x00000036, - Serenum = 0x00000037, - Termsrv = 0x00000038, - Ksec = 0x00000039 + /// + /// Causes a journal to be queried when used with DeviceIoControl + /// + /// FSCTL_QUERY_USN_JOURNAL + QueryUSNJournal = 0x000900f4, + + /// + /// Causes a journal to be read when used with DeviceIoControl + /// + /// FSCTL_READ_USN_JOURNAL + ReadUSNJournal = 0x000900bb, + + /// + /// Enumerates the update sequence number (USN) data between two specified boundaries to obtain master file table (MFT) records. + /// + /// FSCTL_ENUM_USN_DATA + EnumUSNData = 0x000900b3 } [Flags] - public enum EIOControlCode : uint - { - // STORAGE - StorageBase = EFileDevice.MassStorage, - StorageCheckVerify = (StorageBase << 16) | (0x0200 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - StorageCheckVerify2 = (StorageBase << 16) | (0x0200 << 2) | EMethod.Buffered | (0 << 14), // FileAccess.Any - StorageMediaRemoval = (StorageBase << 16) | (0x0201 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - StorageEjectMedia = (StorageBase << 16) | (0x0202 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - StorageLoadMedia = (StorageBase << 16) | (0x0203 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - StorageLoadMedia2 = (StorageBase << 16) | (0x0203 << 2) | EMethod.Buffered | (0 << 14), - StorageReserve = (StorageBase << 16) | (0x0204 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - StorageRelease = (StorageBase << 16) | (0x0205 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - StorageFindNewDevices = (StorageBase << 16) | (0x0206 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - StorageEjectionControl = (StorageBase << 16) | (0x0250 << 2) | EMethod.Buffered | (0 << 14), - StorageMcnControl = (StorageBase << 16) | (0x0251 << 2) | EMethod.Buffered | (0 << 14), - StorageGetMediaTypes = (StorageBase << 16) | (0x0300 << 2) | EMethod.Buffered | (0 << 14), - StorageGetMediaTypesEx = (StorageBase << 16) | (0x0301 << 2) | EMethod.Buffered | (0 << 14), - StorageResetBus = (StorageBase << 16) | (0x0400 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - StorageResetDevice = (StorageBase << 16) | (0x0401 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - StorageGetDeviceNumber = (StorageBase << 16) | (0x0420 << 2) | EMethod.Buffered | (0 << 14), - StoragePredictFailure = (StorageBase << 16) | (0x0440 << 2) | EMethod.Buffered | (0 << 14), - StorageObsoleteResetBus = (StorageBase << 16) | (0x0400 << 2) | EMethod.Buffered | ((EFileAccess.GenericRead | EFileAccess.GenericWrite) << 14), - StorageObsoleteResetDevice = (StorageBase << 16) | (0x0401 << 2) | EMethod.Buffered | ((EFileAccess.GenericRead | EFileAccess.GenericWrite) << 14), - // DISK - DiskBase = EFileDevice.Disk, - DiskGetDriveGeometry = (DiskBase << 16) | (0x0000 << 2) | EMethod.Buffered | (0 << 14), - DiskGetPartitionInfo = (DiskBase << 16) | (0x0001 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - DiskSetPartitionInfo = (DiskBase << 16) | (0x0002 << 2) | EMethod.Buffered | ((EFileAccess.GenericRead | EFileAccess.GenericWrite) << 14), - DiskGetDriveLayout = (DiskBase << 16) | (0x0003 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - DiskSetDriveLayout = (DiskBase << 16) | (0x0004 << 2) | EMethod.Buffered | ((EFileAccess.GenericRead | EFileAccess.GenericWrite) << 14), - DiskVerify = (DiskBase << 16) | (0x0005 << 2) | EMethod.Buffered | (0 << 14), - DiskFormatTracks = (DiskBase << 16) | (0x0006 << 2) | EMethod.Buffered | ((EFileAccess.GenericRead | EFileAccess.GenericWrite) << 14), - DiskReassignBlocks = (DiskBase << 16) | (0x0007 << 2) | EMethod.Buffered | ((EFileAccess.GenericRead | EFileAccess.GenericWrite) << 14), - DiskPerformance = (DiskBase << 16) | (0x0008 << 2) | EMethod.Buffered | (0 << 14), - DiskIsWritable = (DiskBase << 16) | (0x0009 << 2) | EMethod.Buffered | (0 << 14), - DiskLogging = (DiskBase << 16) | (0x000a << 2) | EMethod.Buffered | (0 << 14), - DiskFormatTracksEx = (DiskBase << 16) | (0x000b << 2) | EMethod.Buffered | ((EFileAccess.GenericRead | EFileAccess.GenericWrite) << 14), - DiskHistogramStructure = (DiskBase << 16) | (0x000c << 2) | EMethod.Buffered | (0 << 14), - DiskHistogramData = (DiskBase << 16) | (0x000d << 2) | EMethod.Buffered | (0 << 14), - DiskHistogramReset = (DiskBase << 16) | (0x000e << 2) | EMethod.Buffered | (0 << 14), - DiskRequestStructure = (DiskBase << 16) | (0x000f << 2) | EMethod.Buffered | (0 << 14), - DiskRequestData = (DiskBase << 16) | (0x0010 << 2) | EMethod.Buffered | (0 << 14), - DiskControllerNumber = (DiskBase << 16) | (0x0011 << 2) | EMethod.Buffered | (0 << 14), - DiskSmartGetVersion = (DiskBase << 16) | (0x0020 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - DiskSmartSendDriveCommand = (DiskBase << 16) | (0x0021 << 2) | EMethod.Buffered | ((EFileAccess.GenericRead | EFileAccess.GenericWrite) << 14), - DiskSmartRcvDriveData = (DiskBase << 16) | (0x0022 << 2) | EMethod.Buffered | ((EFileAccess.GenericRead | EFileAccess.GenericWrite) << 14), - DiskUpdateDriveSize = (DiskBase << 16) | (0x0032 << 2) | EMethod.Buffered | ((EFileAccess.GenericRead | EFileAccess.GenericWrite) << 14), - DiskGrowPartition = (DiskBase << 16) | (0x0034 << 2) | EMethod.Buffered | ((EFileAccess.GenericRead | EFileAccess.GenericWrite) << 14), - DiskGetCacheInformation = (DiskBase << 16) | (0x0035 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - DiskSetCacheInformation = (DiskBase << 16) | (0x0036 << 2) | EMethod.Buffered | ((EFileAccess.GenericRead | EFileAccess.GenericWrite) << 14), - DiskDeleteDriveLayout = (DiskBase << 16) | (0x0040 << 2) | EMethod.Buffered | ((EFileAccess.GenericRead | EFileAccess.GenericWrite) << 14), - DiskFormatDrive = (DiskBase << 16) | (0x00f3 << 2) | EMethod.Buffered | ((EFileAccess.GenericRead | EFileAccess.GenericWrite) << 14), - DiskSenseDevice = (DiskBase << 16) | (0x00f8 << 2) | EMethod.Buffered | (0 << 14), - DiskCheckVerify = (DiskBase << 16) | (0x0200 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - DiskMediaRemoval = (DiskBase << 16) | (0x0201 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - DiskEjectMedia = (DiskBase << 16) | (0x0202 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - DiskLoadMedia = (DiskBase << 16) | (0x0203 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - DiskReserve = (DiskBase << 16) | (0x0204 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - DiskRelease = (DiskBase << 16) | (0x0205 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - DiskFindNewDevices = (DiskBase << 16) | (0x0206 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - DiskGetMediaTypes = (DiskBase << 16) | (0x0300 << 2) | EMethod.Buffered | (0 << 14), - // CHANGER - ChangerBase = EFileDevice.Changer, - ChangerGetParameters = (ChangerBase << 16) | (0x0000 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - ChangerGetStatus = (ChangerBase << 16) | (0x0001 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - ChangerGetProductData = (ChangerBase << 16) | (0x0002 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - ChangerSetAccess = (ChangerBase << 16) | (0x0004 << 2) | EMethod.Buffered | ((EFileAccess.GenericRead | EFileAccess.GenericWrite) << 14), - ChangerGetElementStatus = (ChangerBase << 16) | (0x0005 << 2) | EMethod.Buffered | ((EFileAccess.GenericRead | EFileAccess.GenericWrite) << 14), - ChangerInitializeElementStatus = (ChangerBase << 16) | (0x0006 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - ChangerSetPosition = (ChangerBase << 16) | (0x0007 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - ChangerExchangeMedium = (ChangerBase << 16) | (0x0008 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - ChangerMoveMedium = (ChangerBase << 16) | (0x0009 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - ChangerReinitializeTarget = (ChangerBase << 16) | (0x000A << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - ChangerQueryVolumeTags = (ChangerBase << 16) | (0x000B << 2) | EMethod.Buffered | ((EFileAccess.GenericRead | EFileAccess.GenericWrite) << 14), - // FILESYSTEM - FsctlRequestOplockLevel1 = (EFileDevice.FileSystem << 16) | (0 << 2) | EMethod.Buffered | (0 << 14), - FsctlRequestOplockLevel2 = (EFileDevice.FileSystem << 16) | (1 << 2) | EMethod.Buffered | (0 << 14), - FsctlRequestBatchOplock = (EFileDevice.FileSystem << 16) | (2 << 2) | EMethod.Buffered | (0 << 14), - FsctlOplockBreakAcknowledge = (EFileDevice.FileSystem << 16) | (3 << 2) | EMethod.Buffered | (0 << 14), - FsctlOpBatchAckClosePending = (EFileDevice.FileSystem << 16) | (4 << 2) | EMethod.Buffered | (0 << 14), - FsctlOplockBreakNotify = (EFileDevice.FileSystem << 16) | (5 << 2) | EMethod.Buffered | (0 << 14), - FsctlLockVolume = (EFileDevice.FileSystem << 16) | (6 << 2) | EMethod.Buffered | (0 << 14), - FsctlUnlockVolume = (EFileDevice.FileSystem << 16) | (7 << 2) | EMethod.Buffered | (0 << 14), - FsctlDismountVolume = (EFileDevice.FileSystem << 16) | (8 << 2) | EMethod.Buffered | (0 << 14), - FsctlIsVolumeMounted = (EFileDevice.FileSystem << 16) | (10 << 2) | EMethod.Buffered | (0 << 14), - FsctlIsPathnameValid = (EFileDevice.FileSystem << 16) | (11 << 2) | EMethod.Buffered | (0 << 14), - FsctlMarkVolumeDirty = (EFileDevice.FileSystem << 16) | (12 << 2) | EMethod.Buffered | (0 << 14), - FsctlQueryRetrievalPointers = (EFileDevice.FileSystem << 16) | (14 << 2) | EMethod.Neither | (0 << 14), - FsctlGetCompression = (EFileDevice.FileSystem << 16) | (15 << 2) | EMethod.Buffered | (0 << 14), - FsctlSetCompression = (EFileDevice.FileSystem << 16) | (16 << 2) | EMethod.Buffered | ((EFileAccess.GenericRead | EFileAccess.GenericWrite) << 14), - FsctlMarkAsSystemHive = (EFileDevice.FileSystem << 16) | (19 << 2) | EMethod.Neither | (0 << 14), - FsctlOplockBreakAckNo2 = (EFileDevice.FileSystem << 16) | (20 << 2) | EMethod.Buffered | (0 << 14), - FsctlInvalidateVolumes = (EFileDevice.FileSystem << 16) | (21 << 2) | EMethod.Buffered | (0 << 14), - FsctlQueryFatBpb = (EFileDevice.FileSystem << 16) | (22 << 2) | EMethod.Buffered | (0 << 14), - FsctlRequestFilterOplock = (EFileDevice.FileSystem << 16) | (23 << 2) | EMethod.Buffered | (0 << 14), - FsctlFileSystemGetStatistics = (EFileDevice.FileSystem << 16) | (24 << 2) | EMethod.Buffered | (0 << 14), - FsctlGetNtfsVolumeData = (EFileDevice.FileSystem << 16) | (25 << 2) | EMethod.Buffered | (0 << 14), - FsctlGetNtfsFileRecord = (EFileDevice.FileSystem << 16) | (26 << 2) | EMethod.Buffered | (0 << 14), - FsctlGetVolumeBitmap = (EFileDevice.FileSystem << 16) | (27 << 2) | EMethod.Neither | (0 << 14), - FsctlGetRetrievalPointers = (EFileDevice.FileSystem << 16) | (28 << 2) | EMethod.Neither | (0 << 14), - FsctlMoveFile = (EFileDevice.FileSystem << 16) | (29 << 2) | EMethod.Buffered | (0 << 14), - FsctlIsVolumeDirty = (EFileDevice.FileSystem << 16) | (30 << 2) | EMethod.Buffered | (0 << 14), - FsctlGetHfsInformation = (EFileDevice.FileSystem << 16) | (31 << 2) | EMethod.Buffered | (0 << 14), - FsctlAllowExtendedDasdIo = (EFileDevice.FileSystem << 16) | (32 << 2) | EMethod.Neither | (0 << 14), - FsctlReadPropertyData = (EFileDevice.FileSystem << 16) | (33 << 2) | EMethod.Neither | (0 << 14), - FsctlWritePropertyData = (EFileDevice.FileSystem << 16) | (34 << 2) | EMethod.Neither | (0 << 14), - FsctlFindFilesBySid = (EFileDevice.FileSystem << 16) | (35 << 2) | EMethod.Neither | (0 << 14), - FsctlDumpPropertyData = (EFileDevice.FileSystem << 16) | (37 << 2) | EMethod.Neither | (0 << 14), - FsctlSetObjectId = (EFileDevice.FileSystem << 16) | (38 << 2) | EMethod.Buffered | (0 << 14), - FsctlGetObjectId = (EFileDevice.FileSystem << 16) | (39 << 2) | EMethod.Buffered | (0 << 14), - FsctlDeleteObjectId = (EFileDevice.FileSystem << 16) | (40 << 2) | EMethod.Buffered | (0 << 14), - FsctlSetReparsePoint = (EFileDevice.FileSystem << 16) | (41 << 2) | EMethod.Buffered | (0 << 14), - FsctlGetReparsePoint = (EFileDevice.FileSystem << 16) | (42 << 2) | EMethod.Buffered | (0 << 14), - FsctlDeleteReparsePoint = (EFileDevice.FileSystem << 16) | (43 << 2) | EMethod.Buffered | (0 << 14), - FsctlEnumUsnData = (EFileDevice.FileSystem << 16) | (44 << 2) | EMethod.Neither | (0 << 14), - FsctlSecurityIdCheck = (EFileDevice.FileSystem << 16) | (45 << 2) | EMethod.Neither | (EFileAccess.GenericRead << 14), - FsctlReadUsnJournal = (EFileDevice.FileSystem << 16) | (46 << 2) | EMethod.Neither | (0 << 14), - FsctlSetObjectIdExtended = (EFileDevice.FileSystem << 16) | (47 << 2) | EMethod.Buffered | (0 << 14), - FsctlCreateOrGetObjectId = (EFileDevice.FileSystem << 16) | (48 << 2) | EMethod.Buffered | (0 << 14), - FsctlSetSparse = (EFileDevice.FileSystem << 16) | (49 << 2) | EMethod.Buffered | (0 << 14), - FsctlSetZeroData = (EFileDevice.FileSystem << 16) | (50 << 2) | EMethod.Buffered | (EFileAccess.GenericWrite << 14), - FsctlQueryAllocatedRanges = (EFileDevice.FileSystem << 16) | (51 << 2) | EMethod.Neither | (EFileAccess.GenericRead << 14), - FsctlEnableUpgrade = (EFileDevice.FileSystem << 16) | (52 << 2) | EMethod.Buffered | (EFileAccess.GenericWrite << 14), - FsctlSetEncryption = (EFileDevice.FileSystem << 16) | (53 << 2) | EMethod.Neither | (0 << 14), - FsctlEncryptionFsctlIo = (EFileDevice.FileSystem << 16) | (54 << 2) | EMethod.Neither | (0 << 14), - FsctlWriteRawEncrypted = (EFileDevice.FileSystem << 16) | (55 << 2) | EMethod.Neither | (0 << 14), - FsctlReadRawEncrypted = (EFileDevice.FileSystem << 16) | (56 << 2) | EMethod.Neither | (0 << 14), - FsctlCreateUsnJournal = (EFileDevice.FileSystem << 16) | (57 << 2) | EMethod.Neither | (0 << 14), - FsctlReadFileUsnData = (EFileDevice.FileSystem << 16) | (58 << 2) | EMethod.Neither | (0 << 14), - FsctlWriteUsnCloseRecord = (EFileDevice.FileSystem << 16) | (59 << 2) | EMethod.Neither | (0 << 14), - FsctlExtendVolume = (EFileDevice.FileSystem << 16) | (60 << 2) | EMethod.Buffered | (0 << 14), - FsctlQueryUsnJournal = (EFileDevice.FileSystem << 16) | (61 << 2) | EMethod.Buffered | (0 << 14), - FsctlDeleteUsnJournal = (EFileDevice.FileSystem << 16) | (62 << 2) | EMethod.Buffered | (0 << 14), - FsctlMarkHandle = (EFileDevice.FileSystem << 16) | (63 << 2) | EMethod.Buffered | (0 << 14), - FsctlSisCopyFile = (EFileDevice.FileSystem << 16) | (64 << 2) | EMethod.Buffered | (0 << 14), - FsctlSisLinkFiles = (EFileDevice.FileSystem << 16) | (65 << 2) | EMethod.Buffered | ((EFileAccess.GenericRead | EFileAccess.GenericWrite) << 14), - FsctlHsmMsg = (EFileDevice.FileSystem << 16) | (66 << 2) | EMethod.Buffered | ((EFileAccess.GenericRead | EFileAccess.GenericWrite) << 14), - FsctlNssControl = (EFileDevice.FileSystem << 16) | (67 << 2) | EMethod.Buffered | (EFileAccess.GenericWrite << 14), - FsctlHsmData = (EFileDevice.FileSystem << 16) | (68 << 2) | EMethod.Neither | ((EFileAccess.GenericRead | EFileAccess.GenericWrite) << 14), - FsctlRecallFile = (EFileDevice.FileSystem << 16) | (69 << 2) | EMethod.Neither | (0 << 14), - FsctlNssRcontrol = (EFileDevice.FileSystem << 16) | (70 << 2) | EMethod.Buffered | (EFileAccess.GenericRead << 14), - // VIDEO - VideoQuerySupportedBrightness = (EFileDevice.Video << 16) | (0x0125 << 2) | EMethod.Buffered | (0 << 14), - VideoQueryDisplayBrightness = (EFileDevice.Video << 16) | (0x0126 << 2) | EMethod.Buffered | (0 << 14), - VideoSetDisplayBrightness = (EFileDevice.Video << 16) | (0x0127 << 2) | EMethod.Buffered | (0 << 14) - } - - [Flags] - public enum EFileAccess : uint + public enum FileAccess : uint { /// /// @@ -345,7 +195,7 @@ namespace Duplicati.Library.Snapshots } [Flags] - public enum EFileShare : uint + public enum FileShare : uint { /// /// @@ -379,7 +229,7 @@ namespace Duplicati.Library.Snapshots All = None | Read | Write } - public enum ECreationDisposition : uint + public enum CreationDisposition : uint { /// /// Creates a new file. The function fails if a specified file exists. @@ -408,7 +258,7 @@ namespace Duplicati.Library.Snapshots } [Flags] - public enum EFileAttributes : uint + public enum FileAttributes : uint { None = 0x0, Readonly = 0x00000001, @@ -425,7 +275,7 @@ namespace Duplicati.Library.Snapshots Offline = 0x00001000, NotContentIndexed = 0x00002000, Encrypted = 0x00004000, - Write_Through = 0x80000000, + WriteThrough = 0x80000000, Overlapped = 0x40000000, NoBuffering = 0x20000000, RandomAccess = 0x10000000, @@ -514,11 +364,11 @@ namespace Duplicati.Library.Snapshots [DllImport("kernel32.dll", CharSet = CharSet.Auto, CallingConvention = CallingConvention.StdCall, SetLastError = true)] public static extern SafeFileHandle CreateFile( string lpFileName, - EFileAccess dwDesiredAccess, - EFileShare dwShareMode, + FileAccess dwDesiredAccess, + FileShare dwShareMode, IntPtr SecurityAttributes, - ECreationDisposition dwCreationDisposition, - EFileAttributes dwFlagsAndAttributes, + CreationDisposition dwCreationDisposition, + FileAttributes dwFlagsAndAttributes, IntPtr hTemplateFile ); @@ -537,8 +387,8 @@ namespace Duplicati.Library.Snapshots [DllImport("Kernel32.dll", SetLastError = true, CharSet = CharSet.Auto)] public static extern bool DeviceIoControl( SafeFileHandle hDevice, - EIOControlCode IoControlCode, - [In] READ_USN_JOURNAL_DATA InBuffer, + FsCtl IoControlCode, + [In] READ_USN_JOURNAL_DATA_V0 InBuffer, uint nInBufferSize, [In] IntPtr OutBuffer, uint nOutBufferSize, @@ -561,7 +411,7 @@ namespace Duplicati.Library.Snapshots [DllImport("Kernel32.dll", SetLastError = true, CharSet = CharSet.Auto)] public static extern bool DeviceIoControl( SafeFileHandle hDevice, - EIOControlCode IoControlCode, + FsCtl IoControlCode, [In] ref MFT_ENUM_DATA InBuffer, uint nInBufferSize, [In] IntPtr OutBuffer, @@ -574,10 +424,10 @@ namespace Duplicati.Library.Snapshots /// Sends the dwIoControlCode to the device specified by hDevice. /// /// Safe handle to the device - /// Device IO Control Code to send - /// Input buffer if required + /// Device IO Control Code to send + /// Input buffer if required /// Size of input buffer - /// Output buffer if required + /// Output buffer if required /// Size of output buffer /// Number of bytes returned in output buffer /// IntPtr to an 'OVERLAPPED' structure @@ -585,14 +435,13 @@ namespace Duplicati.Library.Snapshots [DllImport("Kernel32.dll", SetLastError = true, CharSet = CharSet.Auto)] public static extern bool DeviceIoControl( SafeFileHandle hDevice, - EIOControlCode IoControlCode, - [MarshalAs(UnmanagedType.AsAny)] - [In] object InBuffer, + uint dwIoControlCode, + IntPtr lpInBuffer, uint nInBufferSize, - [Out] out USN_JOURNAL_DATA OutBuffer, + IntPtr lpOutBuffer, uint nOutBufferSize, - ref uint pBytesReturned, - [In] IntPtr overlapped //[In] ref System.Threading.NativeOverlapped Overlapped + out uint pBytesReturned, + IntPtr overlapped ); [DllImport("kernel32.dll", SetLastError = true)] @@ -602,5 +451,87 @@ namespace Duplicati.Library.Snapshots ); #endregion + + /// + /// Sends the control code to the device specified by handle. + /// + /// + /// + /// + /// + /// Maximum size of returned buffer + /// + public static bool ControlWithInput( + SafeFileHandle handle, FsCtl code, + ref TStructure structure, int bufferSize, out byte[] buffer) + where TStructure : struct + { + uint datalen; + bool controlResult; + + buffer = new byte[bufferSize]; + var bufferHandle = GCHandle.Alloc(buffer, GCHandleType.Pinned); + var structureHandle = GCHandle.Alloc(structure, GCHandleType.Pinned); + var bufferPointer = bufferHandle.AddrOfPinnedObject(); + var structurePointer = structureHandle.AddrOfPinnedObject(); + + try + { + controlResult = + DeviceIoControl(handle, (uint)code, + structurePointer, (uint)Marshal.SizeOf(structure), + bufferPointer, (uint)buffer.Length, + out datalen, IntPtr.Zero); + } + finally + { + structureHandle.Free(); + bufferHandle.Free(); + } + + Array.Resize(ref buffer, (int)datalen); + + return controlResult; + } + + /// + /// Sends the control code to the device specified by handle. + /// + /// + /// + /// + /// + internal static bool ControlWithOutput( + SafeFileHandle handle, FsCtl code, ref TStructure structure) + where TStructure : struct + { + bool controlResult; + + //get our object pointer + var structureHandle = GCHandle.Alloc(structure, GCHandleType.Pinned); + var structurePointer = structureHandle.AddrOfPinnedObject(); + + try + { + controlResult = + DeviceIoControl(handle, (uint)code, + IntPtr.Zero, 0, structurePointer, + (uint)Marshal.SizeOf(structure), + out _, IntPtr.Zero); + } + finally + { + // always release GH handle + structureHandle.Free(); + } + + if (controlResult) + { + structure = (TStructure)Marshal.PtrToStructure(structurePointer, typeof(TStructure)); + } + + return controlResult; + + } } } diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index 4f256367c..1695c7fc2 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -24,7 +24,6 @@ using System; using System.Collections.Generic; using System.IO; using Alphaleonis.Win32.Vss; - using AlphaFS = Alphaleonis.Win32.Filesystem; namespace Duplicati.Library.Snapshots @@ -36,62 +35,69 @@ namespace Duplicati.Library.Snapshots /// The class presents all files and folders with their regular filenames to the caller, /// and internally handles the conversion to the shadow path. /// - public class WindowsSnapshot : ISnapshotService + public sealed class WindowsSnapshot : SnapshotBase { - /// - /// The tag used for logging - /// - private static readonly string LOGTAG = Logging.Log.LogTagFromType(); /// /// The main reference to the backup controller /// private IVssBackupComponents m_backup; - /// - /// The list of paths that will be shadow copied - /// - private List m_sourcepaths; + /// /// The list of snapshot ids for each volume, key is the path root, eg C:\. /// The dictionary is case insensitive /// - private Dictionary m_volumes; + private readonly Dictionary m_volumes; + /// /// The mapping of snapshot sources to their snapshot entries , key is the path root, eg C:\. /// The dictionary is case insensitive /// - private Dictionary m_volumeMap; + private readonly Dictionary m_volumeMap; + + /// + /// The mapping of snapshot sources to their snapshot entries , key is the path root, eg C:\. + /// The dictionary is case insensitive + /// + private readonly Dictionary m_volumeReverseMap; + /// /// A list of mapped drives /// private List m_mappedDrives; - /// - /// Commonly used string element - /// - private static string SLASH = Path.DirectorySeparatorChar.ToString(); + /// /// A cached lookup for windows methods for dealing with long filenames /// - private static readonly SystemIOWindows _ioWin = new SystemIOWindows(); + private static SystemIOWindows IO_WIN = new SystemIOWindows(); + /// /// Constructs a new backup snapshot, using all the required disks /// - /// The folders that are about to be backed up + /// Sources to determine which volumes to include in snapshot /// A set of commandline options - public WindowsSnapshot(string[] sourcepaths, Dictionary options) + public WindowsSnapshot(IEnumerable sources, IDictionary options) { try { - //Substitute for calling VssUtils.LoadImplementation(), as we have the dlls outside the GAC - string alphadir = Path.Combine(Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location), "alphavss"); - string alphadll = System.IO.Path.Combine(alphadir, VssUtils.GetPlatformSpecificAssemblyShortName() + ".dll"); - IVssImplementation vss = (IVssImplementation)System.Reflection.Assembly.LoadFile(alphadll).CreateInstance("Alphaleonis.Win32.Vss.VssImplementation"); + // Substitute for calling VssUtils.LoadImplementation(), as we have the dlls outside the GAC + var assemblyLocation = Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location); + if (assemblyLocation == null) + throw new InvalidOperationException(); + + var alphadir = Path.Combine(assemblyLocation, "alphavss"); + var alphadll = Path.Combine(alphadir, VssUtils.GetPlatformSpecificAssemblyShortName() + ".dll"); + var vss = (IVssImplementation)System.Reflection.Assembly.LoadFile(alphadll).CreateInstance("Alphaleonis.Win32.Vss.VssImplementation"); + if (vss == null) + throw new InvalidOperationException(); var excludedWriters = new Guid[0]; if (options.ContainsKey("vss-exclude-writers")) + { excludedWriters = options["vss-exclude-writers"].Split(';').Where(x => !string.IsNullOrWhiteSpace(x) && x.Trim().Length > 0).Select(x => new Guid(x)).ToArray(); + } //Check if we should map any drives - bool useSubst = Utility.Utility.ParseBoolOption(options, "vss-use-mapping"); + var useSubst = Utility.Utility.ParseBoolOption(options, "vss-use-mapping"); //Prepare the backup m_backup = vss.CreateVssBackupComponents(); @@ -101,8 +107,6 @@ namespace Duplicati.Library.Snapshots if (excludedWriters.Length > 0) m_backup.DisableWriterClasses(excludedWriters.ToArray()); - - m_sourcepaths = sourcepaths.Select(x => Directory.Exists(x) ? Utility.Utility.AppendDirSeparator(x) : x).ToList(); try { @@ -113,31 +117,16 @@ namespace Duplicati.Library.Snapshots m_backup.FreeWriterMetadata(); } - //Sanity check for duplicate files/folders - ISet pathDuplicates; - m_sourcepaths = Utility.Utility.GetUniqueItems(m_sourcepaths, Utility.Utility.ClientFilenameStringComparer, out pathDuplicates).ToList(); - - foreach(var pathDuplicate in pathDuplicates) - Logging.Log.WriteInformationMessage(LOGTAG, "VSSDuplicates", "Removing duplicate source: {0}", pathDuplicate); - - //Sanity check for multiple inclusions of the same files/folders - var pathIncludedPaths = m_sourcepaths.Where(x => m_sourcepaths.Where(y => y != x).Any(z => x.StartsWith(z, Utility.Utility.ClientFilenameStringComparision))).ToList(); - - foreach (var pathIncluded in pathIncludedPaths) - Logging.Log.WriteInformationMessage(LOGTAG, "VSSMultiInclude", "Removing already included source: {0}", pathIncluded); - - if (pathIncludedPaths.Count > 0) - m_sourcepaths = m_sourcepaths.Except(pathIncludedPaths, Utility.Utility.ClientFilenameStringComparer).ToList(); - m_backup.StartSnapshotSet(); //Figure out which volumes are in the set m_volumes = new Dictionary(StringComparer.OrdinalIgnoreCase); - foreach (string s in m_sourcepaths) + foreach (var s in sources) { - string drive = AlphaFS.Path.GetPathRoot(s); + var drive = AlphaFS.Path.GetPathRoot(s); if (!m_volumes.ContainsKey(drive)) { + //TODO: that seems a bit harsh... we could fall-back to not using VSS for that volume only if (!m_backup.IsVolumeSupported(drive)) throw new VssVolumeNotSupportedException(drive); @@ -153,14 +142,18 @@ namespace Duplicati.Library.Snapshots //Make a little lookup table for faster translation m_volumeMap = new Dictionary(StringComparer.OrdinalIgnoreCase); - foreach (KeyValuePair kvp in m_volumes) + foreach (var kvp in m_volumes) + { m_volumeMap.Add(kvp.Key, m_backup.GetSnapshotProperties(kvp.Value).SnapshotDeviceObject); + } + + m_volumeReverseMap = m_volumeMap.ToDictionary(x => x.Value, x => x.Key); //If we should map the drives, we do that now and update the volumeMap if (useSubst) { m_mappedDrives = new List(); - foreach (string k in new List(m_volumeMap.Keys)) + foreach (var k in new List(m_volumeMap.Keys)) { try { @@ -168,67 +161,70 @@ namespace Duplicati.Library.Snapshots m_mappedDrives.Add(d = new DefineDosDevice(m_volumeMap[k])); m_volumeMap[k] = Utility.Utility.AppendDirSeparator(d.Drive); } - catch { } + catch + { + // ignored + } } } } catch { //In case we fail in the constructor, we do not want a snapshot to be active - try { Dispose(); } - catch { } + try + { + Dispose(); + } + catch + { + // ignored + } throw; } } -#if DEBUG - /// - /// Returns all files found in the shadow copies - /// - /// A list of filenames to files found in the shadow volumes - public List AllFiles() - { - return EnumerateFilesAndFolders(null, null).ToList(); - } - -#endif - #region Private functions /// /// A callback function that takes a non-shadow path to a folder, /// and returns all folders found in a non-shadow path format. /// - /// The non-shadow path of the folder to list + /// The non-shadow path of the folder to list /// A list of non-shadow paths - private string[] ListFolders(string folder) + protected override string[] ListFolders(string localFolderPath) { - string root = Utility.Utility.AppendDirSeparator(AlphaFS.Path.GetPathRoot(folder)); - string volumePath = Utility.Utility.AppendDirSeparator(GetSnapshotPath(root)); + var root = Utility.Utility.AppendDirSeparator(AlphaFS.Path.GetPathRoot(localFolderPath)); + var volumePath = Utility.Utility.AppendDirSeparator(ConvertToSnapshotPath(root)); string[] tmp = null; - string spath = GetSnapshotPath(folder); + var spath = ConvertToSnapshotPath(localFolderPath); if (SystemIOWindows.IsPathTooLong(spath)) + { try { tmp = AlphaFS.Directory.GetDirectories(spath); } catch (PathTooLongException) { } catch (DirectoryNotFoundException) { } + } else - try { tmp = System.IO.Directory.GetDirectories(spath); } + { + try { tmp = Directory.GetDirectories(spath); } catch (PathTooLongException) { } + } if (tmp == null) { spath = SystemIOWindows.PrefixWithUNC(spath); - volumePath = SystemIOWindows.PrefixWithUNC(volumePath); tmp = AlphaFS.Directory.GetDirectories(spath); } volumePath = SystemIOWindows.PrefixWithUNC(volumePath); - for (int i = 0; i < tmp.Length; i++) + for (var i = 0; i < tmp.Length; i++) + { tmp[i] = root + SystemIOWindows.PrefixWithUNC(tmp[i]).Substring(volumePath.Length); + } + return tmp; } @@ -237,92 +233,63 @@ namespace Duplicati.Library.Snapshots /// A callback function that takes a non-shadow path to a folder, /// and returns all files found in a non-shadow path format. /// - /// The non-shadow path of the folder to list + /// The non-shadow path of the folder to list /// A list of non-shadow paths - private string[] ListFiles(string folder) + protected override string[] ListFiles(string localFolderPath) { - string root = Utility.Utility.AppendDirSeparator(AlphaFS.Path.GetPathRoot(folder)); - string volumePath = Utility.Utility.AppendDirSeparator(GetSnapshotPath(root)); + var root = Utility.Utility.AppendDirSeparator(AlphaFS.Path.GetPathRoot(localFolderPath)); + var volumePath = Utility.Utility.AppendDirSeparator(ConvertToSnapshotPath(root)); string[] tmp = null; - string spath = GetSnapshotPath(folder); + var spath = ConvertToSnapshotPath(localFolderPath); if (SystemIOWindows.IsPathTooLong(spath)) + { try { tmp = AlphaFS.Directory.GetFiles(spath); } catch (PathTooLongException) { } catch (DirectoryNotFoundException) { } + } else - try { tmp = System.IO.Directory.GetFiles(spath); } + { + try { tmp = Directory.GetFiles(spath); } catch (PathTooLongException) { } + } if (tmp == null) { spath = SystemIOWindows.PrefixWithUNC(spath); - volumePath = SystemIOWindows.PrefixWithUNC(volumePath); tmp = AlphaFS.Directory.GetFiles(spath); } volumePath = SystemIOWindows.PrefixWithUNC(volumePath); - for (int i = 0; i < tmp.Length; i++) + for (var i = 0; i < tmp.Length; i++) + { tmp[i] = root + SystemIOWindows.PrefixWithUNC(tmp[i]).Substring(volumePath.Length); + } + return tmp; } - - /// - /// Helper function to translate a local path into the corresponding shadow volume path - /// - /// The local path to convert - /// The corresponding shadow volume path - private string GetSnapshotPath(string localPath) - { - if (!Path.IsPathRooted(localPath)) - throw new InvalidOperationException(); - - string root = AlphaFS.Path.GetPathRoot(localPath); - - string volumePath; - if (!m_volumeMap.TryGetValue(root, out volumePath)) - throw new InvalidOperationException(); - - localPath = localPath.Replace(root, String.Empty); - - if (!volumePath.EndsWith(SLASH, StringComparison.Ordinal) && !localPath.StartsWith(SLASH, StringComparison.Ordinal)) - localPath = localPath.Insert(0, SLASH); - localPath = localPath.Insert(0, volumePath); - - return localPath; - } #endregion #region ISnapshotService Members - /// - /// Enumerates all files and folders in the shadow copy - /// - /// The callback to invoke with each found path - /// The callback used to report errors - public IEnumerable EnumerateFilesAndFolders(Duplicati.Library.Utility.Utility.EnumerationFilterDelegate callback, Duplicati.Library.Utility.Utility.ReportAccessError errorCallback) - { - return m_sourcepaths.SelectMany( - s => Utility.Utility.EnumerateFileSystemEntries(s, callback, this.ListFolders, this.ListFiles, this.GetAttributes, errorCallback) - ); - } - /// /// Gets the last write time of a given file in UTC /// - /// The full path to the file in non-shadow format + /// The full path to the file in non-shadow format /// The last write time of the file - public DateTime GetLastWriteTimeUtc(string file) + public override DateTime GetLastWriteTimeUtc(string localPath) { - string spath = GetSnapshotPath(file); + var spath = ConvertToSnapshotPath(localPath); if (!SystemIOWindows.IsPathTooLong(spath)) + { try { return File.GetLastWriteTimeUtc(spath); } catch (PathTooLongException) { } + } return AlphaFS.File.GetLastWriteTimeUtc(SystemIOWindows.PrefixWithUNC(spath)); } @@ -330,17 +297,19 @@ namespace Duplicati.Library.Snapshots /// /// Gets the creation of a given file in UTC /// - /// The full path to the file in non-shadow format + /// The full path to the file in non-shadow format /// The last write time of the file - public DateTime GetCreationTimeUtc(string file) + public override DateTime GetCreationTimeUtc(string localPath) { - string spath = GetSnapshotPath(file); + var spath = ConvertToSnapshotPath(localPath); if (!SystemIOWindows.IsPathTooLong(spath)) + { try { return File.GetCreationTimeUtc(spath); } catch (PathTooLongException) { } + } return AlphaFS.File.GetCreationTimeUtc(SystemIOWindows.PrefixWithUNC(spath)); } @@ -348,62 +317,62 @@ namespace Duplicati.Library.Snapshots /// /// Opens a file for reading /// - /// The full path to the file in non-shadow format + /// The full path to the file in non-shadow format /// An open filestream that can be read - public System.IO.Stream OpenRead(string file) + public override Stream OpenRead(string localPath) { - return _ioWin.FileOpenRead(GetSnapshotPath(file)); + return IO_WIN.FileOpenRead(ConvertToSnapshotPath(localPath)); } /// /// Returns the size of a file /// - /// The full path to the file in non-snapshot format + /// The full path to the file in non-snapshot format /// The lenth of the file - public long GetFileSize(string file) + public override long GetFileSize(string localPath) { - return _ioWin.FileLength(GetSnapshotPath(file)); + return IO_WIN.FileLength(ConvertToSnapshotPath(localPath)); } /// /// Gets the attributes for the given file or folder /// /// The file attributes - /// The file or folder to examine - public System.IO.FileAttributes GetAttributes(string file) + /// The file or folder to examine + public override FileAttributes GetAttributes(string localPath) { - return _ioWin.GetFileAttributes(GetSnapshotPath(file)); + return IO_WIN.GetFileAttributes(ConvertToSnapshotPath(localPath)); } /// /// Returns the symlink target if the entry is a symlink, and null otherwise /// - /// The file or folder to examine + /// The file or folder to examine /// The symlink target - public string GetSymlinkTarget(string file) + public override string GetSymlinkTarget(string localPath) { - string spath = GetSnapshotPath(file); - return _ioWin.GetSymlinkTarget(spath); + var spath = ConvertToSnapshotPath(localPath); + return IO_WIN.GetSymlinkTarget(spath); } /// /// Gets the metadata for the given file or folder /// /// The metadata for the given file or folder - /// The file or folder to examine + /// The file or folder to examine /// A flag indicating if the target is a symlink /// A flag indicating if a symlink should be followed - public Dictionary GetMetadata(string file, bool isSymlink, bool followSymlink) + public override Dictionary GetMetadata(string localPath, bool isSymlink, bool followSymlink) { - return _ioWin.GetMetadata(GetSnapshotPath(file), isSymlink, followSymlink); + return IO_WIN.GetMetadata(ConvertToSnapshotPath(localPath), isSymlink, followSymlink); } /// /// Gets a value indicating if the path points to a block device /// /// true if this instance is a block device; otherwise, false. - /// The file or folder to examine - public bool IsBlockDevice(string file) + /// The file or folder to examine + public override bool IsBlockDevice(string localPath) { return false; } @@ -412,46 +381,96 @@ namespace Duplicati.Library.Snapshots /// Gets a unique hardlink target ID /// /// The hardlink ID - /// The file or folder to examine - public string HardlinkTargetID(string path) + /// The file or folder to examine + public override string HardlinkTargetID(string localPath) { return null; } + + /// + public override string ConvertToLocalPath(string snapshotPath) + { + if (!Path.IsPathRooted(snapshotPath)) + throw new InvalidOperationException(); + + foreach (var kvp in m_volumeReverseMap) + { + if (snapshotPath.StartsWith(kvp.Key)) + return Path.Combine(kvp.Value, snapshotPath.Substring(kvp.Key.Length)); + } + + throw new InvalidOperationException(); + } + + /// + public override string ConvertToSnapshotPath(string localPath) + { + if (!Path.IsPathRooted(localPath)) + throw new InvalidOperationException(); + + var root = AlphaFS.Path.GetPathRoot(localPath); + + if (!m_volumeMap.TryGetValue(root, out var volumePath)) + throw new InvalidOperationException(); + + return Path.Combine(volumePath, localPath.Substring(root.Length)); + } + #endregion #region IDisposable Members /// /// Cleans up any resources and closes the backup set /// - public void Dispose() + public override void Dispose() { try { if (m_mappedDrives != null) { - foreach (DefineDosDevice d in m_mappedDrives) + foreach (var d in m_mappedDrives) + { d.Dispose(); + } + m_mappedDrives = null; - m_volumeMap = null; } } - catch { } + catch + { + // ignored + } + + try + { + m_backup?.BackupComplete(); + } + catch + { + // ignored + } try { if (m_backup != null) - m_backup.BackupComplete(); + { + foreach (var g in m_volumes.Values) + { + try + { + m_backup.DeleteSnapshot(g, false); + } + catch + { + // ignored + } + } + } } - catch { } - - try + catch { - if (m_backup != null) - foreach (Guid g in m_volumes.Values) - try { m_backup.DeleteSnapshot(g, false); } - catch { } + // ignored } - catch { } if (m_backup != null) { diff --git a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj index 77b51de14..2536ca9d1 100644 --- a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj +++ b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj @@ -50,6 +50,7 @@ + diff --git a/Duplicati/Library/Utility/FilterExpression.cs b/Duplicati/Library/Utility/FilterExpression.cs index 20b72b383..a196cca0d 100644 --- a/Duplicati/Library/Utility/FilterExpression.cs +++ b/Duplicati/Library/Utility/FilterExpression.cs @@ -293,7 +293,7 @@ namespace Duplicati.Library.Utility /// /// The internal list of expressions /// - private List m_filters; + private readonly List m_filters; /// /// Gets the type of the filter @@ -304,13 +304,13 @@ namespace Duplicati.Library.Utility /// Gets the result returned if an entry matches /// public readonly bool Result; - + /// - /// Gets a value indicating whether this is empty. + /// Gets a value indicating whether this is empty. /// /// true if empty; otherwise, false. - public bool Empty { get { return this.Type == FilterType.Empty; } } - + public bool Empty => Type == FilterType.Empty; + /// /// Gets the simple list, if the type is simple, named or wildcard /// @@ -320,14 +320,13 @@ namespace Duplicati.Library.Utility if (this.Type == FilterType.Simple || this.Type == FilterType.Wildcard) return (from n in m_filters select n.Filter).ToArray(); else - throw new InvalidOperationException(string.Format("Cannot extract simple list when the type is: {0}", this.Type)); + throw new InvalidOperationException($"Cannot extract simple list when the type is: {this.Type}"); } /// /// Gets a value indicating if the filter matches the path /// /// The match result - /// The match result /// The filter that matched public bool Matches(string path, out bool result, out IFilter match) { @@ -338,7 +337,7 @@ namespace Duplicati.Library.Utility return false; } - if (m_filters.Where(x => x.Matches(path)).Any()) + if (m_filters.Any(x => x.Matches(path))) { match = this; result = this.Result; @@ -349,27 +348,35 @@ namespace Duplicati.Library.Utility return false; } + /// + public string GetFilterHash() + { + var hash = MD5HashHelper.GetHash(m_filters.Select(x => x.Filter)); + return MD5HashHelper.GetHashString(hash); + } + /// - /// Creates a new instance, representing an empty filter. + /// Creates a new instance, representing an empty filter. /// public FilterExpression() : this((IEnumerable)null, true) { } - + /// - /// Creates a new instance. + /// Creates a new instance. /// /// The filter string that represents the filter public FilterExpression(string filter, bool result = true) : this(Expand(filter), result) { } - + /// - /// Creates a new instance. + /// Creates a new instance. /// /// The filter string that represents the filter + /// Return value of in case of match public FilterExpression(IEnumerable filter, bool result = true) { this.Result = result; @@ -605,9 +612,9 @@ namespace Duplicati.Library.Utility } /// - /// Returns a that represents the current . + /// Returns a that represents the current . /// - /// A that represents the current . + /// A that represents the current . public override string ToString() { if (this.Empty) @@ -633,11 +640,7 @@ namespace Duplicati.Library.Utility return (from n in m_filters - select string.Format( - "{0}{1}", - this.Result ? "+" : "-", - n.ToString() - ) + select $"{(this.Result ? "+" : "-")}{n.ToString()}" ).ToArray(); } diff --git a/Duplicati/Library/Utility/IFilter.cs b/Duplicati/Library/Utility/IFilter.cs index 9ed54c89d..e7968baf0 100644 --- a/Duplicati/Library/Utility/IFilter.cs +++ b/Duplicati/Library/Utility/IFilter.cs @@ -36,6 +36,12 @@ namespace Duplicati.Library.Utility /// The match result /// The filter that matched bool Matches(string entry, out bool result, out IFilter match); + + /// + /// Returns a MD5 hash string representing the filter + /// + /// + string GetFilterHash(); } } diff --git a/Duplicati/Library/Utility/JoinedFilterExpression.cs b/Duplicati/Library/Utility/JoinedFilterExpression.cs index 882183077..955129774 100644 --- a/Duplicati/Library/Utility/JoinedFilterExpression.cs +++ b/Duplicati/Library/Utility/JoinedFilterExpression.cs @@ -15,7 +15,6 @@ // You should have received a copy of the GNU Lesser General Public // License along with this library; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -using System; namespace Duplicati.Library.Utility { @@ -35,6 +34,16 @@ namespace Duplicati.Library.Utility return First.Matches(entry, out result, out match) || Second.Matches(entry, out result, out match); } + /// + /// Returns MD5 hash of filter expression + /// + /// MD5 hash of filter expression + public string GetFilterHash() + { + var hash = MD5HashHelper.GetHash(new[] {First.GetFilterHash(), Second.GetFilterHash()}); + return MD5HashHelper.GetHashString(hash); + } + public bool Empty { get { return First.Empty && Second.Empty; } } public static IFilter Join(IFilter first, IFilter second) diff --git a/Duplicati/Library/Utility/MD5HashHelper.cs b/Duplicati/Library/Utility/MD5HashHelper.cs new file mode 100644 index 000000000..8d82a876c --- /dev/null +++ b/Duplicati/Library/Utility/MD5HashHelper.cs @@ -0,0 +1,59 @@ +#region Disclaimer / License +// Copyright (C) 2015, The Duplicati Team +// http://www.duplicati.com, info@duplicati.com +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// +#endregion + +using System.Collections.Generic; +using System.Security.Cryptography; +using System.Text; + +namespace Duplicati.Library.Utility +{ + public static class MD5HashHelper + { + public static byte[] GetHash(string inputString) + { + HashAlgorithm algorithm = MD5.Create(); //or use SHA256.Create(); + return algorithm.ComputeHash(Encoding.UTF8.GetBytes(inputString)); + } + + public static byte[] GetHash(IEnumerable inputStrings) + { + using (var md5 = MD5.Create()) + { + foreach (var str in inputStrings) + { + var inputBuffer = Encoding.UTF8.GetBytes(str); + md5.TransformBlock(inputBuffer, 0, inputBuffer.Length, inputBuffer, 0); + } + + md5.TransformFinalBlock(new byte[0], 0, 0); + return md5.Hash; + } + } + + public static string GetHashString(byte[] hash) + { + var sb = new StringBuilder(); + foreach (var b in hash) + sb.Append(b.ToString("X2")); + + return sb.ToString(); + } + } +} diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index d3c092b9f..f7b75be4e 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -16,13 +16,13 @@ // License along with this library; if not, write to the Free Software // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA // +#endregion + +using System; using System.Text.RegularExpressions; using System.Linq; - - -#endregion -using System; using System.Collections.Generic; +using System.IO; using System.Text; using System.Threading.Tasks; @@ -268,19 +268,7 @@ namespace Duplicati.Library.Utility { Stack lst = new Stack(); - var isFolder = false; - try - { - if (attributeReader == null) - isFolder = true; - else - isFolder = (attributeReader(rootpath) & System.IO.FileAttributes.Directory) == System.IO.FileAttributes.Directory; - } - catch - { - } - - if (isFolder) + if (IsFolder(rootpath, attributeReader)) { rootpath = AppendDirSeparator(rootpath); try @@ -406,6 +394,139 @@ namespace Duplicati.Library.Utility } } + /// + /// Test if specified path is a folder + /// + /// Path to test + /// Function to use for testing path + /// True if path is refers to a folder + public static bool IsFolder(string path, ExtractFileAttributes attributeReader) + { + if (attributeReader == null) + return true; + + try + { + return attributeReader(path).HasFlag(System.IO.FileAttributes.Directory); + } + catch + { + return false; + } + } + + /// + /// Tests if path refers to a file, or folder, below the parent folder + /// + /// File or folder to test + /// Candidate parent folder + /// True if below parent folder, false otherwise (note that this returns false if the two argument paths are identical!) + public static bool IsPathBelowFolder(string fileOrFolderPath, string parentFolder) + { + var candidatePath = fileOrFolderPath; + parentFolder = AppendDirSeparator(parentFolder); + + while (true) + { + candidatePath = GetParent(candidatePath, true); + if (candidatePath == null) + return false; + + if (candidatePath.Equals(parentFolder, ClientFilenameStringComparision)) + return true; + } + } + + /// + /// Returns parent folder of path + /// + /// Full file or folder path + /// If true, return value always has trailing separator + /// Parent folder of path (containing folder for file paths, parent folder for folder paths) + public static string GetParent(string path, bool forceTrailingDirectorySeparator = false) + { + var len = path.Length - 1; + if (len > 1 && path[len] == Path.DirectorySeparatorChar) + { + len--; + } + + var last = path.LastIndexOf(Path.DirectorySeparatorChar, len); + if (last == -1 || last == 0 && len == 0) + return null; + + if (last == 0 && !IsClientWindows) + return DirectorySeparatorString; + + var parent = path.Substring(0, last); + + if (forceTrailingDirectorySeparator || + IsClientWindows && parent.Length == 2 && parent[1] == ':' && char.IsLetter(parent[0])) + { + parent += Path.DirectorySeparatorChar; + } + + return parent; + } + + + + /// + /// Given a collection of unique folders, returns only parent-most folders + /// + /// Collection of unique folders + /// Parent-most folders of input collection + public static IEnumerable SimplifyFolderList(ICollection folders) + { + if (!folders.Any()) + return folders; + + var result = new LinkedList(); + result.AddFirst(folders.First()); + + foreach (var folder1 in folders) + { + bool addFolder = true; + LinkedListNode next; + for (var node = result.First; node != null; node = next) + { + next = node.Next; + var folder2 = node.Value; + + if (IsPathBelowFolder(folder1, folder2)) + { + // higher-level folder already present + addFolder = false; + break; + } + + if (IsPathBelowFolder(folder2, folder1)) + { + // retain folder1 + result.Remove(node); + } + } + + if (addFolder) + { + result.AddFirst(folder1); + } + } + + return result.Distinct(); + } + + /// + /// Given a collection of file paths, return those NOT contained within specified collection of folders + /// + /// Collection of files to filter + /// Collection of folders to use as filter + /// Files not in any of specified folders + public static IEnumerable GetFilesNotInFolders(IEnumerable files, IEnumerable folders) + { + return files.Where(x => folders.All(folder => !IsPathBelowFolder(x, folder))); + } + /// /// Calculates the size of files in a given folder ///