From 0a9880c6cff30705e8ba34912eacfe8cf1f4d284 Mon Sep 17 00:00:00 2001 From: Rod Savard Date: Sat, 1 Feb 2020 13:34:21 -0800 Subject: [PATCH 1/4] change config hash calculation --- Duplicati/Library/Main/Operation/BackupHandler.cs | 12 +++++++++++- Duplicati/Library/Snapshots/UsnJournalService.cs | 10 +++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 3d48efebc..75ea8f442 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -103,8 +103,18 @@ namespace Duplicati.Library.Main.Operation private UsnJournalService GetJournalService(IEnumerable sources, ISnapshotService snapshot, IFilter filter, long lastfilesetid) { if (m_options.UsnStrategy == Options.OptimizationStrategy.Off) return null; + + // get hash identifying current source filter / sources configuration + // ReSharper disable once PossibleMultipleEnumeration + var configHash = Library.Utility.Utility.ByteArrayAsHexString(MD5HashHelper.GetHash( + (filter == null ? string.Empty : filter.ToString()) + + string.Join("; ", sources) + + m_options.FileAttributeFilter.ToString() + + m_options.SkipFilesLargerThan.ToString() + )); + var journalData = m_database.GetChangeJournalData(lastfilesetid); - var service = new UsnJournalService(sources, snapshot, filter, journalData, cancellationTokenSource.Token); + var service = new UsnJournalService(sources, snapshot, filter, configHash, journalData, cancellationTokenSource.Token); foreach (var volumeData in service.VolumeDataList) { diff --git a/Duplicati/Library/Snapshots/UsnJournalService.cs b/Duplicati/Library/Snapshots/UsnJournalService.cs index dd8f084ec..6e51ea57c 100644 --- a/Duplicati/Library/Snapshots/UsnJournalService.cs +++ b/Duplicati/Library/Snapshots/UsnJournalService.cs @@ -45,12 +45,12 @@ namespace Duplicati.Library.Snapshots /// /// Emit filter /// Journal-data of previous fileset - public UsnJournalService(IEnumerable sources, ISnapshotService snapshot, IFilter emitFilter, + public UsnJournalService(IEnumerable sources, ISnapshotService snapshot, IFilter emitFilter, string configHash, IEnumerable prevJournalData, CancellationToken token) { m_sources = sources; m_snapshot = snapshot; - m_volumeDataDict = Initialize(emitFilter, prevJournalData); + m_volumeDataDict = Initialize(emitFilter, configHash, prevJournalData); m_token = token; } @@ -62,17 +62,13 @@ namespace Duplicati.Library.Snapshots /// /// /// - private Dictionary Initialize(IFilter emitFilter, IEnumerable prevJournalData) + private Dictionary Initialize(IFilter emitFilter, string configHash, IEnumerable prevJournalData) { if (prevJournalData == null) throw new UsnJournalSoftFailureException(); var result = new Dictionary(); - // get filter identifying current source filter / sources configuration - // ReSharper disable once PossibleMultipleEnumeration - var configHash = (emitFilter == null ? string.Empty : emitFilter.GetFilterHash()) + Utility.Utility.ByteArrayAsHexString(MD5HashHelper.GetHash(m_sources)); - // create lookup for journal data var journalDataDict = prevJournalData.ToDictionary(data => data.Volume); From 685c77a018e9bea29221cd356c475b5dc4c0a421 Mon Sep 17 00:00:00 2001 From: Rod Savard Date: Sat, 1 Feb 2020 23:15:26 -0800 Subject: [PATCH 2/4] remove unused parameter --- Duplicati/Library/Main/Operation/BackupHandler.cs | 2 +- Duplicati/Library/Snapshots/UsnJournalService.cs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 75ea8f442..e0f53e99e 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -114,7 +114,7 @@ namespace Duplicati.Library.Main.Operation )); var journalData = m_database.GetChangeJournalData(lastfilesetid); - var service = new UsnJournalService(sources, snapshot, filter, configHash, journalData, cancellationTokenSource.Token); + var service = new UsnJournalService(sources, snapshot, configHash, journalData, cancellationTokenSource.Token); foreach (var volumeData in service.VolumeDataList) { diff --git a/Duplicati/Library/Snapshots/UsnJournalService.cs b/Duplicati/Library/Snapshots/UsnJournalService.cs index 6e51ea57c..42fcaa49b 100644 --- a/Duplicati/Library/Snapshots/UsnJournalService.cs +++ b/Duplicati/Library/Snapshots/UsnJournalService.cs @@ -45,12 +45,12 @@ namespace Duplicati.Library.Snapshots /// /// Emit filter /// Journal-data of previous fileset - public UsnJournalService(IEnumerable sources, ISnapshotService snapshot, IFilter emitFilter, string configHash, + public UsnJournalService(IEnumerable sources, ISnapshotService snapshot, string configHash, IEnumerable prevJournalData, CancellationToken token) { m_sources = sources; m_snapshot = snapshot; - m_volumeDataDict = Initialize(emitFilter, configHash, prevJournalData); + m_volumeDataDict = Initialize(configHash, prevJournalData); m_token = token; } @@ -62,7 +62,7 @@ namespace Duplicati.Library.Snapshots /// /// /// - private Dictionary Initialize(IFilter emitFilter, string configHash, IEnumerable prevJournalData) + private Dictionary Initialize(string configHash, IEnumerable prevJournalData) { if (prevJournalData == null) throw new UsnJournalSoftFailureException(); From a85c10fd8c6375859efd24eabb3c24112e866b58 Mon Sep 17 00:00:00 2001 From: Rod Savard Date: Sun, 2 Feb 2020 15:09:23 -0800 Subject: [PATCH 3/4] move config hash calculation to UsnJournalService --- .../Library/Main/Operation/BackupHandler.cs | 12 ++-------- .../Library/Snapshots/UsnJournalService.cs | 23 +++++++++++++++---- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index e0f53e99e..7660d2206 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -104,17 +104,9 @@ namespace Duplicati.Library.Main.Operation { if (m_options.UsnStrategy == Options.OptimizationStrategy.Off) return null; - // get hash identifying current source filter / sources configuration - // ReSharper disable once PossibleMultipleEnumeration - var configHash = Library.Utility.Utility.ByteArrayAsHexString(MD5HashHelper.GetHash( - (filter == null ? string.Empty : filter.ToString()) + - string.Join("; ", sources) + - m_options.FileAttributeFilter.ToString() + - m_options.SkipFilesLargerThan.ToString() - )); - var journalData = m_database.GetChangeJournalData(lastfilesetid); - var service = new UsnJournalService(sources, snapshot, configHash, journalData, cancellationTokenSource.Token); + var service = new UsnJournalService(sources, snapshot, filter, m_options.FileAttributeFilter, m_options.SkipFilesLargerThan, + journalData, cancellationTokenSource.Token); foreach (var volumeData in service.VolumeDataList) { diff --git a/Duplicati/Library/Snapshots/UsnJournalService.cs b/Duplicati/Library/Snapshots/UsnJournalService.cs index 42fcaa49b..40f90888f 100644 --- a/Duplicati/Library/Snapshots/UsnJournalService.cs +++ b/Duplicati/Library/Snapshots/UsnJournalService.cs @@ -44,13 +44,16 @@ namespace Duplicati.Library.Snapshots /// Sources to filter /// /// Emit filter + /// + /// /// Journal-data of previous fileset - public UsnJournalService(IEnumerable sources, ISnapshotService snapshot, string configHash, - IEnumerable prevJournalData, CancellationToken token) + /// + public UsnJournalService(IEnumerable sources, ISnapshotService snapshot, IFilter emitFilter, FileAttributes fileAttributeFilter, + long skipFilesLargerThan, IEnumerable prevJournalData, CancellationToken token) { m_sources = sources; m_snapshot = snapshot; - m_volumeDataDict = Initialize(configHash, prevJournalData); + m_volumeDataDict = Initialize(emitFilter, fileAttributeFilter, skipFilesLargerThan, prevJournalData); m_token = token; } @@ -60,15 +63,27 @@ namespace Duplicati.Library.Snapshots /// Initialize list of modified files / folder for each volume /// /// + /// + /// /// /// - private Dictionary Initialize(string configHash, IEnumerable prevJournalData) + private Dictionary Initialize(IFilter emitFilter, FileAttributes fileAttributeFilter, long skipFilesLargerThan, + IEnumerable prevJournalData) { if (prevJournalData == null) throw new UsnJournalSoftFailureException(); var result = new Dictionary(); + // get hash identifying current source filter / sources configuration + // ReSharper disable once PossibleMultipleEnumeration + var configHash = Library.Utility.Utility.ByteArrayAsHexString(MD5HashHelper.GetHash( + (emitFilter == null ? string.Empty : emitFilter.ToString()) + + string.Join("; ", m_sources) + + fileAttributeFilter.ToString() + + skipFilesLargerThan.ToString() + )); + // create lookup for journal data var journalDataDict = prevJournalData.ToDictionary(data => data.Volume); From 247a452d2b385e2024228b2fa97d776068d881f5 Mon Sep 17 00:00:00 2001 From: Rod Savard Date: Sun, 2 Feb 2020 20:53:09 -0800 Subject: [PATCH 4/4] send array of strings instead of single concatenated string --- Duplicati/Library/Snapshots/UsnJournalService.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Duplicati/Library/Snapshots/UsnJournalService.cs b/Duplicati/Library/Snapshots/UsnJournalService.cs index 40f90888f..f51d48291 100644 --- a/Duplicati/Library/Snapshots/UsnJournalService.cs +++ b/Duplicati/Library/Snapshots/UsnJournalService.cs @@ -77,12 +77,12 @@ namespace Duplicati.Library.Snapshots // get hash identifying current source filter / sources configuration // ReSharper disable once PossibleMultipleEnumeration - var configHash = Library.Utility.Utility.ByteArrayAsHexString(MD5HashHelper.GetHash( - (emitFilter == null ? string.Empty : emitFilter.ToString()) + - string.Join("; ", m_sources) + - fileAttributeFilter.ToString() + + var configHash = Utility.Utility.ByteArrayAsHexString(MD5HashHelper.GetHash(new string[] { + emitFilter == null ? string.Empty : emitFilter.ToString(), + string.Join("; ", m_sources), + fileAttributeFilter.ToString(), skipFilesLargerThan.ToString() - )); + })); // create lookup for journal data var journalDataDict = prevJournalData.ToDictionary(data => data.Volume);