diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index d353bda4e..781c64577 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -28,6 +28,11 @@ namespace Duplicati.Library.Main.Database { } + public LocalTestDatabase(LocalDatabase parent) + : base(parent) + { + } + public void UpdateVerificationCount(string name) { using(var cmd = m_connection.CreateCommand()) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 4f0f8a828..861bc2223 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -158,7 +158,6 @@ namespace Duplicati.Library.Main.Operation try { - m_transaction = m_database.BeginTransaction(); m_snapshot = GetSnapshot(sources, m_options, m_result); // Start parallel scan @@ -199,7 +198,19 @@ namespace Duplicati.Library.Main.Operation throw; } } + + if (m_options.BackupTestSampleCount > 0 && m_database.GetRemoteVolumes().Count() > 0) + { + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PreBackupTest); + m_result.TestResults = new TestResults(m_result); + + using(var testdb = new LocalTestDatabase(m_database)) + new TestHandler(m_backendurl, m_options, new TestResults(m_result)) + .DoRun(m_options.BackupTestSampleCount, testdb, m_backend); + } } + + m_transaction = m_database.BeginTransaction(); var incompleteFilesets = m_database.GetIncompleteFilesets(m_transaction).ToArray(); if (incompleteFilesets.Length != 0) diff --git a/Duplicati/Library/Main/Operation/TestHandler.cs b/Duplicati/Library/Main/Operation/TestHandler.cs index f39b6fc3b..561f3172a 100644 --- a/Duplicati/Library/Main/Operation/TestHandler.cs +++ b/Duplicati/Library/Main/Operation/TestHandler.cs @@ -39,16 +39,7 @@ namespace Duplicati.Library.Main.Operation { if (!System.IO.File.Exists(m_options.Dbpath)) throw new Exception(string.Format("Database file does not exist: {0}", m_options.Dbpath)); - - var blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm); - - if (blockhasher == null) - throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.BlockHashAlgorithm)); - if (!blockhasher.CanReuseTransform) - throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, m_options.BlockHashAlgorithm)); - - var hashsize = blockhasher.HashSize / 8; - + using(var db = new LocalTestDatabase(m_options.Dbpath)) using(var backend = new BackendManager(m_backendurl, m_options, m_results.BackendWriter, db)) { @@ -58,101 +49,115 @@ namespace Duplicati.Library.Main.Operation if (!m_options.NoBackendverification) FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_results.BackendWriter); - var files = db.SelectTestTargets(samples, m_options).ToList(); + DoRun(samples, db, backend); + } + } + + public void DoRun(long samples, LocalTestDatabase db, BackendManager backend) + { + + var blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm); + + if (blockhasher == null) + throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.BlockHashAlgorithm)); + if (!blockhasher.CanReuseTransform) + throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, m_options.BlockHashAlgorithm)); - foreach(var vol in new AsyncDownloader(files, backend)) + var hashsize = blockhasher.HashSize / 8; + var files = db.SelectTestTargets(samples, m_options).ToList(); + + foreach(var vol in new AsyncDownloader(files, backend)) + { + var parsedInfo = Volumes.VolumeBase.ParseFilename(vol.Name); + try { - var parsedInfo = Volumes.VolumeBase.ParseFilename(vol.Name); - try + using(var tf = vol.TempFile) { - using(var tf = vol.TempFile) + if (parsedInfo.FileType == RemoteVolumeType.Files) { - if (parsedInfo.FileType == RemoteVolumeType.Files) + //Compare with db and see if all files are accounted for + // with correct file hashes and blocklist hashes + using(var fl = db.CreateFilelist(vol.Name)) { - //Compare with db and see if all files are accounted for - // with correct file hashes and blocklist hashes - using(var fl = db.CreateFilelist(vol.Name)) - { - using(var rd = new Volumes.FilesetVolumeReader(parsedInfo.CompressionModule, tf, m_options)) - foreach(var f in rd.Files) - fl.Add(f.Path, f.Size, f.Hash, f.Metasize, f.Metahash, f.BlocklistHashes, f.Type, f.Time); - - m_results.AddResult(vol.Name, fl.Compare().ToList()); - } - } - else if (parsedInfo.FileType == RemoteVolumeType.Index) - { - var blocklinks = new List>(); - IEnumerable> combined = new KeyValuePair[0]; + using(var rd = new Volumes.FilesetVolumeReader(parsedInfo.CompressionModule, tf, m_options)) + foreach(var f in rd.Files) + fl.Add(f.Path, f.Size, f.Hash, f.Metasize, f.Metahash, f.BlocklistHashes, f.Type, f.Time); - //Compare with db and see that all hashes and volumes are listed - using(var rd = new Volumes.IndexVolumeReader(parsedInfo.CompressionModule, tf, m_options, hashsize)) - foreach(var v in rd.Volumes) - { - blocklinks.Add(new Tuple(v.Filename, v.Hash, v.Length)); - using(var bl = db.CreateBlocklist(v.Filename)) - { - foreach(var h in v.Blocks) - bl.AddBlock(h.Key, h.Value); - - combined = combined.Union(bl.Compare().ToArray()); - } - } - - using(var il = db.CreateIndexlist(vol.Name)) - { - foreach(var t in blocklinks) - il.AddBlockLink(t.Item1, t.Item2, t.Item3); - - combined = combined.Union(il.Compare()).ToList(); - } - - m_results.AddResult(vol.Name, combined.ToList()); - } - else if (parsedInfo.FileType == RemoteVolumeType.Blocks) - { - using(var bl = db.CreateBlocklist(vol.Name)) - using(var rd = new Volumes.BlockVolumeReader(parsedInfo.CompressionModule, tf, m_options)) - { - //Verify that all blocks are in the file - foreach(var b in rd.Blocks) - bl.AddBlock(b.Key, b.Value); - - //Select 20% random blocks and verify their hashes match the filename and size - var hashsamples = new List>(rd.Blocks); - var sampleCount = Math.Min(Math.Max(0, (int)(hashsamples.Count * 0.2)), hashsamples.Count - 1); - var rnd = new Random(); - - while(hashsamples.Count > sampleCount) - hashsamples.RemoveAt(rnd.Next(hashsamples.Count)); - - var blockbuffer = new byte[m_options.Blocksize]; - var changes = new List>(); - foreach(var s in hashsamples) - { - var size = rd.ReadBlock(s.Key, blockbuffer); - if (size != s.Value) - changes.Add(new KeyValuePair(Library.Interface.TestEntryStatus.Modified, s.Key)); - else - { - var hash = Convert.ToBase64String(blockhasher.ComputeHash(blockbuffer, 0, size)); - if (hash != s.Key) - changes.Add(new KeyValuePair(Library.Interface.TestEntryStatus.Modified, s.Key)); - } - } - - m_results.AddResult(vol.Name, changes.Union(bl.Compare().ToList())); - } - - } + m_results.AddResult(vol.Name, fl.Compare().ToList()); + } + } + else if (parsedInfo.FileType == RemoteVolumeType.Index) + { + var blocklinks = new List>(); + IEnumerable> combined = new KeyValuePair[0]; + + //Compare with db and see that all hashes and volumes are listed + using(var rd = new Volumes.IndexVolumeReader(parsedInfo.CompressionModule, tf, m_options, hashsize)) + foreach(var v in rd.Volumes) + { + blocklinks.Add(new Tuple(v.Filename, v.Hash, v.Length)); + using(var bl = db.CreateBlocklist(v.Filename)) + { + foreach(var h in v.Blocks) + bl.AddBlock(h.Key, h.Value); + + combined = combined.Union(bl.Compare().ToArray()); + } + } + + using(var il = db.CreateIndexlist(vol.Name)) + { + foreach(var t in blocklinks) + il.AddBlockLink(t.Item1, t.Item2, t.Item3); + + combined = combined.Union(il.Compare()).ToList(); + } + + m_results.AddResult(vol.Name, combined.ToList()); + } + else if (parsedInfo.FileType == RemoteVolumeType.Blocks) + { + using(var bl = db.CreateBlocklist(vol.Name)) + using(var rd = new Volumes.BlockVolumeReader(parsedInfo.CompressionModule, tf, m_options)) + { + //Verify that all blocks are in the file + foreach(var b in rd.Blocks) + bl.AddBlock(b.Key, b.Value); + + //Select 20% random blocks and verify their hashes match the filename and size + var hashsamples = new List>(rd.Blocks); + var sampleCount = Math.Min(Math.Max(0, (int)(hashsamples.Count * 0.2)), hashsamples.Count - 1); + var rnd = new Random(); + + while(hashsamples.Count > sampleCount) + hashsamples.RemoveAt(rnd.Next(hashsamples.Count)); + + var blockbuffer = new byte[m_options.Blocksize]; + var changes = new List>(); + foreach(var s in hashsamples) + { + var size = rd.ReadBlock(s.Key, blockbuffer); + if (size != s.Value) + changes.Add(new KeyValuePair(Library.Interface.TestEntryStatus.Modified, s.Key)); + else + { + var hash = Convert.ToBase64String(blockhasher.ComputeHash(blockbuffer, 0, size)); + if (hash != s.Key) + changes.Add(new KeyValuePair(Library.Interface.TestEntryStatus.Modified, s.Key)); + } + } + + m_results.AddResult(vol.Name, changes.Union(bl.Compare().ToList())); + } + } - - db.UpdateVerificationCount(vol.Name); - } - catch (Exception ex) - { - m_results.AddError(string.Format("Failed to process file {0}", vol.Name), ex); } + + db.UpdateVerificationCount(vol.Name); + } + catch (Exception ex) + { + m_results.AddError(string.Format("Failed to process file {0}", vol.Name), ex); } } } diff --git a/Duplicati/Library/Main/OperationPhase.cs b/Duplicati/Library/Main/OperationPhase.cs index ba4114e56..213259f6f 100644 --- a/Duplicati/Library/Main/OperationPhase.cs +++ b/Duplicati/Library/Main/OperationPhase.cs @@ -22,6 +22,7 @@ namespace Duplicati.Library.Main { Backup_Begin, Backup_PreBackupVerify, + Backup_PreBackupTest, Backup_PreviousBackupFinalize, Backup_ProcessingFiles, Backup_Finalize, diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index 02b05cc8d..1a8989b29 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -439,6 +439,7 @@ namespace Duplicati.Library.Main new CommandLineArgument("threshold", CommandLineArgument.ArgumentType.Size, Strings.Options.ThresholdShort, Strings.Options.ThresholdLong, DEFAULT_THRESHOLD.ToString()), new CommandLineArgument("index-file-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.IndexfilepolicyShort, Strings.Options.IndexfilepolicyLong, IndexFileStrategy.Lookup.ToString(), null, Enum.GetNames(typeof(IndexFileStrategy))), new CommandLineArgument("no-backend-verification", CommandLineArgument.ArgumentType.Boolean, Strings.Options.NobackendverificationShort, Strings.Options.NobackendverificationLong, "false"), + new CommandLineArgument("backup-test-samples", CommandLineArgument.ArgumentType.Integer, Strings.Options.BackendtestsamplesShort, string.Format(Strings.Options.BackendtestsamplesLong, "no-backend-verification"), "1"), new CommandLineArgument("dry-run", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DryrunShort, Strings.Options.DryrunLong, "false", new string[] { "dryrun" }), new CommandLineArgument("block-hash-algorithm", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.BlockhashalgorithmShort, Strings.Options.BlockhashalgorithmLong, DEFAULT_BLOCK_HASH_ALGORITHM, null, GetSupportedHashes()), @@ -1396,6 +1397,22 @@ namespace Duplicati.Library.Main { get { return Library.Utility.Utility.ParseBoolOption(m_options, "no-backend-verification"); } } + + /// + /// Gets the number of samples to test during a backup operation + /// + public long BackupTestSampleCount + { + get + { + string s; + m_options.TryGetValue("backup-test-samples", out s); + if (string.IsNullOrEmpty(s)) + return 1; + + return long.Parse(s); + } + } /// /// Gets a flag indicating if compacting should not be done automatically diff --git a/Duplicati/Library/Main/ResultClasses.cs b/Duplicati/Library/Main/ResultClasses.cs index 64204881b..0ff40e24a 100644 --- a/Duplicati/Library/Main/ResultClasses.cs +++ b/Duplicati/Library/Main/ResultClasses.cs @@ -471,6 +471,7 @@ namespace Duplicati.Library.Main public ICompactResults CompactResults { get; internal set; } public IDeleteResults DeleteResults { get; internal set; } public IRepairResults RepairResults { get; internal set; } + public ITestResults TestResults { get; internal set; } } internal class RestoreResults : BasicResults, Library.Interface.IRestoreResults @@ -682,6 +683,9 @@ namespace Duplicati.Library.Main internal class TestResults : BasicResults, ITestResults { + public TestResults() : base() { } + public TestResults(BasicResults p) : base(p) { } + public override OperationMode MainOperation { get { return OperationMode.Test; } } public IEnumerable>>> Changes { get { return m_changes; } } private List>>> m_changes = new List>>>(); diff --git a/Duplicati/Library/Main/Strings/Options.Designer.cs b/Duplicati/Library/Main/Strings/Options.Designer.cs index aa60f0646..b29e52a58 100644 --- a/Duplicati/Library/Main/Strings/Options.Designer.cs +++ b/Duplicati/Library/Main/Strings/Options.Designer.cs @@ -130,6 +130,12 @@ namespace Duplicati.Library.Main.Strings { } } + internal static string BackendtestsamplesShort { + get { + return ResourceManager.GetString("BackendtestsamplesShort", resourceCulture); + } + } + internal static string UsnpolicyShort { get { return ResourceManager.GetString("UsnpolicyShort", resourceCulture); @@ -364,6 +370,12 @@ namespace Duplicati.Library.Main.Strings { } } + internal static string BackendtestsamplesLong { + get { + return ResourceManager.GetString("BackendtestsamplesLong", resourceCulture); + } + } + internal static string AllowmissingsourceLong { get { return ResourceManager.GetString("AllowmissingsourceLong", resourceCulture); diff --git a/Duplicati/Library/Main/Strings/Options.resx b/Duplicati/Library/Main/Strings/Options.resx index 3763bcbe8..50fef3c7a 100644 --- a/Duplicati/Library/Main/Strings/Options.resx +++ b/Duplicati/Library/Main/Strings/Options.resx @@ -561,4 +561,10 @@ Use this option to upload a verification file after changing the remote storage. The file is not encrypted and contains the size and SHA256 hashes of all the remote files and can be used to verify the integrity of the files. + + The number of samples to test before a backup + + + Before a backup is started, some files are selected for verification on the remote backend. Use this option to change how many. If this value is set to 0 or the option --{0} is set, no remote files are verified + \ No newline at end of file