haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added automatic backend volume testing with each backup

Browse Source
This commit is contained in:
Kenneth Skovhede
2013-08-24 20:59:09 +02:00
parent 022d1ad76b
commit 1a44eab08b
8 changed files with 159 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Main/Database/LocalTestDatabase.cs
+5
View File
@@ -28,6 +28,11 @@ namespace Duplicati.Library.Main.Database
{
}
public LocalTestDatabase(LocalDatabase parent)
: base(parent)
{
}
public void UpdateVerificationCount(string name)
{
using(var cmd = m_connection.CreateCommand())
Duplicati/Library/Main/Operation/BackupHandler.cs
+12 -1
View File
@@ -158,7 +158,6 @@ namespace Duplicati.Library.Main.Operation
try
{
m_transaction = m_database.BeginTransaction();
m_snapshot = GetSnapshot(sources, m_options, m_result);
// Start parallel scan
@@ -199,7 +198,19 @@ namespace Duplicati.Library.Main.Operation
throw;
}
}
if (m_options.BackupTestSampleCount > 0 && m_database.GetRemoteVolumes().Count() > 0)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PreBackupTest);
m_result.TestResults = new TestResults(m_result);
using(var testdb = new LocalTestDatabase(m_database))
new TestHandler(m_backendurl, m_options, new TestResults(m_result))
.DoRun(m_options.BackupTestSampleCount, testdb, m_backend);
}
}
m_transaction = m_database.BeginTransaction();
var incompleteFilesets = m_database.GetIncompleteFilesets(m_transaction).ToArray();
if (incompleteFilesets.Length != 0)
Duplicati/Library/Main/Operation/TestHandler.cs
+102 -97
View File
@@ -39,16 +39,7 @@ namespace Duplicati.Library.Main.Operation
{
if (!System.IO.File.Exists(m_options.Dbpath))
throw new Exception(string.Format("Database file does not exist: {0}", m_options.Dbpath));
var blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
if (blockhasher == null)
throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.BlockHashAlgorithm));
if (!blockhasher.CanReuseTransform)
throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, m_options.BlockHashAlgorithm));
var hashsize = blockhasher.HashSize / 8;
using(var db = new LocalTestDatabase(m_options.Dbpath))
using(var backend = new BackendManager(m_backendurl, m_options, m_results.BackendWriter, db))
{
@@ -58,101 +49,115 @@ namespace Duplicati.Library.Main.Operation
if (!m_options.NoBackendverification)
FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_results.BackendWriter);
var files = db.SelectTestTargets(samples, m_options).ToList();
DoRun(samples, db, backend);
}
}
public void DoRun(long samples, LocalTestDatabase db, BackendManager backend)
{
var blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
if (blockhasher == null)
throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.BlockHashAlgorithm));
if (!blockhasher.CanReuseTransform)
throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, m_options.BlockHashAlgorithm));
foreach(var vol in new AsyncDownloader(files, backend))
var hashsize = blockhasher.HashSize / 8;
var files = db.SelectTestTargets(samples, m_options).ToList();
foreach(var vol in new AsyncDownloader(files, backend))
{
var parsedInfo = Volumes.VolumeBase.ParseFilename(vol.Name);
try
{
var parsedInfo = Volumes.VolumeBase.ParseFilename(vol.Name);
try
using(var tf = vol.TempFile)
{
using(var tf = vol.TempFile)
if (parsedInfo.FileType == RemoteVolumeType.Files)
{
if (parsedInfo.FileType == RemoteVolumeType.Files)
//Compare with db and see if all files are accounted for
// with correct file hashes and blocklist hashes
using(var fl = db.CreateFilelist(vol.Name))
{
//Compare with db and see if all files are accounted for
// with correct file hashes and blocklist hashes
using(var fl = db.CreateFilelist(vol.Name))
{
using(var rd = new Volumes.FilesetVolumeReader(parsedInfo.CompressionModule, tf, m_options))
foreach(var f in rd.Files)
fl.Add(f.Path, f.Size, f.Hash, f.Metasize, f.Metahash, f.BlocklistHashes, f.Type, f.Time);
m_results.AddResult(vol.Name, fl.Compare().ToList());
}
}
else if (parsedInfo.FileType == RemoteVolumeType.Index)
{
var blocklinks = new List<Tuple<string, string, long>>();
IEnumerable<KeyValuePair<Duplicati.Library.Interface.TestEntryStatus, string>> combined = new KeyValuePair<Duplicati.Library.Interface.TestEntryStatus, string>[0];
using(var rd = new Volumes.FilesetVolumeReader(parsedInfo.CompressionModule, tf, m_options))
foreach(var f in rd.Files)
fl.Add(f.Path, f.Size, f.Hash, f.Metasize, f.Metahash, f.BlocklistHashes, f.Type, f.Time);
//Compare with db and see that all hashes and volumes are listed
using(var rd = new Volumes.IndexVolumeReader(parsedInfo.CompressionModule, tf, m_options, hashsize))
foreach(var v in rd.Volumes)
{
blocklinks.Add(new Tuple<string, string, long>(v.Filename, v.Hash, v.Length));
using(var bl = db.CreateBlocklist(v.Filename))
{
foreach(var h in v.Blocks)
bl.AddBlock(h.Key, h.Value);
combined = combined.Union(bl.Compare().ToArray());
}
}
using(var il = db.CreateIndexlist(vol.Name))
{
foreach(var t in blocklinks)
il.AddBlockLink(t.Item1, t.Item2, t.Item3);
combined = combined.Union(il.Compare()).ToList();
}
m_results.AddResult(vol.Name, combined.ToList());
}
else if (parsedInfo.FileType == RemoteVolumeType.Blocks)
{
using(var bl = db.CreateBlocklist(vol.Name))
using(var rd = new Volumes.BlockVolumeReader(parsedInfo.CompressionModule, tf, m_options))
{
//Verify that all blocks are in the file
foreach(var b in rd.Blocks)
bl.AddBlock(b.Key, b.Value);
//Select 20% random blocks and verify their hashes match the filename and size
var hashsamples = new List<KeyValuePair<string, long>>(rd.Blocks);
var sampleCount = Math.Min(Math.Max(0, (int)(hashsamples.Count * 0.2)), hashsamples.Count - 1);
var rnd = new Random();
while(hashsamples.Count > sampleCount)
hashsamples.RemoveAt(rnd.Next(hashsamples.Count));
var blockbuffer = new byte[m_options.Blocksize];
var changes = new List<KeyValuePair<Library.Interface.TestEntryStatus, string>>();
foreach(var s in hashsamples)
{
var size = rd.ReadBlock(s.Key, blockbuffer);
if (size != s.Value)
changes.Add(new KeyValuePair<Library.Interface.TestEntryStatus, string>(Library.Interface.TestEntryStatus.Modified, s.Key));
else
{
var hash = Convert.ToBase64String(blockhasher.ComputeHash(blockbuffer, 0, size));
if (hash != s.Key)
changes.Add(new KeyValuePair<Library.Interface.TestEntryStatus, string>(Library.Interface.TestEntryStatus.Modified, s.Key));
}
}
m_results.AddResult(vol.Name, changes.Union(bl.Compare().ToList()));
}
}
m_results.AddResult(vol.Name, fl.Compare().ToList());
}
}
else if (parsedInfo.FileType == RemoteVolumeType.Index)
{
var blocklinks = new List<Tuple<string, string, long>>();
IEnumerable<KeyValuePair<Duplicati.Library.Interface.TestEntryStatus, string>> combined = new KeyValuePair<Duplicati.Library.Interface.TestEntryStatus, string>[0];
//Compare with db and see that all hashes and volumes are listed
using(var rd = new Volumes.IndexVolumeReader(parsedInfo.CompressionModule, tf, m_options, hashsize))
foreach(var v in rd.Volumes)
{
blocklinks.Add(new Tuple<string, string, long>(v.Filename, v.Hash, v.Length));
using(var bl = db.CreateBlocklist(v.Filename))
{
foreach(var h in v.Blocks)
bl.AddBlock(h.Key, h.Value);
combined = combined.Union(bl.Compare().ToArray());
}
}
using(var il = db.CreateIndexlist(vol.Name))
{
foreach(var t in blocklinks)
il.AddBlockLink(t.Item1, t.Item2, t.Item3);
combined = combined.Union(il.Compare()).ToList();
}
m_results.AddResult(vol.Name, combined.ToList());
}
else if (parsedInfo.FileType == RemoteVolumeType.Blocks)
{
using(var bl = db.CreateBlocklist(vol.Name))
using(var rd = new Volumes.BlockVolumeReader(parsedInfo.CompressionModule, tf, m_options))
{
//Verify that all blocks are in the file
foreach(var b in rd.Blocks)
bl.AddBlock(b.Key, b.Value);
//Select 20% random blocks and verify their hashes match the filename and size
var hashsamples = new List<KeyValuePair<string, long>>(rd.Blocks);
var sampleCount = Math.Min(Math.Max(0, (int)(hashsamples.Count * 0.2)), hashsamples.Count - 1);
var rnd = new Random();
while(hashsamples.Count > sampleCount)
hashsamples.RemoveAt(rnd.Next(hashsamples.Count));
var blockbuffer = new byte[m_options.Blocksize];
var changes = new List<KeyValuePair<Library.Interface.TestEntryStatus, string>>();
foreach(var s in hashsamples)
{
var size = rd.ReadBlock(s.Key, blockbuffer);
if (size != s.Value)
changes.Add(new KeyValuePair<Library.Interface.TestEntryStatus, string>(Library.Interface.TestEntryStatus.Modified, s.Key));
else
{
var hash = Convert.ToBase64String(blockhasher.ComputeHash(blockbuffer, 0, size));
if (hash != s.Key)
changes.Add(new KeyValuePair<Library.Interface.TestEntryStatus, string>(Library.Interface.TestEntryStatus.Modified, s.Key));
}
}
m_results.AddResult(vol.Name, changes.Union(bl.Compare().ToList()));
}
}
db.UpdateVerificationCount(vol.Name);
}
catch (Exception ex)
{
m_results.AddError(string.Format("Failed to process file {0}", vol.Name), ex);
}
db.UpdateVerificationCount(vol.Name);
}
catch (Exception ex)
{
m_results.AddError(string.Format("Failed to process file {0}", vol.Name), ex);
}
}
}
Duplicati/Library/Main/OperationPhase.cs
+1
View File
@@ -22,6 +22,7 @@ namespace Duplicati.Library.Main
{
Backup_Begin,
Backup_PreBackupVerify,
Backup_PreBackupTest,
Backup_PreviousBackupFinalize,
Backup_ProcessingFiles,
Backup_Finalize,
Duplicati/Library/Main/Options.cs
+17
View File
@@ -439,6 +439,7 @@ namespace Duplicati.Library.Main
new CommandLineArgument("threshold", CommandLineArgument.ArgumentType.Size, Strings.Options.ThresholdShort, Strings.Options.ThresholdLong, DEFAULT_THRESHOLD.ToString()),
new CommandLineArgument("index-file-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.IndexfilepolicyShort, Strings.Options.IndexfilepolicyLong, IndexFileStrategy.Lookup.ToString(), null, Enum.GetNames(typeof(IndexFileStrategy))),
new CommandLineArgument("no-backend-verification", CommandLineArgument.ArgumentType.Boolean, Strings.Options.NobackendverificationShort, Strings.Options.NobackendverificationLong, "false"),
new CommandLineArgument("backup-test-samples", CommandLineArgument.ArgumentType.Integer, Strings.Options.BackendtestsamplesShort, string.Format(Strings.Options.BackendtestsamplesLong, "no-backend-verification"), "1"),
new CommandLineArgument("dry-run", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DryrunShort, Strings.Options.DryrunLong, "false", new string[] { "dryrun" }),
new CommandLineArgument("block-hash-algorithm", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.BlockhashalgorithmShort, Strings.Options.BlockhashalgorithmLong, DEFAULT_BLOCK_HASH_ALGORITHM, null, GetSupportedHashes()),
@@ -1396,6 +1397,22 @@ namespace Duplicati.Library.Main
{
get { return Library.Utility.Utility.ParseBoolOption(m_options, "no-backend-verification"); }
}
/// <summary>
/// Gets the number of samples to test during a backup operation
/// </summary>
public long BackupTestSampleCount
{
get
{
string s;
m_options.TryGetValue("backup-test-samples", out s);
if (string.IsNullOrEmpty(s))
return 1;
return long.Parse(s);
}
}
/// <summary>
/// Gets a flag indicating if compacting should not be done automatically
Duplicati/Library/Main/ResultClasses.cs
+4
View File
@@ -471,6 +471,7 @@ namespace Duplicati.Library.Main
public ICompactResults CompactResults { get; internal set; }
public IDeleteResults DeleteResults { get; internal set; }
public IRepairResults RepairResults { get; internal set; }
public ITestResults TestResults { get; internal set; }
}
internal class RestoreResults : BasicResults, Library.Interface.IRestoreResults
@@ -682,6 +683,9 @@ namespace Duplicati.Library.Main
internal class TestResults : BasicResults, ITestResults
{
public TestResults() : base() { }
public TestResults(BasicResults p) : base(p) { }
public override OperationMode MainOperation { get { return OperationMode.Test; } }
public IEnumerable<KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>> Changes { get { return m_changes; } }
private List<KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>> m_changes = new List<KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>>();
Duplicati/Library/Main/Strings/Options.Designer.cs
Generated
+12
View File
@@ -130,6 +130,12 @@ namespace Duplicati.Library.Main.Strings {
}
}
internal static string BackendtestsamplesShort {
get {
return ResourceManager.GetString("BackendtestsamplesShort", resourceCulture);
}
}
internal static string UsnpolicyShort {
get {
return ResourceManager.GetString("UsnpolicyShort", resourceCulture);
@@ -364,6 +370,12 @@ namespace Duplicati.Library.Main.Strings {
}
}
internal static string BackendtestsamplesLong {
get {
return ResourceManager.GetString("BackendtestsamplesLong", resourceCulture);
}
}
internal static string AllowmissingsourceLong {
get {
return ResourceManager.GetString("AllowmissingsourceLong", resourceCulture);
Duplicati/Library/Main/Strings/Options.resx
+6
View File
@@ -561,4 +561,10 @@
<data name="UploadverificationfileLong" xml:space="preserve">
<value>Use this option to upload a verification file after changing the remote storage. The file is not encrypted and contains the size and SHA256 hashes of all the remote files and can be used to verify the integrity of the files.</value>
</data>
<data name="BackendtestsamplesShort" xml:space="preserve">
<value>The number of samples to test before a backup</value>
</data>
<data name="BackendtestsamplesLong" xml:space="preserve">
<value>Before a backup is started, some files are selected for verification on the remote backend. Use this option to change how many. If this value is set to 0 or the option --{0} is set, no remote files are verified</value>
</data>
</root>