haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updated TestHandler to use the new async database methods

Browse Source
This commit is contained in:
Carl Johnsen
2025-05-21 15:34:40 +02:00
parent 1f8c1747e8
commit 40bc75b585
1 changed files with 31 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Main/Operation/TestHandler.cs
+31 -32
View File
@@ -50,21 +50,20 @@ namespace Duplicati.Library.Main.Operation
if (!System.IO.File.Exists(m_options.Dbpath))
throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist");
using (var db = new LocalTestDatabase(m_options.Dbpath, m_options.SqlitePageCache))
using (var rtr = new ReusableTransaction(db))
using (var db = await LocalTestDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache))
{
Utility.UpdateOptionsFromDb(db, m_options, rtr.Transaction);
Utility.VerifyOptionsAndUpdateDatabase(db, m_options, rtr.Transaction);
db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks, rtr.Transaction);
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_results.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, rtr.Transaction).ConfigureAwait(false);
await DoRunAsync(samples, db, rtr, backendManager).ConfigureAwait(false);
rtr.Commit("TestHandlerComplete");
Utility.UpdateOptionsFromDb(db, m_options);
Utility.VerifyOptionsAndUpdateDatabase(db, m_options);
await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks);
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_results.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false);
await DoRunAsync(samples, db, backendManager).ConfigureAwait(false);
await db.Transaction.CommitAsync("TestHandlerComplete");
}
}
public async Task DoRunAsync(long samples, LocalTestDatabase db, ReusableTransaction rtr, IBackendManager backend)
public async Task DoRunAsync(long samples, LocalTestDatabase db, IBackendManager backend)
{
var files = db.SelectTestTargets(samples, m_options, rtr.Transaction).ToList();
var files = await db.SelectTestTargets(samples, m_options).ToListAsync();
m_results.OperationProgressUpdater.UpdatePhase(OperationPhase.Verify_Running);
m_results.OperationProgressUpdater.UpdateProgress(0);
@@ -79,7 +78,7 @@ namespace Duplicati.Library.Main.Operation
{
if (!await m_results.TaskControl.ProgressRendevouz().ConfigureAwait(false))
{
await backend.WaitForEmptyAsync(db, rtr.Transaction, m_results.TaskControl.ProgressToken).ConfigureAwait(false);
await backend.WaitForEmptyAsync(db, m_results.TaskControl.ProgressToken).ConfigureAwait(false);
m_results.EndTime = DateTime.UtcNow;
return;
}
@@ -89,14 +88,14 @@ namespace Duplicati.Library.Main.Operation
KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>> res;
using (tf)
res = TestVolumeInternals(db, rtr, vol, tf, m_options, m_options.FullBlockVerification ? 1.0 : 0.2);
res = await TestVolumeInternals(db, vol, tf, m_options, m_options.FullBlockVerification ? 1.0 : 0.2);
m_results.AddResult(res.Key, res.Value);
if (!string.IsNullOrWhiteSpace(vol.Hash) && vol.Size > 0)
{
if (res.Value == null || !res.Value.Any())
{
var rv = db.GetRemoteVolume(vol.Name, rtr.Transaction);
var rv = await db.GetRemoteVolume(vol.Name);
if (rv.ID < 0)
{
@@ -109,14 +108,14 @@ namespace Duplicati.Library.Main.Operation
else
{
Logging.Log.WriteInformationMessage(LOGTAG, "CaptureHashAndSize", "Successfully captured hash and size for {0}, updating database", vol.Name);
db.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Verified, vol.Size, vol.Hash, rtr.Transaction);
await db.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Verified, vol.Size, vol.Hash);
}
}
}
}
}
db.UpdateVerificationCount(vol.Name, rtr.Transaction);
await db.UpdateVerificationCount(vol.Name);
}
catch (Exception ex)
{
@@ -153,7 +152,7 @@ namespace Duplicati.Library.Main.Operation
(var tf, var hash, var size) = await backend.GetWithInfoAsync(f.Name, f.Hash, f.Size, m_results.TaskControl.ProgressToken).ConfigureAwait(false);
using (tf)
res = TestVolumeInternals(db, rtr, f, tf, m_options, 1);
res = await TestVolumeInternals(db, f, tf, m_options, 1);
m_results.AddResult(res.Key, res.Value);
if (!string.IsNullOrWhiteSpace(hash) && size > 0)
@@ -167,7 +166,7 @@ namespace Duplicati.Library.Main.Operation
else
{
Logging.Log.WriteInformationMessage(LOGTAG, "CapturedHashAndSize", "Successfully captured hash and size for {0}, updating database", f.Name);
db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Verified, size, hash, rtr.Transaction);
await db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Verified, size, hash);
}
}
}
@@ -178,7 +177,7 @@ namespace Duplicati.Library.Main.Operation
{ }
}
db.UpdateVerificationCount(f.Name, rtr.Transaction);
await db.UpdateVerificationCount(f.Name);
m_results.AddResult(f.Name, []);
}
catch (Exception ex)
@@ -214,7 +213,7 @@ namespace Duplicati.Library.Main.Operation
/// <param name="vol">The remote volume being examined</param>
/// <param name="tf">The path to the downloaded copy of the file</param>
/// <param name="sample_percent">A value between 0 and 1 that indicates how many blocks are tested in a dblock file</param>
public static KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>> TestVolumeInternals(LocalTestDatabase db, ReusableTransaction rtr, IRemoteVolume vol, string tf, Options options, double sample_percent)
public static async Task<KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>> TestVolumeInternals(LocalTestDatabase db, IRemoteVolume vol, string tf, Options options, double sample_percent)
{
var hashsize = HashFactory.HashSizeBytes(options.BlockHashAlgorithm);
var parsedInfo = Volumes.VolumeBase.ParseFilename(vol.Name);
@@ -223,15 +222,15 @@ namespace Duplicati.Library.Main.Operation
switch (parsedInfo.FileType)
{
case RemoteVolumeType.Files:
//Compare with db and see if all files are accounted for
//Compare with db and see if all files are accounted for
// with correct file hashes and blocklist hashes
using (var fl = db.CreateFilelist(vol.Name, rtr))
using (var fl = await db.CreateFilelist(vol.Name))
{
using (var rd = new Volumes.FilesetVolumeReader(parsedInfo.CompressionModule, tf, options))
foreach (var f in rd.Files)
fl.Add(f.Path, f.Size, f.Hash, f.Metasize, f.Metahash, f.BlocklistHashes, f.Type, f.Time);
await fl.Add(f.Path, f.Size, f.Hash, f.Metasize, f.Metahash, f.BlocklistHashes, f.Type, f.Time);
return new KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>(vol.Name, fl.Compare().ToList());
return new KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>(vol.Name, await fl.Compare().ToListAsync());
}
case RemoteVolumeType.Index:
@@ -243,32 +242,32 @@ namespace Duplicati.Library.Main.Operation
foreach (var v in rd.Volumes)
{
blocklinks.Add(new Tuple<string, string, long>(v.Filename, v.Hash, v.Length));
using (var bl = db.CreateBlocklist(v.Filename, rtr))
using (var bl = await db.CreateBlocklist(v.Filename))
{
foreach (var h in v.Blocks)
bl.AddBlock(h.Key, h.Value);
await bl.AddBlock(h.Key, h.Value);
combined = combined.Union(bl.Compare().ToArray());
combined = combined.Union(await bl.Compare().ToArrayAsync());
}
}
using (var il = db.CreateIndexlist(vol.Name, rtr))
using (var il = await db.CreateIndexlist(vol.Name))
{
foreach (var t in blocklinks)
il.AddBlockLink(t.Item1, t.Item2, t.Item3);
await il.AddBlockLink(t.Item1, t.Item2, t.Item3);
combined = combined.Union(il.Compare()).ToList();
combined = combined.Union(await il.Compare().ToArrayAsync()).ToList();
}
return new KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>(vol.Name, combined.ToList());
case RemoteVolumeType.Blocks:
using (var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm))
using (var bl = db.CreateBlocklist(vol.Name, rtr))
using (var bl = await db.CreateBlocklist(vol.Name))
using (var rd = new Volumes.BlockVolumeReader(parsedInfo.CompressionModule, tf, options))
{
//Verify that all blocks are in the file
foreach (var b in rd.Blocks)
bl.AddBlock(b.Key, b.Value);
await bl.AddBlock(b.Key, b.Value);
//Select random blocks and verify their hashes match the filename and size
var hashsamples = new List<KeyValuePair<string, long>>(rd.Blocks);
@@ -293,7 +292,7 @@ namespace Duplicati.Library.Main.Operation
}
}
return new KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>(vol.Name, changes.Union(bl.Compare().ToList()));
return new KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>(vol.Name, changes.Union(await bl.Compare().ToListAsync()));
}
}