haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rewrote testing to use an externally supplied transaction, to avoid temporary transactions from clearing the active transaction.

Browse Source 
This is needed because SQLite does not support multiple active transactions.
This commit is contained in:
Kenneth Skovhede
2025-04-14 12:05:10 +02:00
parent faede65d8d
commit 4773914639
4 changed files with 42 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Main/Database/LocalTestDatabase.cs
+19 -19
View File
@@ -40,9 +40,9 @@ namespace Duplicati.Library.Main.Database
{
}
public void UpdateVerificationCount(string name)
public void UpdateVerificationCount(string name, IDbTransaction? tr)
{
using (var cmd = m_connection.CreateCommand())
using (var cmd = m_connection.CreateCommand(tr))
cmd.SetCommandAndParameters(@"UPDATE ""RemoteVolume"" SET ""VerificationCount"" = MAX(1, CASE WHEN ""VerificationCount"" <= 0 THEN (SELECT MAX(""VerificationCount"") FROM ""RemoteVolume"") ELSE ""VerificationCount"" + 1 END) WHERE ""Name"" = @Name")
.SetParameterValue("@Name", name)
.ExecuteNonQuery();
@@ -118,12 +118,12 @@ namespace Duplicati.Library.Main.Database
return res;
}
public IEnumerable<IRemoteVolume> SelectTestTargets(long samples, Options options)
public IEnumerable<IRemoteVolume> SelectTestTargets(long samples, Options options, IDbTransaction? tr)
{
var tp = GetFilelistWhereClause(options.Time, options.Version);
samples = Math.Max(1, samples);
using (var cmd = m_connection.CreateCommand())
using (var cmd = m_connection.CreateCommand(tr))
{
// Select any broken items
cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"" WHERE (""State"" IN (@States)) AND (""Hash"" = '' OR ""Hash"" IS NULL OR ""Size"" <= 0) ")
@@ -194,14 +194,14 @@ namespace Duplicati.Library.Main.Database
protected readonly IDbConnection m_connection;
protected readonly string m_volumename;
protected string m_tablename;
protected IDbTransaction m_transaction;
protected IDbTransaction? m_transaction;
protected IDbCommand m_insertCommand;
protected Basiclist(IDbConnection connection, string volumename, string tablePrefix, string tableFormat, string insertCommand)
protected Basiclist(IDbConnection connection, IDbTransaction? tr, string volumename, string tablePrefix, string tableFormat, string insertCommand)
{
m_connection = connection;
m_volumename = volumename;
m_transaction = m_connection.BeginTransaction();
m_transaction = tr;
var tablename = tablePrefix + "-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
using (var cmd = m_connection.CreateCommand(m_transaction))
@@ -240,8 +240,8 @@ namespace Duplicati.Library.Main.Database
private const string TABLE_PREFIX = "Filelist";
private const string TABLE_FORMAT = @"(""Path"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Hash"" TEXT NULL, ""Metasize"" INTEGER NOT NULL, ""Metahash"" TEXT NOT NULL)";
private const string INSERT_COMMAND = @"(""Path"", ""Size"", ""Hash"", ""Metasize"", ""Metahash"") VALUES (@Path,@Size,@Hash,@Metasize,@Metahash)";
public Filelist(IDbConnection connection, string volumename)
: base(connection, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND)
public Filelist(IDbConnection connection, string volumename, IDbTransaction? tr)
: base(connection, tr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND)
{
}
@@ -304,8 +304,8 @@ namespace Duplicati.Library.Main.Database
private const string TABLE_FORMAT = @"(""Name"" TEXT NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)";
private const string INSERT_COMMAND = @"(""Name"", ""Hash"", ""Size"") VALUES (@Name,@Hash,@Size)";
public Indexlist(IDbConnection connection, string volumename)
: base(connection, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND)
public Indexlist(IDbConnection connection, string volumename, IDbTransaction? tr)
: base(connection, tr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND)
{
}
@@ -365,8 +365,8 @@ namespace Duplicati.Library.Main.Database
private const string TABLE_FORMAT = @"(""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)";
private const string INSERT_COMMAND = @"(""Hash"", ""Size"") VALUES (@Hash,@Size)";
public Blocklist(IDbConnection connection, string volumename)
: base(connection, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND)
public Blocklist(IDbConnection connection, string volumename, IDbTransaction? tr)
: base(connection, tr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND)
{ }
public void AddBlock(string hash, long size)
@@ -414,19 +414,19 @@ namespace Duplicati.Library.Main.Database
}
}
public IFilelist CreateFilelist(string name)
public IFilelist CreateFilelist(string name, IDbTransaction? tr)
{
return new Filelist(m_connection, name);
return new Filelist(m_connection, name, tr);
}
public IIndexlist CreateIndexlist(string name)
public IIndexlist CreateIndexlist(string name, IDbTransaction? tr)
{
return new Indexlist(m_connection, name);
return new Indexlist(m_connection, name, tr);
}
public IBlocklist CreateBlocklist(string name)
public IBlocklist CreateBlocklist(string name, IDbTransaction? tr)
{
return new Blocklist(m_connection, name);
return new Blocklist(m_connection, name, tr);
}
}
}
Duplicati/Library/Main/Operation/BackupHandler.cs
+1 -1
View File
@@ -532,7 +532,7 @@ namespace Duplicati.Library.Main.Operation
using (var testdb = new LocalTestDatabase(m_database))
await new TestHandler(m_options, (TestResults)m_result.TestResults)
.DoRunAsync(samplesToTest, testdb, backendManager)
.DoRunAsync(samplesToTest, testdb, rtr, backendManager)
.ConfigureAwait(false);
}
}
Duplicati/Library/Main/Operation/RepairHandler.cs
+4 -1
View File
@@ -185,6 +185,7 @@ namespace Duplicati.Library.Main.Operation
if (tp.VerificationRequiredVolumes.Any())
{
using (var testdb = new LocalTestDatabase(db))
using (var rtr = new ReusableTransaction(testdb))
{
foreach (var n in tp.VerificationRequiredVolumes)
try
@@ -201,7 +202,7 @@ namespace Duplicati.Library.Main.Operation
KeyValuePair<string, IEnumerable<KeyValuePair<Duplicati.Library.Interface.TestEntryStatus, string>>> res;
(var tf, var hash, var size) = await backendManager.GetWithInfoAsync(n.Name, n.Hash, n.Size, cancellationToken).ConfigureAwait(false);
using (tf)
res = TestHandler.TestVolumeInternals(testdb, n, tf, m_options, 1);
res = TestHandler.TestVolumeInternals(testdb, rtr, n, tf, m_options, 1);
if (res.Value.Any())
throw new Exception(string.Format("Remote verification failure: {0}", res.Value.First()));
@@ -219,6 +220,8 @@ namespace Duplicati.Library.Main.Operation
if (ex.IsAbortException())
throw;
}
rtr.Commit("CommitVerificationTransaction", false);
}
}
Duplicati/Library/Main/Operation/TestHandler.cs
+18 -16
View File
@@ -52,18 +52,20 @@ namespace Duplicati.Library.Main.Operation
throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist");
using (var db = new LocalTestDatabase(m_options.Dbpath))
using (var rtr = new ReusableTransaction(db))
{
Utility.UpdateOptionsFromDb(db, m_options);
Utility.VerifyOptionsAndUpdateDatabase(db, m_options);
db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks, null);
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_results.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, null).ConfigureAwait(false);
await DoRunAsync(samples, db, backendManager).ConfigureAwait(false);
await DoRunAsync(samples, db, rtr, backendManager).ConfigureAwait(false);
rtr.Commit("TestHandlerComplete");
}
}
public async Task DoRunAsync(long samples, LocalTestDatabase db, IBackendManager backend)
public async Task DoRunAsync(long samples, LocalTestDatabase db, ReusableTransaction rtr, IBackendManager backend)
{
var files = db.SelectTestTargets(samples, m_options).ToList();
var files = db.SelectTestTargets(samples, m_options, rtr.Transaction).ToList();
m_results.OperationProgressUpdater.UpdatePhase(OperationPhase.Verify_Running);
m_results.OperationProgressUpdater.UpdateProgress(0);
@@ -78,7 +80,7 @@ namespace Duplicati.Library.Main.Operation
{
if (!await m_results.TaskControl.ProgressRendevouz().ConfigureAwait(false))
{
await backend.WaitForEmptyAsync(db, null, m_results.TaskControl.ProgressToken).ConfigureAwait(false);
await backend.WaitForEmptyAsync(db, rtr.Transaction, m_results.TaskControl.ProgressToken).ConfigureAwait(false);
m_results.EndTime = DateTime.UtcNow;
return;
}
@@ -88,14 +90,14 @@ namespace Duplicati.Library.Main.Operation
KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>> res;
using (tf)
res = TestVolumeInternals(db, vol, tf, m_options, m_options.FullBlockVerification ? 1.0 : 0.2);
res = TestVolumeInternals(db, rtr, vol, tf, m_options, m_options.FullBlockVerification ? 1.0 : 0.2);
m_results.AddResult(res.Key, res.Value);
if (!string.IsNullOrWhiteSpace(vol.Hash) && vol.Size > 0)
{
if (res.Value == null || !res.Value.Any())
{
var rv = db.GetRemoteVolume(vol.Name, null);
var rv = db.GetRemoteVolume(vol.Name, rtr.Transaction);
if (rv.ID < 0)
{
@@ -108,14 +110,14 @@ namespace Duplicati.Library.Main.Operation
else
{
Logging.Log.WriteInformationMessage(LOGTAG, "CaptureHashAndSize", "Successfully captured hash and size for {0}, updating database", vol.Name);
db.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Verified, vol.Size, vol.Hash);
db.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Verified, vol.Size, vol.Hash, rtr.Transaction);
}
}
}
}
}
db.UpdateVerificationCount(vol.Name);
db.UpdateVerificationCount(vol.Name, rtr.Transaction);
}
catch (Exception ex)
{
@@ -152,7 +154,7 @@ namespace Duplicati.Library.Main.Operation
(var tf, var hash, var size) = await backend.GetWithInfoAsync(f.Name, f.Hash, f.Size, m_results.TaskControl.ProgressToken).ConfigureAwait(false);
using (tf)
res = TestVolumeInternals(db, f, tf, m_options, 1);
res = TestVolumeInternals(db, rtr, f, tf, m_options, 1);
m_results.AddResult(res.Key, res.Value);
if (!string.IsNullOrWhiteSpace(hash) && size > 0)
@@ -166,7 +168,7 @@ namespace Duplicati.Library.Main.Operation
else
{
Logging.Log.WriteInformationMessage(LOGTAG, "CapturedHashAndSize", "Successfully captured hash and size for {0}, updating database", f.Name);
db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Verified, size, hash);
db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Verified, size, hash, rtr.Transaction);
}
}
}
@@ -177,7 +179,7 @@ namespace Duplicati.Library.Main.Operation
{ }
}
db.UpdateVerificationCount(f.Name);
db.UpdateVerificationCount(f.Name, rtr.Transaction);
m_results.AddResult(f.Name, []);
}
catch (Exception ex)
@@ -213,7 +215,7 @@ namespace Duplicati.Library.Main.Operation
/// <param name="vol">The remote volume being examined</param>
/// <param name="tf">The path to the downloaded copy of the file</param>
/// <param name="sample_percent">A value between 0 and 1 that indicates how many blocks are tested in a dblock file</param>
public static KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>> TestVolumeInternals(LocalTestDatabase db, IRemoteVolume vol, string tf, Options options, double sample_percent)
public static KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>> TestVolumeInternals(LocalTestDatabase db, ReusableTransaction rtr, IRemoteVolume vol, string tf, Options options, double sample_percent)
{
var hashsize = HashFactory.HashSizeBytes(options.BlockHashAlgorithm);
var parsedInfo = Volumes.VolumeBase.ParseFilename(vol.Name);
@@ -224,7 +226,7 @@ namespace Duplicati.Library.Main.Operation
case RemoteVolumeType.Files:
//Compare with db and see if all files are accounted for
// with correct file hashes and blocklist hashes
using (var fl = db.CreateFilelist(vol.Name))
using (var fl = db.CreateFilelist(vol.Name, rtr.Transaction))
{
using (var rd = new Volumes.FilesetVolumeReader(parsedInfo.CompressionModule, tf, options))
foreach (var f in rd.Files)
@@ -242,7 +244,7 @@ namespace Duplicati.Library.Main.Operation
foreach (var v in rd.Volumes)
{
blocklinks.Add(new Tuple<string, string, long>(v.Filename, v.Hash, v.Length));
using (var bl = db.CreateBlocklist(v.Filename))
using (var bl = db.CreateBlocklist(v.Filename, rtr.Transaction))
{
foreach (var h in v.Blocks)
bl.AddBlock(h.Key, h.Value);
@@ -251,7 +253,7 @@ namespace Duplicati.Library.Main.Operation
}
}
using (var il = db.CreateIndexlist(vol.Name))
using (var il = db.CreateIndexlist(vol.Name, rtr.Transaction))
{
foreach (var t in blocklinks)
il.AddBlockLink(t.Item1, t.Item2, t.Item3);
@@ -262,7 +264,7 @@ namespace Duplicati.Library.Main.Operation
return new KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>(vol.Name, combined.ToList());
case RemoteVolumeType.Blocks:
using (var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm))
using (var bl = db.CreateBlocklist(vol.Name))
using (var bl = db.CreateBlocklist(vol.Name, rtr.Transaction))
using (var rd = new Volumes.BlockVolumeReader(parsedInfo.CompressionModule, tf, options))
{
//Verify that all blocks are in the file