From 47739146396b6cc09ecbd3dfd337651ddf618871 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 14 Apr 2025 12:05:10 +0200 Subject: [PATCH] Rewrote testing to use an externally supplied transaction, to avoid temporary transactions from clearing the active transaction. This is needed because SQLite does not support multiple active transactions. --- .../Main/Database/LocalTestDatabase.cs | 38 +++++++++---------- .../Library/Main/Operation/BackupHandler.cs | 2 +- .../Library/Main/Operation/RepairHandler.cs | 5 ++- .../Library/Main/Operation/TestHandler.cs | 34 +++++++++-------- 4 files changed, 42 insertions(+), 37 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index 10f8caa6d..86d32378d 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -40,9 +40,9 @@ namespace Duplicati.Library.Main.Database { } - public void UpdateVerificationCount(string name) + public void UpdateVerificationCount(string name, IDbTransaction? tr) { - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(tr)) cmd.SetCommandAndParameters(@"UPDATE ""RemoteVolume"" SET ""VerificationCount"" = MAX(1, CASE WHEN ""VerificationCount"" <= 0 THEN (SELECT MAX(""VerificationCount"") FROM ""RemoteVolume"") ELSE ""VerificationCount"" + 1 END) WHERE ""Name"" = @Name") .SetParameterValue("@Name", name) .ExecuteNonQuery(); @@ -118,12 +118,12 @@ namespace Duplicati.Library.Main.Database return res; } - public IEnumerable SelectTestTargets(long samples, Options options) + public IEnumerable SelectTestTargets(long samples, Options options, IDbTransaction? tr) { var tp = GetFilelistWhereClause(options.Time, options.Version); samples = Math.Max(1, samples); - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(tr)) { // Select any broken items cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"" WHERE (""State"" IN (@States)) AND (""Hash"" = '' OR ""Hash"" IS NULL OR ""Size"" <= 0) ") @@ -194,14 +194,14 @@ namespace Duplicati.Library.Main.Database protected readonly IDbConnection m_connection; protected readonly string m_volumename; protected string m_tablename; - protected IDbTransaction m_transaction; + protected IDbTransaction? m_transaction; protected IDbCommand m_insertCommand; - protected Basiclist(IDbConnection connection, string volumename, string tablePrefix, string tableFormat, string insertCommand) + protected Basiclist(IDbConnection connection, IDbTransaction? tr, string volumename, string tablePrefix, string tableFormat, string insertCommand) { m_connection = connection; m_volumename = volumename; - m_transaction = m_connection.BeginTransaction(); + m_transaction = tr; var tablename = tablePrefix + "-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); using (var cmd = m_connection.CreateCommand(m_transaction)) @@ -240,8 +240,8 @@ namespace Duplicati.Library.Main.Database private const string TABLE_PREFIX = "Filelist"; private const string TABLE_FORMAT = @"(""Path"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Hash"" TEXT NULL, ""Metasize"" INTEGER NOT NULL, ""Metahash"" TEXT NOT NULL)"; private const string INSERT_COMMAND = @"(""Path"", ""Size"", ""Hash"", ""Metasize"", ""Metahash"") VALUES (@Path,@Size,@Hash,@Metasize,@Metahash)"; - public Filelist(IDbConnection connection, string volumename) - : base(connection, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) + public Filelist(IDbConnection connection, string volumename, IDbTransaction? tr) + : base(connection, tr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) { } @@ -304,8 +304,8 @@ namespace Duplicati.Library.Main.Database private const string TABLE_FORMAT = @"(""Name"" TEXT NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)"; private const string INSERT_COMMAND = @"(""Name"", ""Hash"", ""Size"") VALUES (@Name,@Hash,@Size)"; - public Indexlist(IDbConnection connection, string volumename) - : base(connection, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) + public Indexlist(IDbConnection connection, string volumename, IDbTransaction? tr) + : base(connection, tr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) { } @@ -365,8 +365,8 @@ namespace Duplicati.Library.Main.Database private const string TABLE_FORMAT = @"(""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)"; private const string INSERT_COMMAND = @"(""Hash"", ""Size"") VALUES (@Hash,@Size)"; - public Blocklist(IDbConnection connection, string volumename) - : base(connection, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) + public Blocklist(IDbConnection connection, string volumename, IDbTransaction? tr) + : base(connection, tr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) { } public void AddBlock(string hash, long size) @@ -414,19 +414,19 @@ namespace Duplicati.Library.Main.Database } } - public IFilelist CreateFilelist(string name) + public IFilelist CreateFilelist(string name, IDbTransaction? tr) { - return new Filelist(m_connection, name); + return new Filelist(m_connection, name, tr); } - public IIndexlist CreateIndexlist(string name) + public IIndexlist CreateIndexlist(string name, IDbTransaction? tr) { - return new Indexlist(m_connection, name); + return new Indexlist(m_connection, name, tr); } - public IBlocklist CreateBlocklist(string name) + public IBlocklist CreateBlocklist(string name, IDbTransaction? tr) { - return new Blocklist(m_connection, name); + return new Blocklist(m_connection, name, tr); } } } diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 518d8e5d8..226b265b8 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -532,7 +532,7 @@ namespace Duplicati.Library.Main.Operation using (var testdb = new LocalTestDatabase(m_database)) await new TestHandler(m_options, (TestResults)m_result.TestResults) - .DoRunAsync(samplesToTest, testdb, backendManager) + .DoRunAsync(samplesToTest, testdb, rtr, backendManager) .ConfigureAwait(false); } } diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index ee2edb257..d28cbd44e 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -185,6 +185,7 @@ namespace Duplicati.Library.Main.Operation if (tp.VerificationRequiredVolumes.Any()) { using (var testdb = new LocalTestDatabase(db)) + using (var rtr = new ReusableTransaction(testdb)) { foreach (var n in tp.VerificationRequiredVolumes) try @@ -201,7 +202,7 @@ namespace Duplicati.Library.Main.Operation KeyValuePair>> res; (var tf, var hash, var size) = await backendManager.GetWithInfoAsync(n.Name, n.Hash, n.Size, cancellationToken).ConfigureAwait(false); using (tf) - res = TestHandler.TestVolumeInternals(testdb, n, tf, m_options, 1); + res = TestHandler.TestVolumeInternals(testdb, rtr, n, tf, m_options, 1); if (res.Value.Any()) throw new Exception(string.Format("Remote verification failure: {0}", res.Value.First())); @@ -219,6 +220,8 @@ namespace Duplicati.Library.Main.Operation if (ex.IsAbortException()) throw; } + + rtr.Commit("CommitVerificationTransaction", false); } } diff --git a/Duplicati/Library/Main/Operation/TestHandler.cs b/Duplicati/Library/Main/Operation/TestHandler.cs index b50bc7917..03f77acc0 100644 --- a/Duplicati/Library/Main/Operation/TestHandler.cs +++ b/Duplicati/Library/Main/Operation/TestHandler.cs @@ -52,18 +52,20 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); using (var db = new LocalTestDatabase(m_options.Dbpath)) + using (var rtr = new ReusableTransaction(db)) { Utility.UpdateOptionsFromDb(db, m_options); Utility.VerifyOptionsAndUpdateDatabase(db, m_options); db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks, null); await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_results.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, null).ConfigureAwait(false); - await DoRunAsync(samples, db, backendManager).ConfigureAwait(false); + await DoRunAsync(samples, db, rtr, backendManager).ConfigureAwait(false); + rtr.Commit("TestHandlerComplete"); } } - public async Task DoRunAsync(long samples, LocalTestDatabase db, IBackendManager backend) + public async Task DoRunAsync(long samples, LocalTestDatabase db, ReusableTransaction rtr, IBackendManager backend) { - var files = db.SelectTestTargets(samples, m_options).ToList(); + var files = db.SelectTestTargets(samples, m_options, rtr.Transaction).ToList(); m_results.OperationProgressUpdater.UpdatePhase(OperationPhase.Verify_Running); m_results.OperationProgressUpdater.UpdateProgress(0); @@ -78,7 +80,7 @@ namespace Duplicati.Library.Main.Operation { if (!await m_results.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backend.WaitForEmptyAsync(db, null, m_results.TaskControl.ProgressToken).ConfigureAwait(false); + await backend.WaitForEmptyAsync(db, rtr.Transaction, m_results.TaskControl.ProgressToken).ConfigureAwait(false); m_results.EndTime = DateTime.UtcNow; return; } @@ -88,14 +90,14 @@ namespace Duplicati.Library.Main.Operation KeyValuePair>> res; using (tf) - res = TestVolumeInternals(db, vol, tf, m_options, m_options.FullBlockVerification ? 1.0 : 0.2); + res = TestVolumeInternals(db, rtr, vol, tf, m_options, m_options.FullBlockVerification ? 1.0 : 0.2); m_results.AddResult(res.Key, res.Value); if (!string.IsNullOrWhiteSpace(vol.Hash) && vol.Size > 0) { if (res.Value == null || !res.Value.Any()) { - var rv = db.GetRemoteVolume(vol.Name, null); + var rv = db.GetRemoteVolume(vol.Name, rtr.Transaction); if (rv.ID < 0) { @@ -108,14 +110,14 @@ namespace Duplicati.Library.Main.Operation else { Logging.Log.WriteInformationMessage(LOGTAG, "CaptureHashAndSize", "Successfully captured hash and size for {0}, updating database", vol.Name); - db.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Verified, vol.Size, vol.Hash); + db.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Verified, vol.Size, vol.Hash, rtr.Transaction); } } } } } - db.UpdateVerificationCount(vol.Name); + db.UpdateVerificationCount(vol.Name, rtr.Transaction); } catch (Exception ex) { @@ -152,7 +154,7 @@ namespace Duplicati.Library.Main.Operation (var tf, var hash, var size) = await backend.GetWithInfoAsync(f.Name, f.Hash, f.Size, m_results.TaskControl.ProgressToken).ConfigureAwait(false); using (tf) - res = TestVolumeInternals(db, f, tf, m_options, 1); + res = TestVolumeInternals(db, rtr, f, tf, m_options, 1); m_results.AddResult(res.Key, res.Value); if (!string.IsNullOrWhiteSpace(hash) && size > 0) @@ -166,7 +168,7 @@ namespace Duplicati.Library.Main.Operation else { Logging.Log.WriteInformationMessage(LOGTAG, "CapturedHashAndSize", "Successfully captured hash and size for {0}, updating database", f.Name); - db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Verified, size, hash); + db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Verified, size, hash, rtr.Transaction); } } } @@ -177,7 +179,7 @@ namespace Duplicati.Library.Main.Operation { } } - db.UpdateVerificationCount(f.Name); + db.UpdateVerificationCount(f.Name, rtr.Transaction); m_results.AddResult(f.Name, []); } catch (Exception ex) @@ -213,7 +215,7 @@ namespace Duplicati.Library.Main.Operation /// The remote volume being examined /// The path to the downloaded copy of the file /// A value between 0 and 1 that indicates how many blocks are tested in a dblock file - public static KeyValuePair>> TestVolumeInternals(LocalTestDatabase db, IRemoteVolume vol, string tf, Options options, double sample_percent) + public static KeyValuePair>> TestVolumeInternals(LocalTestDatabase db, ReusableTransaction rtr, IRemoteVolume vol, string tf, Options options, double sample_percent) { var hashsize = HashFactory.HashSizeBytes(options.BlockHashAlgorithm); var parsedInfo = Volumes.VolumeBase.ParseFilename(vol.Name); @@ -224,7 +226,7 @@ namespace Duplicati.Library.Main.Operation case RemoteVolumeType.Files: //Compare with db and see if all files are accounted for // with correct file hashes and blocklist hashes - using (var fl = db.CreateFilelist(vol.Name)) + using (var fl = db.CreateFilelist(vol.Name, rtr.Transaction)) { using (var rd = new Volumes.FilesetVolumeReader(parsedInfo.CompressionModule, tf, options)) foreach (var f in rd.Files) @@ -242,7 +244,7 @@ namespace Duplicati.Library.Main.Operation foreach (var v in rd.Volumes) { blocklinks.Add(new Tuple(v.Filename, v.Hash, v.Length)); - using (var bl = db.CreateBlocklist(v.Filename)) + using (var bl = db.CreateBlocklist(v.Filename, rtr.Transaction)) { foreach (var h in v.Blocks) bl.AddBlock(h.Key, h.Value); @@ -251,7 +253,7 @@ namespace Duplicati.Library.Main.Operation } } - using (var il = db.CreateIndexlist(vol.Name)) + using (var il = db.CreateIndexlist(vol.Name, rtr.Transaction)) { foreach (var t in blocklinks) il.AddBlockLink(t.Item1, t.Item2, t.Item3); @@ -262,7 +264,7 @@ namespace Duplicati.Library.Main.Operation return new KeyValuePair>>(vol.Name, combined.ToList()); case RemoteVolumeType.Blocks: using (var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm)) - using (var bl = db.CreateBlocklist(vol.Name)) + using (var bl = db.CreateBlocklist(vol.Name, rtr.Transaction)) using (var rd = new Volumes.BlockVolumeReader(parsedInfo.CompressionModule, tf, options)) { //Verify that all blocks are in the file