diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 595ca1c24..d5b33e86c 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -364,9 +364,9 @@ namespace Duplicati.Library.Main.Database return RemoteVolumeEntry.Empty; } - public IEnumerable> DuplicateRemoteVolumes() + public IEnumerable> DuplicateRemoteVolumes(IDbTransaction? transaction) { - foreach (var rd in m_selectduplicateRemoteVolumesCommand.ExecuteReaderEnumerable()) + foreach (var rd in m_selectduplicateRemoteVolumesCommand.SetTransaction(transaction).ExecuteReaderEnumerable()) { yield return new KeyValuePair( rd.ConvertValueToString(0) ?? throw new Exception("Name was null"), diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 153217795..147cd449a 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -344,12 +344,12 @@ namespace Duplicati.Library.Main.Operation { if (options.NoBackendverification) { - await FilelistProcessor.VerifyLocalList(backendManager, database, result.TaskControl.ProgressToken).ConfigureAwait(false); + await FilelistProcessor.VerifyLocalList(backendManager, database, null, result.TaskControl.ProgressToken).ConfigureAwait(false); await UpdateStorageStatsFromDatabase(result, database, options, backendManager, result.TaskControl.ProgressToken).ConfigureAwait(false); } else { - await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: false, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, options, database, null, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: false, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean).ConfigureAwait(false); } } catch (RemoteListVerificationException ex) @@ -369,7 +369,7 @@ namespace Duplicati.Library.Main.Operation database = new LocalBackupDatabase(options.Dbpath, options); Log.WriteInformationMessage(LOGTAG, "BackendCleanupFinished", "Backend cleanup finished, retrying verification"); - await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, options, database, null, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); } else throw; @@ -515,7 +515,7 @@ namespace Duplicati.Library.Main.Operation { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupVerify); using (new Logging.Timer(LOGTAG, "AfterBackupVerify", "AfterBackupVerify")) - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, m_result.BackendWriter, [currentFilelistVolume], [previousTemporaryFilelist], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, rtr.Transaction, m_result.BackendWriter, [currentFilelistVolume], [previousTemporaryFilelist], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); await backendManager.WaitForEmptyAsync(m_database, rtr.Transaction, m_taskReader.ProgressToken); // Calculate the number of samples to test, using the largest number of file of a given type diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index a1f1a99e5..cc92439e1 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -66,9 +66,9 @@ namespace Duplicati.Library.Main.Operation /// Throws an error if there are issues with the remote storage /// /// The database to compare with - public static async Task VerifyLocalList(IBackendManager backendManager, LocalDatabase database, CancellationToken cancellationToken) + public static async Task VerifyLocalList(IBackendManager backendManager, LocalDatabase database, IDbTransaction transaction, CancellationToken cancellationToken) { - var locallist = database.GetRemoteVolumes(); + var locallist = database.GetRemoteVolumes(transaction); foreach (var i in locallist) { switch (i.State) @@ -120,7 +120,7 @@ namespace Duplicati.Library.Main.Operation { LocalBackupDatabase backupDatabase = new LocalBackupDatabase(database, options); IEnumerable protectedFiles = backupDatabase.GetTemporaryFilelistVolumeNames(latestVolumesOnly, transaction); - await VerifyRemoteList(backend, options, database, backendWriter, protectedFiles, [], logErrors: true, verifyMode: verifyMode).ConfigureAwait(false); + await VerifyRemoteList(backend, options, database, transaction, backendWriter, protectedFiles, [], logErrors: true, verifyMode: verifyMode).ConfigureAwait(false); } } @@ -137,9 +137,9 @@ namespace Duplicati.Library.Main.Operation /// The mode to use for verification /// Disable the logging of errors to prevent spamming the log; exceptions will be thrown regardless /// An awaitable task - public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, bool logErrors, VerifyMode verifyMode) + public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IDbTransaction transaction, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, bool logErrors, VerifyMode verifyMode) { - var tp = await RemoteListAnalysis(backend, options, database, log, protectedFiles, strictExcemptFiles, verifyMode).ConfigureAwait(false); + var tp = await RemoteListAnalysis(backend, options, database, transaction, log, protectedFiles, strictExcemptFiles, verifyMode).ConfigureAwait(false); long extraCount = 0; long missingCount = 0; @@ -249,7 +249,7 @@ namespace Duplicati.Library.Main.Operation /// Filenames that should be exempted from strict verification /// The mode to use for verification /// The result of the analysis - public static async Task RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, VerifyMode verifyMode) + public static async Task RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IDbTransaction transaction, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, VerifyMode verifyMode) { // If the last operation completed, no cleanup should be required if (verifyMode == VerifyMode.VerifyAndClean && !database.TerminatedWithActiveUploads) @@ -303,15 +303,15 @@ namespace Duplicati.Library.Main.Operation var temporaryAndDeletedVolumes = new HashSet(); var missingUploadingVolumes = new HashSet(); - foreach (var e in database.DuplicateRemoteVolumes()) + foreach (var e in database.DuplicateRemoteVolumes(transaction)) { if (e.Value == RemoteVolumeState.Uploading || e.Value == RemoteVolumeState.Temporary) - database.UnlinkRemoteVolume(e.Key, e.Value); + database.UnlinkRemoteVolume(e.Key, e.Value, transaction); else throw new RemoteListVerificationException(string.Format("The remote volume {0} appears in the database with state {1} and a deleted state, cannot continue", e.Key, e.Value.ToString()), "AmbiguousStateRemoteFiles"); } - var locallist = database.GetRemoteVolumes(); + var locallist = database.GetRemoteVolumes(transaction); foreach (var i in locallist) { var remoteFound = lookup.TryGetValue(i.Name, out var r); @@ -376,14 +376,14 @@ namespace Duplicati.Library.Main.Operation if (remoteFound && correctSize && r.File.Size >= 0) { Logging.Log.WriteInformationMessage(LOGTAG, "PromotingCompleteFile", "Promoting uploaded complete file from {0} to {2}: {1}", i.State, i.Name, RemoteVolumeState.Uploaded); - database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash); + database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash, transaction); } else if (!remoteFound) { if (protectedFiles.Any(pf => pf == i.Name)) { Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "Keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name); - database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null, null); + database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null, transaction); } else if (verifyMode == VerifyMode.VerifyStrict && !strictExcemptFiles.Any(pf => pf == i.Name)) { @@ -394,7 +394,7 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "SchedulingMissingFileForDelete", "Scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name); missingUploadingVolumes.Add(i.Name); - database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null, null); + database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null, transaction); } } else @@ -431,7 +431,7 @@ namespace Duplicati.Library.Main.Operation if (!remoteFound) missing.Add(i); else if (correctSize) - database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash); + database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash, transaction); else missingHash.Add(new Tuple(r.File.Size, i)); @@ -458,7 +458,7 @@ namespace Duplicati.Library.Main.Operation throw new RemoteListVerificationException($"The remote volumes {string.Join(", ", missingUploadingVolumes)} are supposed to be deleted, but strict mode is on. Try running the \"repair\" command", "DeleteDuringStrictMode"); else if (verifyMode == VerifyMode.VerifyAndClean || verifyMode == VerifyMode.VerifyStrict) { - database.RemoveRemoteVolumes(missingUploadingVolumes.Concat(temporaryAndDeletedVolumes), null); + database.RemoveRemoteVolumes(missingUploadingVolumes.Concat(temporaryAndDeletedVolumes), transaction); // Clear the flag after we have cleaned up if (!options.Dryrun) database.TerminatedWithActiveUploads = false; diff --git a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs index 7e82c7754..ea713941a 100644 --- a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs @@ -67,7 +67,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsInDatabase", "No broken filesets found in database, checking for missing remote files"); - var remotestate = await FilelistProcessor.RemoteListAnalysis(backendManager, options, db, result.BackendWriter, null, null, FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false); + var remotestate = await FilelistProcessor.RemoteListAnalysis(backendManager, options, db, transaction, result.BackendWriter, null, null, FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false); if (!remotestate.ParsedVolumes.Any()) throw new UserInformationException("No remote volumes were found, refusing purge", "CannotPurgeWithNoRemoteVolumes"); diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index cc873661e..236496e7d 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -90,9 +90,9 @@ namespace Duplicati.Library.Main.Operation db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, false, null); if (m_options.NoBackendverification) - await FilelistProcessor.VerifyLocalList(backendManager, db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await FilelistProcessor.VerifyLocalList(backendManager, db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); else - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, null, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, null, m_result.BackendWriter, null, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); } var filesets = db.FilesetTimes.OrderByDescending(x => x.Value).ToArray(); diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 3a967e007..44ecfe176 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -141,7 +141,7 @@ namespace Duplicati.Library.Main.Operation // If the last backup failed, guard the incomplete fileset, so we can create a synthetic filelist var lastTempFilelist = db.GetLastIncompleteFilesetVolume(null); - var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false); + var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, null, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false); var buffer = new byte[m_options.Blocksize]; var hashsize = HashFactory.HashSizeBytes(m_options.BlockHashAlgorithm); diff --git a/Duplicati/Library/Main/Operation/TestHandler.cs b/Duplicati/Library/Main/Operation/TestHandler.cs index 20aad1250..7f815d4fd 100644 --- a/Duplicati/Library/Main/Operation/TestHandler.cs +++ b/Duplicati/Library/Main/Operation/TestHandler.cs @@ -25,7 +25,6 @@ using System.Linq; using System.Collections.Generic; using Duplicati.Library.Interface; using Duplicati.Library.Utility; -using System.Threading; using System.Threading.Tasks; namespace Duplicati.Library.Main.Operation