diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index ade62473d..96d311f64 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -71,7 +71,9 @@ namespace Duplicati.Library.Main.Database /// /// Asynchronously retrieves the sizes of the file versions. /// - IAsyncEnumerable Sizes(); + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of file sizes. + IAsyncEnumerable Sizes(CancellationToken token); } /// @@ -301,12 +303,13 @@ namespace Duplicati.Library.Main.Database More = true; } - public async IAsyncEnumerable Sizes() + /// + public async IAsyncEnumerable Sizes([EnumeratorCancellation] CancellationToken token) { while (More && Path == m_reader.ConvertValueToString(0)) { yield return m_reader.ConvertValueToInt64(1, -1); - More = await m_reader.ReadAsync().ConfigureAwait(false); + More = await m_reader.ReadAsync(token).ConfigureAwait(false); } } } @@ -316,8 +319,10 @@ namespace Duplicati.Library.Main.Database /// private class FileversionFixed : IFileversion { + /// public string? Path { get; internal set; } - public IAsyncEnumerable Sizes() => AsyncEnumerable.Empty(); + /// + public IAsyncEnumerable Sizes(CancellationToken token) => AsyncEnumerable.Empty(); } /// diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index b00e3207a..ab5cbf7cf 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -214,7 +214,7 @@ namespace Duplicati.Library.Main.Operation // all of the local database classes handle this), as the constructor // is not async and shouldn't be executing "heavy" code, such as // database queries. - var journalData = m_database.GetChangeJournalData(lastfilesetid).ToEnumerable(); + var journalData = m_database.GetChangeJournalData(lastfilesetid, m_taskReader.ProgressToken).ToEnumerable(); var service = new UsnJournalService(fileProvider.SnapshotService, filter, m_options.FileAttributeFilter, m_options.SkipFilesLargerThan, journalData, cancellationTokenSource.Token); @@ -300,9 +300,9 @@ namespace Duplicati.Library.Main.Operation { try { - database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options) + database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options, result.TaskControl.ProgressToken) .ConfigureAwait(false); - var lastFilestTime = await database.FilesetTimes() + var lastFilestTime = await database.FilesetTimes(result.TaskControl.ProgressToken) .Select(x => x.Value) .Append(DateTime.UnixEpoch) .Select(Library.Utility.Utility.NormalizeDateTimeToEpochSeconds) @@ -319,40 +319,40 @@ namespace Duplicati.Library.Main.Operation Log.WriteVerboseMessage(LOGTAG, "DatabaseTimestampError", "The database has a timestamp of {0}, but the last fileset has a timestamp of {1}. Sleeping for {2} seconds to allow the clock to catch up.", database.OperationTimestamp.ToLocalTime(), Library.Utility.Utility.EPOCH.AddSeconds(lastFilestTime).ToLocalTime(), sleepSeconds); Thread.Sleep(TimeSpan.FromSeconds(sleepSeconds)); database.Dispose(); - database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options) + database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options, result.TaskControl.ProgressToken) .ConfigureAwait(false); } result.Dryrun = options.Dryrun; // Check the database integrity - await Utility.UpdateOptionsFromDb(database, options) + await Utility.UpdateOptionsFromDb(database, options, result.TaskControl.ProgressToken) .ConfigureAwait(false); - await Utility.VerifyOptionsAndUpdateDatabase(database, options) + await Utility.VerifyOptionsAndUpdateDatabase(database, options, result.TaskControl.ProgressToken) .ConfigureAwait(false); var probe_path = await database - .GetFirstPath() + .GetFirstPath(result.TaskControl.ProgressToken) .ConfigureAwait(false); if (probe_path != null && Util.GuessDirSeparator(probe_path) != Util.DirectorySeparatorString) throw new UserInformationException(string.Format("The backup contains files that belong to another operating system. Proceeding with a backup would cause the database to contain paths from two different operation systems, which is not supported. To proceed without losing remote data, delete all filesets and make sure the --{0} option is set, then run the backup again to re-use the existing data on the remote store.", "no-auto-compact"), "CrossOsDatabaseReuseNotSupported"); - if (await database.PartiallyRecreated().ConfigureAwait(false)) + if (await database.PartiallyRecreated(result.TaskControl.ProgressToken).ConfigureAwait(false)) throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated"); - if (await database.RepairInProgress().ConfigureAwait(false)) + if (await database.RepairInProgress(result.TaskControl.ProgressToken).ConfigureAwait(false)) throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the backup process cannot continue. You may delete the local database and attempt to repair it again.", "DatabaseRepairInProgress"); // Make sure the database is sane await database - .VerifyConsistency(options.Blocksize, options.BlockhashSize, !options.DisableFilelistConsistencyChecks) + .VerifyConsistency(options.Blocksize, options.BlockhashSize, !options.DisableFilelistConsistencyChecks, result.TaskControl.ProgressToken) .ConfigureAwait(false); // Guard the last filelist if (!options.DisableSyntheticFilelist) lastTempFilelist = await database - .GetLastIncompleteFilesetVolume() + .GetLastIncompleteFilesetVolume(result.TaskControl.ProgressToken) .ConfigureAwait(false); try @@ -364,8 +364,8 @@ namespace Duplicati.Library.Main.Operation } else { - await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: false, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean).ConfigureAwait(false); - var updatedLastTemp = string.IsNullOrWhiteSpace(lastTempFilelist.Name) ? default : await database.GetRemoteVolume(lastTempFilelist.Name).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: false, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean, result.TaskControl.ProgressToken).ConfigureAwait(false); + var updatedLastTemp = string.IsNullOrWhiteSpace(lastTempFilelist.Name) ? default : await database.GetRemoteVolume(lastTempFilelist.Name, result.TaskControl.ProgressToken).ConfigureAwait(false); if (!string.IsNullOrWhiteSpace(updatedLastTemp.Name) && updatedLastTemp.State == RemoteVolumeState.Deleting) { // The last temporary filelist was emptied, and scheduled for deletion, so we need to delete it @@ -381,7 +381,7 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(database, result.TaskControl.ProgressToken).ConfigureAwait(false); // In case the backend delete failed, manually set the state to deleted await database - .UpdateRemoteVolume(updatedLastTemp.Name, RemoteVolumeState.Deleted, updatedLastTemp.Size, updatedLastTemp.Hash, false, TimeSpan.FromHours(2), null) + .UpdateRemoteVolume(updatedLastTemp.Name, RemoteVolumeState.Deleted, updatedLastTemp.Size, updatedLastTemp.Hash, false, TimeSpan.FromHours(2), null, result.TaskControl.ProgressToken) .ConfigureAwait(false); // The last temporary filelist is no more lastTempFilelist = default; @@ -402,18 +402,18 @@ namespace Duplicati.Library.Main.Operation .ConfigureAwait(false); // Re-open the database and backend manager - database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options) + database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options, result.TaskControl.ProgressToken) .ConfigureAwait(false); Log.WriteInformationMessage(LOGTAG, "BackendCleanupFinished", "Backend cleanup finished, retrying verification"); - await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, result.TaskControl.ProgressToken).ConfigureAwait(false); } else throw; } // Return the updated version of the fileset, in case it was actually complete - return new PreBackupVerifyResult(database, await database.GetLastIncompleteFilesetVolume().ConfigureAwait(false)); + return new PreBackupVerifyResult(database, await database.GetLastIncompleteFilesetVolume(result.TaskControl.ProgressToken).ConfigureAwait(false)); } catch { @@ -540,7 +540,7 @@ namespace Duplicati.Library.Main.Operation { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Delete); m_result.DeleteResults = new DeleteResults(m_result); - await using var db = await LocalDeleteDatabase.CreateAsync(m_database).ConfigureAwait(false); + await using var db = await LocalDeleteDatabase.CreateAsync(m_database, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await new DeleteHandler(m_options, (DeleteResults)m_result.DeleteResults) .DoRunAsync(db, true, currentIsSmall, backendManager) .ConfigureAwait(false); @@ -550,7 +550,7 @@ namespace Duplicati.Library.Main.Operation { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Compact); m_result.CompactResults = new CompactResults(m_result); - await using var db = await LocalDeleteDatabase.CreateAsync(m_database).ConfigureAwait(false); + await using var db = await LocalDeleteDatabase.CreateAsync(m_database, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await new CompactHandler(m_options, (CompactResults)m_result.CompactResults) .DoCompactAsync(db, true, backendManager) .ConfigureAwait(false); @@ -561,7 +561,7 @@ namespace Duplicati.Library.Main.Operation { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupVerify); using (new Logging.Timer(LOGTAG, "AfterBackupVerify", "AfterBackupVerify")) - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, m_result.BackendWriter, [currentFilelistVolume], [previousTemporaryFilelist], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, m_result.BackendWriter, [currentFilelistVolume], [previousTemporaryFilelist], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await backendManager .WaitForEmptyAsync(m_database, m_taskReader.ProgressToken) @@ -569,7 +569,7 @@ namespace Duplicati.Library.Main.Operation // Calculate the number of samples to test, using the largest number of file of a given type long remoteVolumeCount = await m_database - .GetRemoteVolumes() + .GetRemoteVolumes(m_result.TaskControl.ProgressToken) .Where(x => x.State == RemoteVolumeState.Verified) .GroupBy(x => x.Type) .SelectAwait(async x => await x.LongCountAsync()) @@ -582,7 +582,7 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupTest); m_result.TestResults = new TestResults(m_result); - await using var testdb = await LocalTestDatabase.CreateAsync(m_database).ConfigureAwait(false); + await using var testdb = await LocalTestDatabase.CreateAsync(m_database, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await new TestHandler(m_options, (TestResults)m_result.TestResults) .DoRunAsync(samplesToTest, testdb, backendManager) .ConfigureAwait(false); @@ -597,33 +597,33 @@ namespace Duplicati.Library.Main.Operation if (result.BackendWriter != null) { result.BackendWriter.KnownFileCount = await database - .GetRemoteVolumes() - .CountAsync() + .GetRemoteVolumes(cancelToken) + .CountAsync(cancelToken) .ConfigureAwait(false); result.BackendWriter.KnownFilesets = await database - .GetRemoteVolumes() + .GetRemoteVolumes(cancelToken) .CountAsync(x => x.Type == RemoteVolumeType.Files) .ConfigureAwait(false); result.BackendWriter.KnownFileSize = await database - .GetRemoteVolumes() + .GetRemoteVolumes(cancelToken) .Select(x => Math.Max(0, x.Size)) - .SumAsync() + .SumAsync(cancelToken) .ConfigureAwait(false); result.BackendWriter.UnknownFileCount = 0; result.BackendWriter.UnknownFileSize = 0; result.BackendWriter.BackupListCount = await database - .FilesetTimes() - .CountAsync() + .FilesetTimes(cancelToken) + .CountAsync(cancelToken) .ConfigureAwait(false); result.BackendWriter.LastBackupDate = ( await database - .FilesetTimes() - .FirstOrDefaultAsync() + .FilesetTimes(cancelToken) + .FirstOrDefaultAsync(cancelToken) .ConfigureAwait(false) ) .Value.ToLocalTime(); @@ -711,7 +711,7 @@ namespace Duplicati.Library.Main.Operation // If we crash now, there is a chance we leave behind partial remote files if (!m_options.Dryrun) await database - .TerminatedWithActiveUploads(true) + .TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, true) .ConfigureAwait(false); await using (m_database = database) @@ -731,7 +731,7 @@ namespace Duplicati.Library.Main.Operation // Grab the previous backup ID, if any var prevfileset = await m_database - .FilesetTimes() + .FilesetTimes(m_result.TaskControl.ProgressToken) .FirstOrDefaultAsync() .ConfigureAwait(false); @@ -768,7 +768,7 @@ namespace Duplicati.Library.Main.Operation if (m_options.DisableFileScanner) { var d = await m_database - .GetLastBackupFileCountAndSize() + .GetLastBackupFileCountAndSize(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); m_result.OperationProgressUpdater.UpdatefileCount(d.Item1, d.Item2, true); @@ -815,7 +815,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) await database - .TerminatedWithActiveUploads(false) + .TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, false) .ConfigureAwait(false); // Make sure we have the database up-to-date @@ -826,7 +826,7 @@ namespace Duplicati.Library.Main.Operation if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { var lastVolumeSize = await m_database - .GetLastWrittenDBlockVolumeSize() + .GetLastWrittenDBlockVolumeSize(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await CompactIfRequired(backendManager, lastVolumeSize) @@ -836,7 +836,7 @@ namespace Duplicati.Library.Main.Operation if (m_options.UploadVerificationFile && await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload); - await FilelistProcessor.UploadVerificationFile(backendManager, m_options, m_database) + await FilelistProcessor.UploadVerificationFile(backendManager, m_options, m_database, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index 781ca99f2..5374947fc 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -52,17 +52,17 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new Exception(string.Format("Database file does not exist: {0}", m_options.Dbpath)); - await using (var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Compact", m_options.SqlitePageCache).ConfigureAwait(false)) + await using (var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Compact", m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { - await Utility.UpdateOptionsFromDb(db, m_options) + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var changed = await DoCompactAsync(db, false, backendManager).ConfigureAwait(false); if (changed && m_options.UploadVerificationFile) - await FilelistProcessor.UploadVerificationFile(backendManager, m_options, db); + await FilelistProcessor.UploadVerificationFile(backendManager, m_options, db, m_result.TaskControl.ProgressToken); if (!m_options.Dryrun) { @@ -72,7 +72,8 @@ namespace Duplicati.Library.Main.Operation if (changed) { - await db.WriteResults(m_result).ConfigureAwait(false); + await db.WriteResults(m_result, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); if (m_options.AutoVacuum) { m_result.VacuumResults = new VacuumResults(m_result); @@ -86,7 +87,7 @@ namespace Duplicati.Library.Main.Operation internal async Task DoCompactAsync(LocalDeleteDatabase db, bool hasVerifiedBackend, IBackendManager backendManager) { var report = await db - .GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount) + .GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); report.ReportCompactData(); @@ -95,11 +96,11 @@ namespace Duplicati.Library.Main.Operation { // Workaround where we allow a running backendmanager to be used if (!hasVerifiedBackend) - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, FilelistProcessor.VerifyMode.VerifyStrict, m_result.TaskControl.ProgressToken).ConfigureAwait(false); var newvol = new BlockVolumeWriter(m_options); newvol.VolumeID = await db - .RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary) + .RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); IndexVolumeWriter newvolindex = null; @@ -107,20 +108,20 @@ namespace Duplicati.Library.Main.Operation { newvolindex = new IndexVolumeWriter(m_options); newvolindex.VolumeID = await db - .RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary) + .RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await db - .AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID) + .AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } var blocksInVolume = 0L; var buffer = new byte[m_options.Blocksize]; var remoteList = await db - .GetRemoteVolumes() + .GetRemoteVolumes(m_result.TaskControl.ProgressToken) .Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified) - .ToArrayAsync() + .ToArrayAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); //These are for bookkeeping @@ -152,7 +153,7 @@ namespace Duplicati.Library.Main.Operation // If we crash now, we may leave partial files if (!m_options.Dryrun) await db - .TerminatedWithActiveUploads(true) + .TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, true) .ConfigureAwait(false); newvolindex?.StartVolume(newvol.RemoteFilename); @@ -167,7 +168,7 @@ namespace Duplicati.Library.Main.Operation .ToList(); } - using (var q = await db.CreateBlockQueryHelper().ConfigureAwait(false)) + using (var q = await db.CreateBlockQueryHelper(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(volumesToDownload, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { @@ -182,7 +183,7 @@ namespace Duplicati.Library.Main.Operation downloadedVolumes.Add(new KeyValuePair(entry.Name, entry.Size)); var volumeid = await db - .GetRemoteVolumeID(entry.Name) + .GetRemoteVolumeID(entry.Name, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var inst = VolumeBase.ParseFilename(entry.Name); @@ -190,7 +191,7 @@ namespace Duplicati.Library.Main.Operation { foreach (var e in f.Blocks) { - if (await q.UseBlock(e.Key, e.Value, volumeid).ConfigureAwait(false)) + if (await q.UseBlock(e.Key, e.Value, volumeid, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { //TODO: How do we get the compression hint? Reverse query for filename in db? var s = f.ReadBlock(e.Key, buffer); @@ -205,7 +206,7 @@ namespace Duplicati.Library.Main.Operation newvolindex.AddBlock(e.Key, e.Value); await db - .RegisterDuplicatedBlock(e.Key, e.Value, newvol.VolumeID) + .RegisterDuplicatedBlock(e.Key, e.Value, newvol.VolumeID, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); blocksInVolume++; @@ -217,18 +218,18 @@ namespace Duplicati.Library.Main.Operation newvol = new BlockVolumeWriter(m_options); newvol.VolumeID = await db - .RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary) + .RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None) { newvolindex = new IndexVolumeWriter(m_options); newvolindex.VolumeID = await db - .RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary) + .RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await db - .AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID) + .AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); newvolindex.StartVolume(newvol.RemoteFilename); @@ -242,7 +243,7 @@ namespace Duplicati.Library.Main.Operation // Commit as we have uploaded a volume if (!m_options.Dryrun) await db.Transaction - .CommitAsync("CommitCompact") + .CommitAsync("CommitCompact", true, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (deleteableVolumes.Any()) @@ -268,13 +269,13 @@ namespace Duplicati.Library.Main.Operation else { await db - .RemoveRemoteVolume(newvol.RemoteFilename) + .RemoveRemoteVolume(newvol.RemoteFilename, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (newvolindex != null) { await db - .RemoveRemoteVolume(newvolindex.RemoteFilename) + .RemoveRemoteVolume(newvolindex.RemoteFilename, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); newvolindex.FinishVolume(null, 0); @@ -285,7 +286,7 @@ namespace Duplicati.Library.Main.Operation // The remainder of the operation cannot leave partial files if (!m_options.Dryrun) await db - .TerminatedWithActiveUploads(false) + .TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, false) .ConfigureAwait(false); } else @@ -355,27 +356,27 @@ namespace Duplicati.Library.Main.Operation { // Find volumes that can be deleted var remoteFilesToRemove = await db - .ReOrderDeleteableVolumes(deleteableVolumes) - .ToListAsync() + .ReOrderDeleteableVolumes(deleteableVolumes, cancellationToken) + .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); // Make sure we do not re-assign blocks to any of the volumes we are about to delete var toRemoveVolumeIds = await db - .GetRemoteVolumeIDs(remoteFilesToRemove.Select(x => x.Name)) + .GetRemoteVolumeIDs(remoteFilesToRemove.Select(x => x.Name), cancellationToken) .Select(x => x.Value) .Distinct() - .ToListAsync() + .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); // Mark all volumes and relevant index files as disposable foreach (var f in remoteFilesToRemove) { await db - .PrepareForDelete(f.Name, toRemoveVolumeIds) + .PrepareForDelete(f.Name, toRemoveVolumeIds, cancellationToken) .ConfigureAwait(false); await db - .UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash) + .UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, cancellationToken) .ConfigureAwait(false); } @@ -384,7 +385,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) await db.Transaction - .CommitAsync("CommitDelete") + .CommitAsync("CommitDelete", token: cancellationToken) .ConfigureAwait(false); await foreach (var d in PerformDelete(backend, remoteFilesToRemove, cancellationToken).ConfigureAwait(false)) @@ -397,7 +398,7 @@ namespace Duplicati.Library.Main.Operation if (newvolindex != null && m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) indexVolumeFinished = async () => { - await foreach (var blocklist in db.GetBlocklists(newvol.VolumeID, m_options.Blocksize, m_options.BlockhashSize).ConfigureAwait(false)) + await foreach (var blocklist in db.GetBlocklists(newvol.VolumeID, m_options.Blocksize, m_options.BlockhashSize, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) newvolindex.WriteBlocklist(blocklist.Hash, blocklist.Buffer, 0, blocklist.Size); }; @@ -409,7 +410,7 @@ namespace Duplicati.Library.Main.Operation // because the transaction is not thread-safe, and shared with the upload await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await db - .UpdateRemoteVolume(newvol.RemoteFilename, RemoteVolumeState.Uploading, -1, null) + .UpdateRemoteVolume(newvol.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); // TODO: The upload here does not flush the database messages, diff --git a/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs b/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs index 69492ae26..9ff8ea208 100644 --- a/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs +++ b/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs @@ -68,11 +68,11 @@ namespace Duplicati.Library.Main.Operation using (var tmp = new TempFile()) { File.Copy(m_options.Dbpath, tmp, true); - await using (var db = await LocalBugReportDatabase.CreateAsync(tmp, m_options.SqlitePageCache).ConfigureAwait(false)) + await using (var db = await LocalBugReportDatabase.CreateAsync(tmp, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { - await db.Fix().ConfigureAwait(false); + await db.Fix(m_result.TaskControl.ProgressToken).ConfigureAwait(false); if (m_options.AutoVacuum) - await db.Vacuum().ConfigureAwait(false); + await db.Vacuum(m_result.TaskControl.ProgressToken).ConfigureAwait(false); } // Apply zip64 option if not already set diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs index 19498295f..260c33891 100644 --- a/Duplicati/Library/Main/Operation/DeleteHandler.cs +++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs @@ -51,11 +51,11 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseFileMissing"); - await using var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Delete", m_options.SqlitePageCache).ConfigureAwait(false); - await Utility.UpdateOptionsFromDb(db, m_options) + await using var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Delete", m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await DoRunAsync(db, false, false, backendManager).ConfigureAwait(false); @@ -69,13 +69,13 @@ namespace Duplicati.Library.Main.Operation public async Task DoRunAsync(LocalDeleteDatabase db, bool hasVerifiedBackend, bool forceCompact, IBackendManager backendManager) { if (!hasVerifiedBackend) - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, m_result.TaskControl.ProgressToken).ConfigureAwait(false); // We collapse the async enumerablo into a array to avoid multiple // enumerations (and thus multiple database queries) var filesets = await db - .FilesetsWithBackupVersion() - .ToArrayAsync() + .FilesetsWithBackupVersion(m_result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); List versionsToDelete = @@ -109,19 +109,20 @@ namespace Duplicati.Library.Main.Operation .DropFilesetsFromTable( versionsToDelete .Select(x => x.Time) - .ToArray() + .ToArray(), + m_result.TaskControl.ProgressToken ) - .ToArrayAsync() + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); foreach (var f in lst) await db - .UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null) + .UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (!m_options.Dryrun) await db.Transaction - .CommitAsync("CommitBeforeDelete") + .CommitAsync("CommitBeforeDelete", true, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); foreach (var f in lst) diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index 39d0e8134..5bf01708b 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -71,7 +71,7 @@ namespace Duplicati.Library.Main.Operation /// A task that completes when the verification is done. public static async Task VerifyLocalList(IBackendManager backendManager, LocalDatabase database, CancellationToken cancellationToken) { - var locallist = database.GetRemoteVolumes(); + var locallist = database.GetRemoteVolumes(cancellationToken); await foreach (var i in locallist.ConfigureAwait(false)) { switch (i.State) @@ -115,20 +115,21 @@ namespace Duplicati.Library.Main.Operation /// The backend writer to use. /// True if only the latest volumes should be verified. /// The mode to use for verification. + /// The cancellation token to use. /// An awaitable task. - public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter backendWriter, bool latestVolumesOnly, VerifyMode verifyMode) + public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter backendWriter, bool latestVolumesOnly, VerifyMode verifyMode, CancellationToken cancellationToken) { if (!options.NoBackendverification) { LocalBackupDatabase backupDatabase = - await LocalBackupDatabase.CreateAsync(database, options) + await LocalBackupDatabase.CreateAsync(database, options, null, cancellationToken) .ConfigureAwait(false); IEnumerable protectedFiles = await backupDatabase - .GetTemporaryFilelistVolumeNames(latestVolumesOnly) + .GetTemporaryFilelistVolumeNames(latestVolumesOnly, cancellationToken) .ConfigureAwait(false); - await VerifyRemoteList(backend, options, database, backendWriter, protectedFiles, [], logErrors: true, verifyMode: verifyMode).ConfigureAwait(false); + await VerifyRemoteList(backend, options, database, backendWriter, protectedFiles, [], logErrors: true, verifyMode: verifyMode, cancellationToken).ConfigureAwait(false); } } @@ -144,10 +145,11 @@ namespace Duplicati.Library.Main.Operation /// Filenames that should be exempted from strict verification. /// Disable the logging of errors to prevent spamming the log; exceptions will be thrown regardless. /// The mode to use for verification. + /// The cancellation token to use. /// An awaitable task. - public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, bool logErrors, VerifyMode verifyMode) + public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, bool logErrors, VerifyMode verifyMode, CancellationToken cancellationToken) { - var tp = await RemoteListAnalysis(backend, options, database, log, protectedFiles, strictExcemptFiles, verifyMode).ConfigureAwait(false); + var tp = await RemoteListAnalysis(backend, options, database, log, protectedFiles, strictExcemptFiles, verifyMode, cancellationToken).ConfigureAwait(false); long extraCount = 0; long missingCount = 0; @@ -213,10 +215,17 @@ namespace Duplicati.Library.Main.Operation /// The verification file. /// The database instance /// The stream to write to - public static void CreateVerificationFile(LocalDatabase db, System.IO.StreamWriter stream) + /// The cancellation token to use. + public static void CreateVerificationFile(LocalDatabase db, System.IO.StreamWriter stream, CancellationToken cancellationToken) { var s = new Newtonsoft.Json.JsonSerializer(); - s.Serialize(stream, db.GetRemoteVolumes().Where(x => x.State != RemoteVolumeState.Temporary).ToEnumerable().Cast().ToArray()); + s.Serialize(stream, + db.GetRemoteVolumes(cancellationToken) + .Where(x => x.State != RemoteVolumeState.Temporary) + .ToEnumerable() + .Cast() + .ToArray() + ); } /// @@ -225,14 +234,15 @@ namespace Duplicati.Library.Main.Operation /// The backend manager instance to use. /// The options to use. /// The attached database. + /// The cancellation token to use. /// A task that completes when the upload is done. - public static async Task UploadVerificationFile(IBackendManager backendManager, Options options, LocalDatabase db) + public static async Task UploadVerificationFile(IBackendManager backendManager, Options options, LocalDatabase db, CancellationToken cancellationToken) { using (var tempfile = new Library.Utility.TempFile()) { var remotename = options.Prefix + "-verification.json"; await using (var stream = new System.IO.StreamWriter(tempfile, false, System.Text.Encoding.UTF8)) - CreateVerificationFile(db, stream); + CreateVerificationFile(db, stream, cancellationToken); if (options.Dryrun) { @@ -240,8 +250,8 @@ namespace Duplicati.Library.Main.Operation } else { - await backendManager.PutVerificationFileAsync(remotename, tempfile, CancellationToken.None).ConfigureAwait(false); - await backendManager.WaitForEmptyAsync(db, CancellationToken.None).ConfigureAwait(false); + await backendManager.PutVerificationFileAsync(remotename, tempfile, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); } } } @@ -257,18 +267,19 @@ namespace Duplicati.Library.Main.Operation /// Filenames that should be exempted from deletion. /// Filenames that should be exempted from strict verification. /// The mode to use for verification. + /// The cancellation token to use. /// A task that when awaited contains the analysis result. - public static async Task RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, VerifyMode verifyMode) + public static async Task RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, VerifyMode verifyMode, CancellationToken cancellationToken) { // If the last operation completed, no cleanup should be required - if (verifyMode == VerifyMode.VerifyAndClean && !await database.TerminatedWithActiveUploads().ConfigureAwait(false)) + if (verifyMode == VerifyMode.VerifyAndClean && !await database.TerminatedWithActiveUploads(cancellationToken).ConfigureAwait(false)) verifyMode = VerifyMode.VerifyStrict; // Force cleanup should set the mode to cleanup if (verifyMode == VerifyMode.VerifyAndCleanForced) verifyMode = VerifyMode.VerifyAndClean; - var rawlist = await backendManager.ListAsync(CancellationToken.None).ConfigureAwait(false); + var rawlist = await backendManager.ListAsync(cancellationToken).ConfigureAwait(false); var lookup = new Dictionary(); protectedFiles = protectedFiles ?? []; strictExcemptFiles = strictExcemptFiles ?? []; @@ -300,8 +311,8 @@ namespace Duplicati.Library.Main.Operation log.UnknownFileCount = unknownlist.Count; log.UnknownFileSize = unknownlist.Select(x => Math.Max(0, x.Size)).Sum(); log.BackupListCount = await database - .FilesetTimes() - .CountAsync() + .FilesetTimes(cancellationToken) + .CountAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime(); @@ -316,17 +327,17 @@ namespace Duplicati.Library.Main.Operation var missingUploadingVolumes = new HashSet(); var protectedVolumes = new HashSet(); - await foreach (var e in database.DuplicateRemoteVolumes().ConfigureAwait(false)) + await foreach (var e in database.DuplicateRemoteVolumes(cancellationToken).ConfigureAwait(false)) { if (e.Value == RemoteVolumeState.Uploading || e.Value == RemoteVolumeState.Temporary) await database - .UnlinkRemoteVolume(e.Key, e.Value) + .UnlinkRemoteVolume(e.Key, e.Value, cancellationToken) .ConfigureAwait(false); else throw new RemoteListVerificationException(string.Format("The remote volume {0} appears in the database with state {1} and a deleted state, cannot continue", e.Key, e.Value.ToString()), "AmbiguousStateRemoteFiles"); } - var locallist = database.GetRemoteVolumes(); + var locallist = database.GetRemoteVolumes(cancellationToken); await foreach (var i in locallist.ConfigureAwait(false)) { var remoteFound = lookup.TryGetValue(i.Name, out var r); @@ -336,11 +347,11 @@ namespace Duplicati.Library.Main.Operation correctSize = true; if (archived && i.ArchiveTime <= DateTime.UnixEpoch) await database - .UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, true) + .UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, true, cancellationToken) .ConfigureAwait(false); else if (!archived && i.ArchiveTime > DateTime.UnixEpoch) await database - .UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, false) + .UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, false, cancellationToken) .ConfigureAwait(false); lookup.Remove(i.Name); @@ -397,7 +408,7 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "PromotingCompleteFile", "Promoting uploaded complete file from {0} to {2}: {1}", i.State, i.Name, RemoteVolumeState.Uploaded); await database - .UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash) + .UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash, cancellationToken) .ConfigureAwait(false); } else if (!remoteFound) @@ -407,7 +418,7 @@ namespace Duplicati.Library.Main.Operation protectedVolumes.Add(i.Name); Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "Keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name); await database - .UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null) + .UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null, cancellationToken) .ConfigureAwait(false); } else if (verifyMode == VerifyMode.VerifyStrict && !strictExcemptFiles.Any(pf => pf == i.Name)) @@ -420,7 +431,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "SchedulingMissingFileForDelete", "Scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name); missingUploadingVolumes.Add(i.Name); await database - .UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null) + .UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null, cancellationToken) .ConfigureAwait(false); } } @@ -460,7 +471,7 @@ namespace Duplicati.Library.Main.Operation missing.Add(i); else if (correctSize) await database - .UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash) + .UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash, cancellationToken) .ConfigureAwait(false); else missingHash.Add(new Tuple(r.File.Size, i)); @@ -491,17 +502,18 @@ namespace Duplicati.Library.Main.Operation await database .RemoveRemoteVolumes( missingUploadingVolumes - .Concat(temporaryAndDeletedVolumes) + .Concat(temporaryAndDeletedVolumes), + cancellationToken ) .ConfigureAwait(false); // Clear the flag after we have cleaned up if (!options.Dryrun && protectedVolumes.Count == 0) await database - .TerminatedWithActiveUploads(false) + .TerminatedWithActiveUploads(cancellationToken, false) .ConfigureAwait(false); } - else if (verifyMode == VerifyMode.VerifyOnly && await database.TerminatedWithActiveUploads().ConfigureAwait(false)) + else if (verifyMode == VerifyMode.VerifyOnly && await database.TerminatedWithActiveUploads(cancellationToken).ConfigureAwait(false)) { Logging.Log.WriteWarningMessage(LOGTAG, "ActiveUploadsDetected", null, "Active uploads detected, but no cleanup was performed. Run the \"repair\" command to clean up the incomplete files"); } diff --git a/Duplicati/Library/Main/Operation/ListAffected.cs b/Duplicati/Library/Main/Operation/ListAffected.cs index 33b0fa1bc..97284398e 100644 --- a/Duplicati/Library/Main/Operation/ListAffected.cs +++ b/Duplicati/Library/Main/Operation/ListAffected.cs @@ -44,21 +44,21 @@ namespace Duplicati.Library.Main.Operation if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - await using var db = await Database.LocalListAffectedDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); + await using var db = await Database.LocalListAffectedDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); if (callback == null) { m_result.SetResult( - await db.GetFilesets(args) + await db.GetFilesets(args, m_result.TaskControl.ProgressToken) .OrderByDescending(x => x.Time) .ToArrayAsync() .ConfigureAwait(false), - await db.GetFiles(args) + await db.GetFiles(args, m_result.TaskControl.ProgressToken) .ToArrayAsync() .ConfigureAwait(false), - await db.GetLogLines(args) + await db.GetLogLines(args, m_result.TaskControl.ProgressToken) .ToArrayAsync() .ConfigureAwait(false), - await db.GetVolumes(args) + await db.GetVolumes(args, m_result.TaskControl.ProgressToken) .ToArrayAsync() .ConfigureAwait(false) ); @@ -66,17 +66,17 @@ namespace Duplicati.Library.Main.Operation else { m_result.SetResult( - await db.GetFilesets(args) + await db.GetFilesets(args, m_result.TaskControl.ProgressToken) .OrderByDescending(x => x.Time) .ToArrayAsync() .ConfigureAwait(false), - await db.GetFiles(args) + await db.GetFiles(args, m_result.TaskControl.ProgressToken) .ToArrayAsync() .ConfigureAwait(false), - await db.GetLogLines(args) + await db.GetLogLines(args, m_result.TaskControl.ProgressToken) .ToArrayAsync() .ConfigureAwait(false), - await db.GetVolumes(args) + await db.GetVolumes(args, m_result.TaskControl.ProgressToken) .ToArrayAsync() .ConfigureAwait(false) ); diff --git a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs index 13dc5502c..8f37f4226 100644 --- a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs @@ -52,10 +52,10 @@ namespace Duplicati.Library.Main.Operation if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - await using var db = await Database.LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); - await Utility.UpdateOptionsFromDb(db, m_options) + await using var db = await Database.LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await DoRunAsync(backendManager, db, filter, callbackhandler).ConfigureAwait(false); await db.Transaction.RollBackAsync().ConfigureAwait(false); @@ -65,18 +65,18 @@ namespace Duplicati.Library.Main.Operation { List? missing = null; var brokensets = await db - .GetBrokenFilesets(options.Time, options.Version) - .ToArrayAsync() + .GetBrokenFilesets(options.Time, options.Version, result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken) .ConfigureAwait(false); if (brokensets.Length == 0) { - if (await db.RepairInProgress().ConfigureAwait(false)) + if (await db.RepairInProgress(result.TaskControl.ProgressToken).ConfigureAwait(false)) throw new UserInformationException("Cannot continue because the database is marked as being under repair, but does not have broken files.", "CannotListOnDatabaseInRepair"); Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsInDatabase", "No broken filesets found in database, checking for missing remote files"); - var remotestate = await FilelistProcessor.RemoteListAnalysis(backendManager, options, db, result.BackendWriter, null, null, FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false); + var remotestate = await FilelistProcessor.RemoteListAnalysis(backendManager, options, db, result.BackendWriter, null, null, FilelistProcessor.VerifyMode.VerifyOnly, result.TaskControl.ProgressToken).ConfigureAwait(false); if (!remotestate.ParsedVolumes.Any()) throw new UserInformationException("No remote volumes were found, refusing purge", "CannotPurgeWithNoRemoteVolumes"); @@ -90,24 +90,24 @@ namespace Duplicati.Library.Main.Operation // Mark all volumes as disposable foreach (var f in missing) await db - .UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash) + .UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, result.TaskControl.ProgressToken) .ConfigureAwait(false); Logging.Log.WriteInformationMessage(LOGTAG, "MarkedRemoteFilesForDeletion", "Marked {0} remote files for deletion", missing.Count); // Drop all content from tables await db - .RemoveMissingBlocks(missing.Select(x => x.Name)) + .RemoveMissingBlocks(missing.Select(x => x.Name), result.TaskControl.ProgressToken) .ConfigureAwait(false); // Mark all orphaned index files as disposable after removing the missing block files - await foreach (var f in db.GetOrphanedIndexFiles().ConfigureAwait(false)) + await foreach (var f in db.GetOrphanedIndexFiles(result.TaskControl.ProgressToken).ConfigureAwait(false)) await db - .UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash) + .UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, result.TaskControl.ProgressToken) .ConfigureAwait(false); brokensets = await db - .GetBrokenFilesets(options.Time, options.Version) + .GetBrokenFilesets(options.Time, options.Version, result.TaskControl.ProgressToken) .ToArrayAsync() .ConfigureAwait(false); } @@ -120,7 +120,7 @@ namespace Duplicati.Library.Main.Operation if (filter != null && !filter.Empty) throw new UserInformationException("Filters are not supported for this operation", "FiltersAreNotSupportedForListBrokenFiles"); - if (await db.PartiallyRecreated().ConfigureAwait(false)) + if (await db.PartiallyRecreated(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) throw new UserInformationException("The command does not work on partially recreated databases", "ListBrokenFilesDoesNotWorkOnPartialDatabase"); (var brokensets, var missing) = await GetBrokenFilesetsFromRemote(backendManager, m_result, db, m_options).ConfigureAwait(false); @@ -142,8 +142,8 @@ namespace Duplicati.Library.Main.Operation } var fstimes = await db - .FilesetTimes() - .ToListAsync() + .FilesetTimes(m_result.TaskControl.ProgressToken) + .ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var brokenfilesets = @@ -165,8 +165,8 @@ namespace Duplicati.Library.Main.Operation x.Timestamp, callbackhandler == null && !m_options.ListSetsOnly ? await db - .GetBrokenFilenames(x.FilesetID) - .ToArrayAsync() + .GetBrokenFilenames(x.FilesetID, m_result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false) : new MockList>((int)x.BrokenCount) ))) @@ -175,7 +175,7 @@ namespace Duplicati.Library.Main.Operation if (callbackhandler != null) foreach (var bs in brokenfilesets) - await foreach (var fe in db.GetBrokenFilenames(bs.FilesetID).ConfigureAwait(false)) + await foreach (var fe in db.GetBrokenFilenames(bs.FilesetID, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) if (!callbackhandler(bs.Version, bs.Timestamp, bs.BrokenCount, fe.Item1, fe.Item2)) break; } diff --git a/Duplicati/Library/Main/Operation/ListChangesHandler.cs b/Duplicati/Library/Main/Operation/ListChangesHandler.cs index 8ee6cc459..e1c82db45 100644 --- a/Duplicati/Library/Main/Operation/ListChangesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListChangesHandler.cs @@ -80,13 +80,13 @@ namespace Duplicati.Library.Main.Operation DateTime compareVersionTime; using var tmpdb = useLocalDb ? null : new TempFile(); - await using var db = await Database.LocalListChangesDatabase.CreateAsync(useLocalDb ? m_options.Dbpath : (string)tmpdb, m_options.SqlitePageCache).ConfigureAwait(false); - await using var storageKeeper = await db.CreateStorageHelper().ConfigureAwait(false); + await using var db = await Database.LocalListChangesDatabase.CreateAsync(useLocalDb ? m_options.Dbpath : (string)tmpdb, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await using var storageKeeper = await db.CreateStorageHelper(m_result.TaskControl.ProgressToken).ConfigureAwait(false); if (useLocalDb) { var dbtimes = await db - .FilesetTimes() - .ToListAsync() + .FilesetTimes(m_result.TaskControl.ProgressToken) + .ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (dbtimes.Count < 2) @@ -101,10 +101,10 @@ namespace Duplicati.Library.Main.Operation SelectTime(compareVersion, times, out compareVersionIndex, out compareVersionTime, out compareVersionId); await storageKeeper - .AddFromDb(baseVersionId, false, filter) + .AddFromDb(baseVersionId, false, filter, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await storageKeeper - .AddFromDb(compareVersionId, true, filter) + .AddFromDb(compareVersionId, true, filter, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } else @@ -151,7 +151,7 @@ namespace Duplicati.Library.Main.Operation foreach (var f in rd.Files) if (FilterExpression.Matches(filter, f.Path)) await storageKeeper - .AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), false) + .AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), false, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) @@ -162,20 +162,20 @@ namespace Duplicati.Library.Main.Operation foreach (var f in rd.Files) if (FilterExpression.Matches(filter, f.Path)) await storageKeeper - .AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), true) + .AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), true, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } var changes = await storageKeeper - .CreateChangeCountReport() + .CreateChangeCountReport(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var sizes = await storageKeeper - .CreateChangeSizeReport() + .CreateChangeSizeReport(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var lst = (m_options.FullResult || callback != null) ? - (from n in storageKeeper.CreateChangedFileReport() + (from n in storageKeeper.CreateChangedFileReport(m_result.TaskControl.ProgressToken) select n) : null; m_result.SetResult( @@ -184,7 +184,7 @@ namespace Duplicati.Library.Main.Operation changes.DeletedFolders, changes.DeletedSymlinks, changes.DeletedFiles, changes.ModifiedFolders, changes.ModifiedSymlinks, changes.ModifiedFiles, sizes.AddedSize, sizes.DeletedSize, sizes.PreviousSize, sizes.CurrentSize, - (lst == null || callback == null) ? null : await lst.ToArrayAsync().ConfigureAwait(false) + (lst == null || callback == null) ? null : await lst.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken).ConfigureAwait(false) ); if (callback != null) diff --git a/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs b/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs index ee5d229a3..d2f6bf925 100644 --- a/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs @@ -43,7 +43,7 @@ namespace Duplicati.Library.Main.Operation { var cancellationToken = CancellationToken.None; using var tmpdb = new TempFile(); - await using var db = await LocalDatabase.CreateLocalDatabaseAsync(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true, m_options.SqlitePageCache).ConfigureAwait(false); + await using var db = await LocalDatabase.CreateLocalDatabaseAsync(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); var filter = JoinedFilterExpression.Join(new Library.Utility.FilterExpression(filterstrings), compositefilter); try @@ -62,7 +62,7 @@ namespace Duplicati.Library.Main.Operation var file = fileversion.Value.File; var entry = await db - .GetRemoteVolume(file.Name) + .GetRemoteVolume(file.Name, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var files = new List(); diff --git a/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs b/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs index 90533677b..fb0cf4427 100644 --- a/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs @@ -48,14 +48,14 @@ internal static class ListFileVersionsHandler throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase"); await using var db = - await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache) + await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache, null, result.TaskControl.ProgressToken) .ConfigureAwait(false); long[]? filesetIds = null; if (!options.AllVersions) { filesetIds = await db - .GetFilesetIDs(options.Time, options.Version) - .ToArrayAsync() + .GetFilesetIDs(options.Time, options.Version, false, result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken) .ConfigureAwait(false); if (filesetIds.Length == 0) @@ -67,7 +67,7 @@ internal static class ListFileVersionsHandler paths = paths.Select(path => Util.AppendDirSeparator(path)).ToArray(); result.FileVersions = await db - .ListFileVersions(paths, filesetIds, offset, limit) + .ListFileVersions(paths, filesetIds, offset, limit, result.TaskControl.ProgressToken) .ConfigureAwait(false); } } diff --git a/Duplicati/Library/Main/Operation/ListFilesHandler.cs b/Duplicati/Library/Main/Operation/ListFilesHandler.cs index 77479350c..6d3375ddd 100644 --- a/Duplicati/Library/Main/Operation/ListFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFilesHandler.cs @@ -55,15 +55,15 @@ namespace Duplicati.Library.Main.Operation //Use a speedy local query if (!m_options.NoLocalDb && System.IO.File.Exists(m_options.Dbpath)) - await using (var db = await Database.LocalListDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false)) + await using (var db = await Database.LocalListDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { - await using var filesets = await db.SelectFileSets(m_options.Time, m_options.Version).ConfigureAwait(false); + await using var filesets = await db.SelectFileSets(m_options.Time, m_options.Version, m_result.TaskControl.ProgressToken).ConfigureAwait(false); if (!filter.Empty) { if (simpleList || (m_options.ListFolderContents && !m_options.AllVersions)) { await filesets - .TakeFirst() + .TakeFirst(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } } @@ -71,11 +71,11 @@ namespace Duplicati.Library.Main.Operation IAsyncEnumerable files; if (m_options.ListFolderContents) { - files = filesets.SelectFolderContents(filter); + files = filesets.SelectFolderContents(filter, m_result.TaskControl.ProgressToken); } else if (m_options.ListPrefixOnly) { - files = filesets.GetLargestPrefix(filter); + files = filesets.GetLargestPrefix(filter, m_result.TaskControl.ProgressToken); } else if (filter.Empty) { @@ -83,16 +83,16 @@ namespace Duplicati.Library.Main.Operation } else { - files = filesets.SelectFiles(filter); + files = filesets.SelectFiles(filter, m_result.TaskControl.ProgressToken); } if (m_options.ListSetsOnly) { m_result.SetResult( await filesets - .QuickSets() + .QuickSets(m_result.TaskControl.ProgressToken) .Select(x => new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes)) - .ToArrayAsync() + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false), null ); @@ -101,9 +101,9 @@ namespace Duplicati.Library.Main.Operation { m_result.SetResult( await filesets - .Sets() + .Sets(m_result.TaskControl.ProgressToken) .Select(x => new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes)) - .ToArrayAsync() + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false), files == null ? null @@ -112,14 +112,14 @@ namespace Duplicati.Library.Main.Operation new ListResultFile( n.Path, await n - .Sizes() - .ToArrayAsync() + .Sizes(m_result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false) ) ) .Select(x => x.Result) .Cast() - .ToArrayAsync() + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false) ); } @@ -137,7 +137,7 @@ namespace Duplicati.Library.Main.Operation // Otherwise, grab info from remote location using (var tmpdb = new TempFile()) - await using (var db = await LocalDatabase.CreateLocalDatabaseAsync(tmpdb, "List", true, m_options.SqlitePageCache).ConfigureAwait(false)) + await using (var db = await LocalDatabase.CreateLocalDatabaseAsync(tmpdb, "List", true, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { var filteredList = ParseAndFilterFilesets(await backendManager.ListAsync(cancellationToken).ConfigureAwait(false), m_options); if (filteredList.Count == 0) diff --git a/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs b/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs index b1169b1a2..8629f33e8 100644 --- a/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs @@ -22,6 +22,7 @@ #nullable enable using System.Linq; +using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Interface; @@ -49,12 +50,12 @@ internal static class ListFilesetsHandler // Use a speedy local query if (System.IO.File.Exists(options.Dbpath) && !options.NoLocalDb) { - await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache) + await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache, null, result.TaskControl.ProgressToken) .ConfigureAwait(false); result.Filesets = await db - .ListFilesetsExtended() - .ToArrayAsync() + .ListFilesetsExtended(result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken) .ConfigureAwait(false); return; diff --git a/Duplicati/Library/Main/Operation/ListFolderHandler.cs b/Duplicati/Library/Main/Operation/ListFolderHandler.cs index b71e0a81c..5a296f564 100644 --- a/Duplicati/Library/Main/Operation/ListFolderHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFolderHandler.cs @@ -46,12 +46,12 @@ internal static class ListFolderHandler if (!System.IO.File.Exists(options.Dbpath) || options.NoLocalDb) throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase"); - await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache) + await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache, null, result.TaskControl.ProgressToken) .ConfigureAwait(false); var filesetIds = await db - .GetFilesetIDs(options.Time, options.Version, singleTimeMatch: true) - .ToArrayAsync() + .GetFilesetIDs(options.Time, options.Version, singleTimeMatch: true, result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken) .ConfigureAwait(false); if (filesetIds.Length == 0) @@ -65,10 +65,11 @@ internal static class ListFolderHandler throw new UserInformationException("When no folder is specified, only one folder can be listed", "MultipleFoldersFound"); result.Entries = await db .ListFilesetEntries( - db.GetRootPrefixes(filesetIds[0]).ToEnumerable(), + db.GetRootPrefixes(filesetIds[0], result.TaskControl.ProgressToken).ToEnumerable(), filesetIds[0], offset, - limit + limit, + result.TaskControl.ProgressToken ) .ConfigureAwait(false); } @@ -76,10 +77,11 @@ internal static class ListFolderHandler { result.Entries = await db .ListFolder( - db.GetPrefixIds(folders).ToEnumerable(), + db.GetPrefixIds(folders, result.TaskControl.ProgressToken).ToEnumerable(), filesetIds[0], offset, - limit + limit, + result.TaskControl.ProgressToken ) .ConfigureAwait(false); } diff --git a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs index 47afcb0eb..348ebf383 100644 --- a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs @@ -54,13 +54,13 @@ namespace Duplicati.Library.Main.Operation if (filter != null && !filter.Empty) throw new UserInformationException("Filters are not supported for this operation", "FiltersNotAllowedOnPurgeBrokenFiles"); - await using var db = await LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); - if (await db.PartiallyRecreated().ConfigureAwait(false)) + await using var db = await LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + if (await db.PartiallyRecreated(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) throw new UserInformationException("The command does not work on partially recreated databases", "CannotPurgeOnPartialDatabase"); - await Utility.UpdateOptionsFromDb(db, m_options) + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var (sets, missing) = await ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(backendManager, m_result, db, m_options).ConfigureAwait(false); @@ -84,8 +84,8 @@ namespace Duplicati.Library.Main.Operation var pgspan = 0.95f / sets.Length; var filesets = await db - .FilesetTimes() - .ToListAsync() + .FilesetTimes(m_result.TaskControl.ProgressToken) + .ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var compare_list = sets.Select(async x => new @@ -95,7 +95,7 @@ namespace Duplicati.Library.Main.Operation RemoveCount = x.RemoveCount, Version = filesets.FindIndex(y => y.Key == x.FilesetID), SetCount = await db - .GetFilesetFileCount(x.FilesetID) + .GetFilesetFileCount(x.FilesetID, m_result.TaskControl.ProgressToken) .ConfigureAwait(false) }) .Select(x => x.Result) @@ -109,7 +109,8 @@ namespace Duplicati.Library.Main.Operation .GetEmptyMetadataBlocksetId( (missing ?? []).Select(x => x.ID), emptymetadata.FileHash, - emptymetadata.Blob.Length + emptymetadata.Blob.Length, + m_result.TaskControl.ProgressToken ) .ConfigureAwait(false); if (replacementMetadataBlocksetId < 0) @@ -119,11 +120,11 @@ namespace Duplicati.Library.Main.Operation var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray(); var to_purge = compare_list.Where(x => x.RemoveCount != x.SetCount).ToArray(); - if (fully_emptied.Length == await db.FilesetTimes().CountAsync().ConfigureAwait(false)) + if (fully_emptied.Length == await db.FilesetTimes(m_result.TaskControl.ProgressToken).CountAsync(cancellationToken: m_result.TaskControl.ProgressToken).ConfigureAwait(false)) throw new UserInformationException("All filesets are fully broken and needs to be removed. To avoid unexpected deletions, you must manually remove the remote files and delete the database.", "AllFilesetsBroken"); if (!m_options.Dryrun) - await db.Transaction.CommitAsync().ConfigureAwait(false); + await db.Transaction.CommitAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false); if (fully_emptied.Length != 0) @@ -134,7 +135,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing {0} filesets where all file(s) are broken: {1}", fully_emptied.Length, string.Join(", ", fully_emptied.Select(x => x.Timestamp.ToLocalTime().ToString()))); m_result.DeleteResults = new DeleteResults(m_result); - await using (var rmdb = await LocalDeleteDatabase.CreateAsync(db).ConfigureAwait(false)) + await using (var rmdb = await LocalDeleteDatabase.CreateAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { var opts = new Options(new Dictionary(m_options.RawOptions)); opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString())); @@ -146,7 +147,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) await rmdb.Transaction - .CommitAsync("CommitDelete") + .CommitAsync("CommitDelete", true, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } @@ -163,12 +164,12 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "PurgingFiles", "Purging {0} file(s) from fileset {1}", bs.RemoveCount, bs.Timestamp.ToLocalTime()); var opts = new Options(new Dictionary(m_options.RawOptions)); - await using (var pgdb = await Database.LocalPurgeDatabase.CreateAsync(db).ConfigureAwait(false)) + await using (var pgdb = await Database.LocalPurgeDatabase.CreateAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { // Recompute the version number after we deleted the versions before filesets = await pgdb - .FilesetTimes() - .ToListAsync() + .FilesetTimes(m_result.TaskControl.ProgressToken) + .ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var thisversion = filesets.FindIndex(y => y.Key == bs.FilesetID); if (thisversion < 0) @@ -186,9 +187,9 @@ namespace Duplicati.Library.Main.Operation // Update entries that would be removed because of missing metadata var updatedEntries = 0; if (!m_options.DisableReplaceMissingMetadata) - updatedEntries = await db.ReplaceMetadata(filesetid, replacementMetadataBlocksetId); + updatedEntries = await db.ReplaceMetadata(filesetid, replacementMetadataBlocksetId, m_result.TaskControl.ProgressToken); - await db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID"); + await db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID", m_result.TaskControl.ProgressToken); return updatedEntries; }).ConfigureAwait(false); } @@ -201,20 +202,20 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(0.95f); - if (!m_options.Dryrun && await db.RepairInProgress().ConfigureAwait(false)) + if (!m_options.Dryrun && await db.RepairInProgress(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { Logging.Log.WriteInformationMessage(LOGTAG, "ValidatingDatabase", "Database was previously marked as in-progress, checking if it is valid after purging files"); await db - .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true) + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); Logging.Log.WriteInformationMessage(LOGTAG, "UpdatingDatabase", "Purge completed, and consistency checks completed, marking database as complete"); - await db.RepairInProgress(false).ConfigureAwait(false); + await db.RepairInProgress(m_result.TaskControl.ProgressToken, false).ConfigureAwait(false); } else { - await db.Transaction.RollBackAsync().ConfigureAwait(false); + await db.Transaction.RollBackAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false); await db - .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true) + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index dd7aece5a..378439872 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -52,10 +52,10 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - await using var db = await Database.LocalPurgeDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); + await using var db = await Database.LocalPurgeDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await DoRunAsync(backendManager, db, filter, null, 0, 1).ConfigureAwait(false); await db - .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true) + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } @@ -69,44 +69,44 @@ namespace Duplicati.Library.Main.Operation var doCompactStep = !m_options.NoAutoCompact && filtercommand == null; - if (await db.PartiallyRecreated().ConfigureAwait(false)) + if (await db.PartiallyRecreated(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) throw new UserInformationException("The purge command does not work on partially recreated databases", "PurgeNotAllowedOnPartialDatabase"); - if (await db.RepairInProgress().ConfigureAwait(false) && filtercommand == null) + if (await db.RepairInProgress(m_result.TaskControl.ProgressToken).ConfigureAwait(false) && filtercommand == null) throw new UserInformationException(string.Format("The purge command does not work on an incomplete database, try the {0} operation.", "purge-broken-files"), "PurgeNotAllowedOnIncompleteDatabase"); var versions = await db - .GetFilesetIDs(m_options.Time, m_options.Version) + .GetFilesetIDs(m_options.Time, m_options.Version, false, m_result.TaskControl.ProgressToken) .OrderByDescending(x => x) - .ToArrayAsync() + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (versions.Length <= 0) throw new UserInformationException("No filesets matched the supplied time or versions", "NoFilesetFoundForTimeOrVersion"); - var orphans = await db.CountOrphanFiles().ConfigureAwait(false); + var orphans = await db.CountOrphanFiles(m_result.TaskControl.ProgressToken).ConfigureAwait(false); if (orphans != 0) throw new UserInformationException(string.Format("Unable to start the purge process as there are {0} orphan file(s)", orphans), "CannotPurgeWithOrphans"); - await Utility.UpdateOptionsFromDb(db, m_options) + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (filtercommand == null) { - await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, false) + await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, false, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (m_options.NoBackendverification) await FilelistProcessor.VerifyLocalList(backendManager, db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); else - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, null, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, null, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } var filesets = await db - .FilesetTimes() + .FilesetTimes(m_result.TaskControl.ProgressToken) .OrderByDescending(x => x.Value) - .ToArrayAsync() + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var versionprogress = ((doCompactStep ? 0.75f : 1.0f) / versions.Length) * pgspan; @@ -118,7 +118,7 @@ namespace Duplicati.Library.Main.Operation // If we crash now, it is possible that the remote storage contains partial files if (!m_options.Dryrun) - await db.TerminatedWithActiveUploads(true).ConfigureAwait(false); + await db.TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, true).ConfigureAwait(false); // Reverse makes sure we re-write the old versions first foreach (var versionid in versions.Reverse()) @@ -132,18 +132,18 @@ namespace Duplicati.Library.Main.Operation var ts = await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, tsOriginal) .ConfigureAwait(false); - var prevfilename = await db.GetRemoteVolumeNameForFileset(filesets[ix].Key) + var prevfilename = await db.GetRemoteVolumeNameForFileset(filesets[ix].Key, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (ix != 0 && filesets[ix - 1].Value <= ts) throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is larger than the next timestamp {2}", prevfilename, ts, filesets[ix - 1].Value)); - await using (var tempset = await db.CreateTemporaryFileset(versionid).ConfigureAwait(false)) + await using (var tempset = await db.CreateTemporaryFileset(versionid, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { if (filtercommand == null) - await tempset.ApplyFilter(filter).ConfigureAwait(false); + await tempset.ApplyFilter(filter, m_result.TaskControl.ProgressToken).ConfigureAwait(false); else - await tempset.ApplyFilter(filtercommand) + await tempset.ApplyFilter(filtercommand, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (tempset.RemovedFileCount + tempset.UpdatedFileCount == 0) @@ -161,10 +161,10 @@ namespace Duplicati.Library.Main.Operation using (var vol = new FilesetVolumeWriter(m_options, ts)) { var isOriginalFilesetFullBackup = await db - .IsFilesetFullBackup(tsOriginal) + .IsFilesetFullBackup(tsOriginal, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var newids = await tempset - .ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup) + .ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); vol.VolumeID = newids.Item1; vol.CreateFilesetFile(isOriginalFilesetFullBackup); @@ -172,7 +172,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "ReplacingFileset", "Replacing fileset {0} with {1} which has with {2} fewer file(s) ({3} reduction)", prevfilename, vol.RemoteFilename, tempset.RemovedFileCount, Library.Utility.Utility.FormatSizeString(tempset.RemovedFileSize)); await db - .WriteFileset(vol, newids.Item2) + .WriteFileset(vol, newids.Item2, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); m_result.RemovedFileSize += tempset.RemovedFileSize; @@ -185,7 +185,7 @@ namespace Duplicati.Library.Main.Operation if (m_options.Dryrun || m_options.FullResult) { - await foreach (var fe in tempset.ListAllDeletedFiles().ConfigureAwait(false)) + await foreach (var fe in tempset.ListAllDeletedFiles(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { var msg = string.Format(" Purging file {0} ({1})", fe.Key, Library.Utility.Utility.FormatSizeString(fe.Value)); @@ -212,19 +212,19 @@ namespace Duplicati.Library.Main.Operation else { await db - .UpdateRemoteVolume(vol.RemoteFilename, RemoteVolumeState.Uploading, -1, null) + .UpdateRemoteVolume(vol.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var lst = await db - .DropFilesetsFromTable(new[] { tsOriginal }) - .ToArrayAsync() + .DropFilesetsFromTable(new[] { tsOriginal }, m_result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); foreach (var f in lst) await db - .UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null) + .UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await db.Transaction - .CommitAsync() + .CommitAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await backendManager.PutAsync(vol, null, null, true, async () => @@ -235,7 +235,7 @@ namespace Duplicati.Library.Main.Operation // passing down a transaction, but maybe is // not needed anymore. await db.Transaction - .CommitAsync() + .CommitAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); }, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await backendManager.DeleteAsync(prevfilename, -1, true, m_result.TaskControl.ProgressToken).ConfigureAwait(false); @@ -251,7 +251,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) await db - .TerminatedWithActiveUploads(false) + .TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, false) .ConfigureAwait(false); if (doCompactStep) @@ -265,12 +265,12 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(pgoffset + (0.75f * pgspan)); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Compact); m_result.CompactResults = new CompactResults(m_result); - await using var cdb = await Database.LocalDeleteDatabase.CreateAsync(db).ConfigureAwait(false); + await using var cdb = await Database.LocalDeleteDatabase.CreateAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await new CompactHandler(m_options, (CompactResults)m_result.CompactResults) .DoCompactAsync(cdb, true, backendManager) .ConfigureAwait(false); - await cdb.Transaction.CommitAsync("PostCompact"); + await cdb.Transaction.CommitAsync("PostCompact", true, m_result.TaskControl.ProgressToken); } m_result.OperationProgressUpdater.UpdateProgress(pgoffset + pgspan); diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index 3c00e6c08..c5a116c54 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -23,6 +23,7 @@ using System.Collections.Generic; //using System.Data; using System.IO; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; @@ -65,14 +66,14 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(string.Format("Cannot recreate database because file already exists: {0}", path), "RecreateTargetDatabaseExists"); await using var db = - await LocalDatabase.CreateLocalDatabaseAsync(path, "Recreate", true, m_options.SqlitePageCache) + await LocalDatabase.CreateLocalDatabaseAsync(path, "Recreate", true, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await DoRunAsync(backendManager, db, false, filter, filelistfilter, blockprocessor).ConfigureAwait(false); // Ensure database is consistent after the recreate await db - .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true) + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } @@ -89,9 +90,9 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(string.Format("Can only update with paths, try setting --{0}", "repair-only-paths"), "RepairUpdateRequiresPathsOnly"); await using var db = - await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Recreate", true, m_options.SqlitePageCache) + await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Recreate", true, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - if ((await db.FindMatchingFilesets(m_options.Time, m_options.Version).ConfigureAwait(false)).Any()) + if ((await db.FindMatchingFilesets(m_options.Time, m_options.Version, m_result.TaskControl.ProgressToken).ConfigureAwait(false)).Any()) { if (m_options.IgnoreUpdateIfVersionExists) { @@ -103,17 +104,17 @@ namespace Duplicati.Library.Main.Operation } // Mark as incomplete - await db.PartiallyRecreated(true).ConfigureAwait(false); + await db.PartiallyRecreated(m_result.TaskControl.ProgressToken, true).ConfigureAwait(false); var preexistingOptionsInDatabase = - await Utility.ContainsOptionsForVerification(db) + await Utility.ContainsOptionsForVerification(db, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - await Utility.UpdateOptionsFromDb(db, m_options) + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); // Make sure the options have not changed between calls, unless there are no previous options if (preexistingOptionsInDatabase) - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await DoRunAsync(backendManager, db, true, filter, filelistfilter, blockprocessor).ConfigureAwait(false); @@ -135,9 +136,9 @@ namespace Duplicati.Library.Main.Operation //We build a local database in steps. await using var restoredb = - await LocalRecreateDatabase.CreateAsync(dbparent, m_options) + await LocalRecreateDatabase.CreateAsync(dbparent, m_options, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - await restoredb.RepairInProgress(true).ConfigureAwait(false); + await restoredb.RepairInProgress(m_result.TaskControl.ProgressToken, true).ConfigureAwait(false); var expRecreateDb = false; // experimental recreate db code flag var volumeIds = new Dictionary(); @@ -205,9 +206,9 @@ namespace Duplicati.Library.Main.Operation // If we are updating, all files should be accounted for foreach (var fl in remotefiles) volumeIds[fl.File.Name] = updating - ? await restoredb.GetRemoteVolumeID(fl.File.Name) + ? await restoredb.GetRemoteVolumeID(fl.File.Name, m_result.TaskControl.ProgressToken) .ConfigureAwait(false) - : await restoredb.RegisterRemoteVolume(fl.File.Name, fl.FileType, fl.File.Size, RemoteVolumeState.Uploaded) + : await restoredb.RegisterRemoteVolume(fl.File.Name, fl.FileType, fl.File.Size, RemoteVolumeState.Uploaded, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var hasUpdatedOptions = false; @@ -224,7 +225,7 @@ namespace Duplicati.Library.Main.Operation foreach (var n in filelists) if (volumeIds[n.File.Name] == -1) volumeIds[n.File.Name] = await restoredb - .RegisterRemoteVolume(n.File.Name, n.FileType, RemoteVolumeState.Uploaded, n.File.Size, new TimeSpan(0)) + .RegisterRemoteVolume(n.File.Name, n.FileType, RemoteVolumeState.Uploaded, n.File.Size, new TimeSpan(0), m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } @@ -240,7 +241,7 @@ namespace Duplicati.Library.Main.Operation m_result.EndTime = DateTime.UtcNow; // Implicit rollback await restoredb.Transaction - .RollBackAsync() + .RollBackAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); return; } @@ -262,7 +263,7 @@ namespace Duplicati.Library.Main.Operation if (!string.IsNullOrWhiteSpace(hash) && size > 0) await restoredb - .UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, size, hash) + .UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, size, hash, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var parsed = VolumeBase.ParseFilename(entry.Name); @@ -280,10 +281,10 @@ namespace Duplicati.Library.Main.Operation // Create timestamped operations based on the file timestamp var filesetid = await restoredb - .CreateFileset(volumeIds[entry.Name], parsed.Time) + .CreateFileset(volumeIds[entry.Name], parsed.Time, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - await RecreateFilesetFromRemoteList(restoredb, compressor, filesetid, m_options, filter) + await RecreateFilesetFromRemoteList(restoredb, compressor, filesetid, m_options, filter, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } } @@ -295,7 +296,7 @@ namespace Duplicati.Library.Main.Operation m_result.EndTime = DateTime.UtcNow; // Implicit rollback await restoredb.Transaction - .RollBackAsync() + .RollBackAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); throw; } @@ -305,7 +306,7 @@ namespace Duplicati.Library.Main.Operation m_result.EndTime = DateTime.UtcNow; // Implicit rollback await restoredb.Transaction - .RollBackAsync() + .RollBackAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); throw; } @@ -314,7 +315,7 @@ namespace Duplicati.Library.Main.Operation { // Implicit rollback await restoredb.Transaction - .RollBackAsync() + .RollBackAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); throw; } @@ -323,12 +324,12 @@ namespace Duplicati.Library.Main.Operation //Make sure we write the config if it has been read from a manifest if (hasUpdatedOptions) - await Utility.VerifyOptionsAndUpdateDatabase(restoredb, m_options) + await Utility.VerifyOptionsAndUpdateDatabase(restoredb, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); using (new Logging.Timer(LOGTAG, "CommitUpdateFilesetFromRemote", "CommitUpdateFilesetFromRemote")) await restoredb.Transaction - .CommitAsync() + .CommitAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); // do we stop after just handling the dlist files ? @@ -361,7 +362,7 @@ namespace Duplicati.Library.Main.Operation m_result.EndTime = DateTime.UtcNow; // Implicit rollback await restoredb.Transaction - .RollBackAsync() + .RollBackAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); return; } @@ -374,7 +375,7 @@ namespace Duplicati.Library.Main.Operation { if (!string.IsNullOrWhiteSpace(hash) && size > 0) await restoredb - .UpdateRemoteVolume(name, RemoteVolumeState.Verified, size, hash) + .UpdateRemoteVolume(name, RemoteVolumeState.Verified, size, hash, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); using (var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options, hashsize)) @@ -383,13 +384,13 @@ namespace Duplicati.Library.Main.Operation { var filename = a.Filename; var volumeID = await restoredb - .GetRemoteVolumeID(filename) + .GetRemoteVolumeID(filename, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); // No such file if (volumeID < 0) (volumeID, filename) = - await ProbeForMatchingFilename(filename, restoredb) + await ProbeForMatchingFilename(filename, restoredb, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var missing = false; @@ -402,7 +403,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteWarningMessage(LOGTAG, "MissingFileDetected", null, "Remote file referenced as {0} by {1}, but not found in list, registering a missing remote file", filename, name); missing = true; volumeID = await restoredb - .RegisterRemoteVolume(filename, p.FileType, RemoteVolumeState.Temporary) + .RegisterRemoteVolume(filename, p.FileType, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } @@ -410,16 +411,17 @@ namespace Duplicati.Library.Main.Operation //Add all block/volume mappings foreach (var b in a.Blocks) { - anyChange |= (await restoredb.UpdateBlock(b.Key, b.Value, volumeID).ConfigureAwait(false)).Item1; + anyChange |= (await restoredb.UpdateBlock(b.Key, b.Value, volumeID, m_result.TaskControl.ProgressToken).ConfigureAwait(false)).Item1; } await restoredb - .UpdateRemoteVolume(filename, missing ? RemoteVolumeState.Temporary : RemoteVolumeState.Verified, a.Length, a.Hash) + .UpdateRemoteVolume(filename, missing ? RemoteVolumeState.Temporary : RemoteVolumeState.Verified, a.Length, a.Hash, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await restoredb .AddIndexBlockLink( - await restoredb.GetRemoteVolumeID(name).ConfigureAwait(false), - volumeID + await restoredb.GetRemoteVolumeID(name, m_result.TaskControl.ProgressToken).ConfigureAwait(false), + volumeID, + m_result.TaskControl.ProgressToken ) .ConfigureAwait(false); } @@ -436,7 +438,7 @@ namespace Duplicati.Library.Main.Operation // done before we add it to the database, since we do not have nested transactions var list = b.Blocklist.ToList(); await restoredb - .AddTempBlockListHash(b.Hash, list) + .AddTempBlockListHash(b.Hash, list, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } catch (System.IO.InvalidDataException e) @@ -461,7 +463,7 @@ namespace Duplicati.Library.Main.Operation m_result.EndTime = DateTime.UtcNow; // Implicit rollback await restoredb.Transaction - .RollBackAsync() + .RollBackAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); throw; } @@ -470,7 +472,7 @@ namespace Duplicati.Library.Main.Operation { // Implicit rollback await restoredb.Transaction - .RollBackAsync() + .RollBackAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); throw; } @@ -479,25 +481,25 @@ namespace Duplicati.Library.Main.Operation using (new Logging.Timer(LOGTAG, "CommitRecreateDb", "CommitRecreatedDb")) await restoredb.Transaction - .CommitAsync() + .CommitAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); // TODO: In some cases, we can avoid downloading all index files, // if we are lucky and pick the right ones } - await restoredb.CleanupMissingVolumes().ConfigureAwait(false); + await restoredb.CleanupMissingVolumes(m_result.TaskControl.ProgressToken).ConfigureAwait(false); // Update the real tables from the temp tables if (expRecreateDb) // add missing blocks and blocksetentry data (at this point // we have not yet anything in the blocksetentry table) await restoredb - .AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize) + .AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, false, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); else await restoredb - .FindMissingBlocklistHashes(hashsize, m_options.Blocksize) + .FindMissingBlocklistHashes(hashsize, m_options.Blocksize, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); // We have now grabbed as much information as possible, @@ -507,7 +509,7 @@ namespace Duplicati.Library.Main.Operation { // Grab the list matching the pass type var lst = await restoredb - .GetMissingBlockListVolumes(i, m_options.Blocksize, hashsize, m_options.RepairForceBlockUse) + .GetMissingBlockListVolumes(i, m_options.Blocksize, hashsize, m_options.RepairForceBlockUse, m_result.TaskControl.ProgressToken) .ToListAsync() .ConfigureAwait(false); @@ -545,7 +547,7 @@ namespace Duplicati.Library.Main.Operation m_result.EndTime = DateTime.UtcNow; // Implicit rollback await restoredb.Transaction - .RollBackAsync() + .RollBackAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); return; } @@ -555,26 +557,26 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingBlocklistVolumes", "Pass {0} of 3, processing blocklist volume {1} of {2}", (i + 1), progress, lst.Count); var volumeid = await restoredb - .GetRemoteVolumeID(name) + .GetRemoteVolumeID(name, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await restoredb - .UpdateRemoteVolume(name, RemoteVolumeState.Uploaded, size, hash) + .UpdateRemoteVolume(name, RemoteVolumeState.Uploaded, size, hash, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); bool anyChange = false; // Update the block table so we know about the block/volume map foreach (var h in rd.Blocks) { - anyChange |= (await restoredb.UpdateBlock(h.Key, h.Value, volumeid).ConfigureAwait(false)).Item1; + anyChange |= (await restoredb.UpdateBlock(h.Key, h.Value, volumeid, m_result.TaskControl.ProgressToken).ConfigureAwait(false)).Item1; } // now that we have the blocks/volume relationships, we can go from the (already known from dlist step) blocklisthashes // to the needed list blocks in the volume, so grab them from the database // read the blocks list hashes from the volume data (the handled file) and insert them into the temp blocklisthash table - await foreach (var blocklisthash in restoredb.GetBlockLists(volumeid).ConfigureAwait(false)) + await foreach (var blocklisthash in restoredb.GetBlockLists(volumeid, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { - if (await restoredb.AddTempBlockListHash(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize)).ConfigureAwait(false)) + if (await restoredb.AddTempBlockListHash(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize), m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { anyChange = true; } @@ -593,17 +595,17 @@ namespace Duplicati.Library.Main.Operation } if (expRecreateDb) await restoredb - .AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, false) + .AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, false, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); else await restoredb - .FindMissingBlocklistHashes(hashsize, m_options.Blocksize) + .FindMissingBlocklistHashes(hashsize, m_options.Blocksize, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } using (new Logging.Timer(LOGTAG, "CommitRestoredBlocklist", "CommitRestoredBlocklist")) await restoredb.Transaction - .CommitAsync() + .CommitAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); //At this point we can patch files with data from the block volume @@ -618,7 +620,7 @@ namespace Duplicati.Library.Main.Operation { // Implicit rollback await restoredb.Transaction - .RollBackAsync() + .RollBackAsync(m_result.TaskControl.ProgressToken) .ConfigureAwait(false); throw; } @@ -634,10 +636,10 @@ namespace Duplicati.Library.Main.Operation // All blocks are collected and added into the Block table // Find out which blocks are deleted and move them into DeletedBlock, // so that compact can calculate the unused space - await restoredb.CleanupDeletedBlocks().ConfigureAwait(false); + await restoredb.CleanupDeletedBlocks(m_result.TaskControl.ProgressToken).ConfigureAwait(false); } - await restoredb.CleanupMissingVolumes().ConfigureAwait(false); + await restoredb.CleanupMissingVolumes(m_result.TaskControl.ProgressToken).ConfigureAwait(false); if (m_options.RepairOnlyPaths) { @@ -652,11 +654,11 @@ namespace Duplicati.Library.Main.Operation // except to continue a backup m_result.EndTime = DateTime.UtcNow; - await using (var lbfdb = await LocalListBrokenFilesDatabase.CreateAsync(restoredb).ConfigureAwait(false)) + await using (var lbfdb = await LocalListBrokenFilesDatabase.CreateAsync(restoredb, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { var broken = await lbfdb - .GetBrokenFilesets(new DateTime(0), null) - .CountAsync() + .GetBrokenFilesets(new DateTime(0), null, m_result.TaskControl.ProgressToken) + .CountAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (broken != 0) @@ -664,18 +666,28 @@ namespace Duplicati.Library.Main.Operation } await restoredb - .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true) + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); Logging.Log.WriteInformationMessage(LOGTAG, "RecreateCompleted", "Recreate completed, and consistency checks completed, marking database as complete"); - await restoredb.RepairInProgress(false).ConfigureAwait(false); + await restoredb.RepairInProgress(m_result.TaskControl.ProgressToken, false).ConfigureAwait(false); } m_result.EndTime = DateTime.UtcNow; } - public static async Task RecreateFilesetFromRemoteList(LocalRecreateDatabase restoredb, ICompression compressor, long filesetid, Options options, IFilter filter) + /// + /// Recreate a fileset from the remote filelist. + /// + /// The database to restore to. + /// The compression module to use. + /// The ID of the fileset to recreate. + /// The options to use for the operation. + /// The filter to apply to the files. + /// The cancellation token to monitor for cancellation requests. + /// A task that completes when the fileset has been recreated. + public static async Task RecreateFilesetFromRemoteList(LocalRecreateDatabase restoredb, ICompression compressor, long filesetid, Options options, IFilter filter, CancellationToken cancellationToken) { var blocksize = options.Blocksize; var hashes_pr_block = blocksize / options.BlockhashSize; @@ -685,11 +697,11 @@ namespace Duplicati.Library.Main.Operation // update fileset using filesetData await restoredb - .UpdateFullBackupStateInFileset(filesetid, filesetData.IsFullBackup) + .UpdateFullBackupStateInFileset(filesetid, filesetData.IsFullBackup, cancellationToken) .ConfigureAwait(false); // clear any existing fileset entries - await restoredb.ClearFilesetEntries(filesetid).ConfigureAwait(false); + await restoredb.ClearFilesetEntries(filesetid, cancellationToken).ConfigureAwait(false); using (var filelistreader = new FilesetVolumeReader(compressor, options)) foreach (var fe in filelistreader.Files.Where(x => Library.Utility.FilterExpression.Matches(filter, x.Path))) @@ -703,17 +715,17 @@ namespace Duplicati.Library.Main.Operation var metadataid = long.MinValue; var split = LocalDatabase.SplitIntoPrefixAndName(fe.Path); var prefixid = await restoredb - .GetOrCreatePathPrefix(split.Key) + .GetOrCreatePathPrefix(split.Key, cancellationToken) .ConfigureAwait(false); switch (fe.Type) { case FilelistEntryType.Folder: metadataid = await restoredb - .AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes) + .AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, cancellationToken) .ConfigureAwait(false); await restoredb - .AddDirectoryEntry(filesetid, prefixid, split.Value, fe.Time, metadataid) + .AddDirectoryEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, cancellationToken) .ConfigureAwait(false); break; @@ -723,24 +735,24 @@ namespace Duplicati.Library.Main.Operation if (expectedblocks <= 1) expectedblocklisthashes = 0; var blocksetid = await restoredb - .AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes) + .AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes, cancellationToken) .ConfigureAwait(false); metadataid = await restoredb - .AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes) + .AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, cancellationToken) .ConfigureAwait(false); await restoredb - .AddFileEntry(filesetid, prefixid, split.Value, fe.Time, blocksetid, metadataid) + .AddFileEntry(filesetid, prefixid, split.Value, fe.Time, blocksetid, metadataid, cancellationToken) .ConfigureAwait(false); if (fe.Size <= blocksize) { if (!string.IsNullOrWhiteSpace(fe.Blockhash)) await restoredb - .AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize) + .AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize, cancellationToken) .ConfigureAwait(false); else if (options.BlockHashAlgorithm == options.FileHashAlgorithm) await restoredb - .AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size) + .AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size, cancellationToken) .ConfigureAwait(false); else if (fe.Size > 0) Logging.Log.WriteWarningMessage(LOGTAG, "MissingBlockHash", null, "No block hash found for file: {0}", fe.Path); @@ -749,10 +761,10 @@ namespace Duplicati.Library.Main.Operation break; case FilelistEntryType.Symlink: metadataid = await restoredb - .AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes) + .AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, cancellationToken) .ConfigureAwait(false); await restoredb - .AddSymlinkEntry(filesetid, prefixid, split.Value, fe.Time, metadataid) + .AddSymlinkEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, cancellationToken) .ConfigureAwait(false); break; default: @@ -764,11 +776,11 @@ namespace Duplicati.Library.Main.Operation { if (!string.IsNullOrWhiteSpace(fe.Metablockhash)) await restoredb - .AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize) + .AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize, cancellationToken) .ConfigureAwait(false); else if (options.BlockHashAlgorithm == options.FileHashAlgorithm) await restoredb - .AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize) + .AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize, cancellationToken) .ConfigureAwait(false); else Logging.Log.WriteWarningMessage(LOGTAG, "MissingMetadataBlockHash", null, "No block hash found for file metadata: {0}", fe.Path); @@ -780,7 +792,7 @@ namespace Duplicati.Library.Main.Operation } } - await restoredb.Transaction.CommitAsync().ConfigureAwait(false); + await restoredb.Transaction.CommitAsync(cancellationToken).ConfigureAwait(false); } /// @@ -788,8 +800,9 @@ namespace Duplicati.Library.Main.Operation /// /// The filename read and written. /// The database to query. + /// A cancellation token to cancel the operation. /// The volume id of the item. - public static async Task<(long, string)> ProbeForMatchingFilename(string filename, LocalRecreateDatabase restoredb) + public static async Task<(long, string)> ProbeForMatchingFilename(string filename, LocalRecreateDatabase restoredb, CancellationToken cancellationToken) { var p = VolumeBase.ParseFilename(filename); if (p != null) @@ -799,7 +812,7 @@ namespace Duplicati.Library.Main.Operation { var testfilename = VolumeBase.GenerateFilename(p.FileType, p.Prefix, p.Guid, p.Time, compmodule, encmodule); var tvid = await restoredb - .GetRemoteVolumeID(testfilename) + .GetRemoteVolumeID(testfilename, cancellationToken) .ConfigureAwait(false); if (tvid >= 0) diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 1383e1fda..547abae58 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -66,11 +66,11 @@ namespace Duplicati.Library.Main.Operation try { await using var db = - await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache) + await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); knownRemotes = await db - .GetRemoteVolumes() - .CountAsync() + .GetRemoteVolumes(m_result.TaskControl.ProgressToken) + .CountAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } catch (Exception ex) @@ -134,50 +134,50 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(0); await using var db = - await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache) + await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache, null, cancellationToken) .ConfigureAwait(false); try { - await Utility.UpdateOptionsFromDb(db, m_options) + await Utility.UpdateOptionsFromDb(db, m_options, cancellationToken) .ConfigureAwait(false); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, cancellationToken) .ConfigureAwait(false); - if (await db.PartiallyRecreated().ConfigureAwait(false)) + if (await db.PartiallyRecreated(cancellationToken).ConfigureAwait(false)) throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated"); - if (await db.RepairInProgress().ConfigureAwait(false)) + if (await db.RepairInProgress(cancellationToken).ConfigureAwait(false)) throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsInRepairState"); // Ensure the database is consistent before we start fixing the remote await db - .VerifyConsistencyForRepair(m_options.Blocksize, m_options.BlockhashSize, true) + .VerifyConsistencyForRepair(m_options.Blocksize, m_options.BlockhashSize, true, cancellationToken) .ConfigureAwait(false); // If the last backup failed, guard the incomplete fileset, so we can create a synthetic filelist var lastTempFilelist = await db - .GetLastIncompleteFilesetVolume() + .GetLastIncompleteFilesetVolume(cancellationToken) .ConfigureAwait(false); - var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false); + var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced, cancellationToken).ConfigureAwait(false); await db.Transaction - .CommitAsync("CommitRemoteListAnalysisTransaction") + .CommitAsync("CommitRemoteListAnalysisTransaction", token: cancellationToken) .ConfigureAwait(false); var buffer = new byte[m_options.Blocksize]; var hashsize = m_options.BlockhashSize; var missingRemoteFilesets = await db - .MissingRemoteFilesets() + .MissingRemoteFilesets(cancellationToken) .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); var missingLocalFilesets = await db - .MissingLocalFilesets() + .MissingLocalFilesets(cancellationToken) .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); var emptyIndexFiles = await db - .EmptyIndexFiles() + .EmptyIndexFiles(cancellationToken) .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); @@ -192,12 +192,12 @@ namespace Duplicati.Library.Main.Operation // Find the most recent timestamp from either a fileset or a remote volume var mostRecentLocal = await db - .GetRemoteVolumes() + .GetRemoteVolumes(cancellationToken) .Where(x => x.Type == RemoteVolumeType.Files) .Select(x => VolumeBase.ParseFilename(x.Name).Time.ToLocalTime()) - .Concat(db.FilesetTimes().Select(x => x.Value.ToLocalTime())) + .Concat(db.FilesetTimes(cancellationToken).Select(x => x.Value.ToLocalTime())) .Append(DateTime.MinValue) - .MaxAsync() + .MaxAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); var mostRecentRemote = tp.ParsedVolumes.Select(x => x.Time.ToLocalTime()).Append(DateTime.MinValue).Max(); @@ -231,7 +231,7 @@ namespace Duplicati.Library.Main.Operation if (tp.VerificationRequiredVolumes.Any()) { await using var testdb = - await LocalTestDatabase.CreateAsync(db) + await LocalTestDatabase.CreateAsync(db, null, cancellationToken) .ConfigureAwait(false); foreach (var n in tp.VerificationRequiredVolumes) @@ -242,11 +242,11 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(testdb, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) await testdb.Transaction - .CommitAsync("CommitEarlyExit", false) + .CommitAsync("CommitEarlyExit", false, cancellationToken) .ConfigureAwait(false); else await testdb.Transaction - .RollBackAsync() + .RollBackAsync(cancellationToken) .ConfigureAwait(false); return; } @@ -258,7 +258,7 @@ namespace Duplicati.Library.Main.Operation var (tf, hash, size) = await backendManager.GetWithInfoAsync(n.Name, n.Hash, n.Size, cancellationToken).ConfigureAwait(false); using (tf) res = - await TestHandler.TestVolumeInternals(testdb, n, tf, m_options, 1) + await TestHandler.TestVolumeInternals(testdb, n, tf, m_options, 1, cancellationToken) .ConfigureAwait(false); if (res.Value.Any()) @@ -268,7 +268,7 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "CapturedRemoteFileHash", "Sucessfully captured hash for {0}, updating database", n.Name); await db - .UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash) + .UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash, cancellationToken) .ConfigureAwait(false); } @@ -281,7 +281,7 @@ namespace Duplicati.Library.Main.Operation } await db.Transaction - .CommitAsync("CommitVerificationTransaction") + .CommitAsync("CommitVerificationTransaction", true, cancellationToken) .ConfigureAwait(false); } @@ -294,11 +294,11 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) await db.Transaction - .CommitAsync("CommitEarlyExit", false) + .CommitAsync("CommitEarlyExit", false, cancellationToken) .ConfigureAwait(false); else await db.Transaction - .RollBackAsync() + .RollBackAsync(cancellationToken) .ConfigureAwait(false); return; } @@ -319,7 +319,7 @@ namespace Duplicati.Library.Main.Operation foreach (var rv in ifr.Volumes) { var entry = await db - .GetRemoteVolume(rv.Filename) + .GetRemoteVolume(rv.Filename, cancellationToken) .ConfigureAwait(false); if (entry.ID < 0) throw new Exception(string.Format("Unknown remote file {0} detected", rv.Filename)); @@ -331,44 +331,44 @@ namespace Duplicati.Library.Main.Operation throw new Exception(string.Format("Volume {0} hash/size mismatch ({1} - {2}) vs ({3} - {4})", rv.Filename, entry.Hash, entry.Size, rv.Hash, rv.Length)); await db - .CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks) + .CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks, cancellationToken) .ConfigureAwait(false); } var blocksize = m_options.Blocksize; foreach (var ixb in ifr.BlockLists) await db - .CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize) + .CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize, cancellationToken) .ConfigureAwait(false); // Register the new index file and link it to the block files var selfid = await db - .RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, RemoteVolumeState.Uploading, size, new TimeSpan(0)) + .RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, RemoteVolumeState.Uploading, size, new TimeSpan(0), cancellationToken) .ConfigureAwait(false); foreach (var rv in ifr.Volumes) { // Guard against unknown block files long id = await db - .GetRemoteVolumeID(rv.Filename) + .GetRemoteVolumeID(rv.Filename, cancellationToken) .ConfigureAwait(false); if (id == -1) Logging.Log.WriteWarningMessage(LOGTAG, "UnknownBlockFile", null, "Index file {0} references unknown block file: {1}", n.File.Name, rv.Filename); else await db - .AddIndexBlockLink(selfid, id) + .AddIndexBlockLink(selfid, id, cancellationToken) .ConfigureAwait(false); } if (!m_options.Dryrun) await db.Transaction - .CommitAsync("CommitIndexFileTransaction") + .CommitAsync("CommitIndexFileTransaction", true, cancellationToken) .ConfigureAwait(false); } // All checks fine, we accept the new index file Logging.Log.WriteInformationMessage(LOGTAG, "AcceptNewIndexFile", "Accepting new index file {0}", n.File.Name); await db - .UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash) + .UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash, cancellationToken) .ConfigureAwait(false); continue; @@ -382,7 +382,7 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) { await db - .RegisterRemoteVolume(n.File.Name, n.FileType, n.File.Size, RemoteVolumeState.Deleting) + .RegisterRemoteVolume(n.File.Name, n.FileType, n.File.Size, RemoteVolumeState.Deleting, cancellationToken) .ConfigureAwait(false); await backendManager.DeleteAsync(n.File.Name, n.File.Size, false, cancellationToken).ConfigureAwait(false); } @@ -408,11 +408,11 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) await db.Transaction - .CommitAsync("CommitEarlyExit", false) + .CommitAsync("CommitEarlyExit", false, cancellationToken) .ConfigureAwait(false); else await db.Transaction - .RollBackAsync() + .RollBackAsync(cancellationToken) .ConfigureAwait(false); return; } @@ -432,17 +432,17 @@ namespace Duplicati.Library.Main.Operation fsw.CreateFilesetFile(isfull); await db - .WriteFileset(fsw, filesetId) + .WriteFileset(fsw, filesetId, cancellationToken) .ConfigureAwait(false); fsw.Close(); fsw.VolumeID = await db - .RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, -1, RemoteVolumeState.Temporary) + .RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, -1, RemoteVolumeState.Temporary, cancellationToken) .ConfigureAwait(false); await db - .LinkFilesetToVolume(filesetId, fsw.VolumeID) + .LinkFilesetToVolume(filesetId, fsw.VolumeID, cancellationToken) .ConfigureAwait(false); if (m_options.Dryrun) @@ -460,39 +460,39 @@ namespace Duplicati.Library.Main.Operation await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) await db.Transaction - .CommitAsync("CommitFilesetTransaction") + .CommitAsync("CommitFilesetTransaction", token: cancellationToken) .ConfigureAwait(false); } foreach (var volumename in missingLocalFilesets) { var remoteVolume = await db - .GetRemoteVolume(volumename) + .GetRemoteVolume(volumename, cancellationToken) .ConfigureAwait(false); using var tmpfile = await backendManager.GetAsync(remoteVolume.Name, remoteVolume.Hash, remoteVolume.Size, cancellationToken).ConfigureAwait(false); var parsed = VolumeBase.ParseFilename(remoteVolume.Name); using var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read); using var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions); - await using var recreatedb = await LocalRecreateDatabase.CreateAsync(db, m_options).ConfigureAwait(false); + await using var recreatedb = await LocalRecreateDatabase.CreateAsync(db, m_options, null, cancellationToken).ConfigureAwait(false); if (compressor == null) throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); var filesetid = await db - .CreateFileset(remoteVolume.ID, parsed.Time) + .CreateFileset(remoteVolume.ID, parsed.Time, cancellationToken) .ConfigureAwait(false); - await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, compressor, filesetid, m_options, new FilterExpression()) + await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, compressor, filesetid, m_options, new FilterExpression(), cancellationToken) .ConfigureAwait(false); if (!m_options.Dryrun) await recreatedb.Transaction - .CommitAsync("CommitRecreateFilesetTransaction") + .CommitAsync("CommitRecreateFilesetTransaction", token: cancellationToken) .ConfigureAwait(false); } if (!m_options.Dryrun && tp.MissingVolumes.Any()) await db - .TerminatedWithActiveUploads(true) + .TerminatedWithActiveUploads(cancellationToken, true) .ConfigureAwait(false); if (tp.MissingVolumes.Any(x => x.Type != RemoteVolumeType.Index && x.Type != RemoteVolumeType.Files && x.Type != RemoteVolumeType.Blocks)) @@ -548,7 +548,7 @@ namespace Duplicati.Library.Main.Operation try { // Check if the index file has already been deleted, beause the dblock was recreated - var currentState = (await db.GetRemoteVolume(n.Name).ConfigureAwait(false)).State; + var currentState = (await db.GetRemoteVolume(n.Name, cancellationToken).ConfigureAwait(false)).State; if (currentState == RemoteVolumeState.Deleted || currentState == RemoteVolumeState.Temporary || currentState == RemoteVolumeState.Deleting) continue; @@ -577,7 +577,7 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); // Implicit rollback await db.Transaction - .RollBackAsync() + .RollBackAsync(cancellationToken) .ConfigureAwait(false); return; } @@ -604,7 +604,7 @@ namespace Duplicati.Library.Main.Operation // FlushPendingMessagesAsync, but maybe it // is not needed anymore. await db.Transaction - .CommitAsync() + .CommitAsync(cancellationToken) .ConfigureAwait(false); } } @@ -628,23 +628,23 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) { await db - .TerminatedWithActiveUploads(false) + .TerminatedWithActiveUploads(cancellationToken, false) .ConfigureAwait(false); await db.Transaction - .CommitAsync("CommitRepairTransaction") + .CommitAsync("CommitRepairTransaction", true, cancellationToken) .ConfigureAwait(false); } else { await db.Transaction - .RollBackAsync("RollbackRepairTransaction") + .RollBackAsync("RollbackRepairTransaction", true, cancellationToken) .ConfigureAwait(false); } } catch (Exception) { // Implicit rollback - await db.Transaction.RollBackAsync().ConfigureAwait(false); + await db.Transaction.RollBackAsync(cancellationToken).ConfigureAwait(false); throw; } } @@ -662,10 +662,10 @@ namespace Duplicati.Library.Main.Operation private async Task RunRepairDlist(IBackendManager backendManager, LocalRepairDatabase db, FilesetVolumeWriter volumeWriter, RemoteVolumeEntry originalVolume, DateTime filesetTime, CancellationToken cancellationToken) { volumeWriter.VolumeID = await db - .RegisterRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, -1, TimeSpan.Zero) + .RegisterRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, cancellationToken) .ConfigureAwait(false); (var prevFilesetId, var _, var isPrevFull) = await db - .GetFilesetFromRemotename(originalVolume.Name) + .GetFilesetFromRemotename(originalVolume.Name, cancellationToken) .ConfigureAwait(false); if (!string.IsNullOrEmpty(m_options.ControlFiles)) @@ -674,28 +674,28 @@ namespace Duplicati.Library.Main.Operation volumeWriter.CreateFilesetFile(isPrevFull); var newFilesetID = await db - .CreateFileset(volumeWriter.VolumeID, filesetTime) + .CreateFileset(volumeWriter.VolumeID, filesetTime, cancellationToken) .ConfigureAwait(false); await db - .UpdateFullBackupStateInFileset(newFilesetID, isPrevFull) + .UpdateFullBackupStateInFileset(newFilesetID, isPrevFull, cancellationToken) .ConfigureAwait(false); await db - .LinkFilesetToVolume(newFilesetID, volumeWriter.VolumeID) + .LinkFilesetToVolume(newFilesetID, volumeWriter.VolumeID, cancellationToken) .ConfigureAwait(false); await db - .MoveFilesFromFileset(newFilesetID, prevFilesetId) + .MoveFilesFromFileset(newFilesetID, prevFilesetId, cancellationToken) .ConfigureAwait(false); await db - .WriteFileset(volumeWriter, newFilesetID) + .WriteFileset(volumeWriter, newFilesetID, cancellationToken) .ConfigureAwait(false); volumeWriter.Close(); await db - .UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null) + .UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null, cancellationToken) .ConfigureAwait(false); await db - .UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null) + .UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, cancellationToken) .ConfigureAwait(false); await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); @@ -704,7 +704,7 @@ namespace Duplicati.Library.Main.Operation else { await db.Transaction - .CommitAsync("CommitPriorToFilesetUpload") + .CommitAsync("CommitPriorToFilesetUpload", true, cancellationToken) .ConfigureAwait(false); await backendManager.PutAsync(volumeWriter, null, null, false, null, cancellationToken).ConfigureAwait(false); } @@ -712,7 +712,7 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) await db.Transaction - .CommitAsync("CommitFilesetTransaction") + .CommitAsync("CommitFilesetTransaction", true, cancellationToken) .ConfigureAwait(false); } @@ -729,26 +729,26 @@ namespace Duplicati.Library.Main.Operation public static async Task RunRepairDindex(IBackendManager backendManager, LocalRepairDatabase db, IndexVolumeWriter indexWriter, IRemoteVolume originalVolume, Options options, CancellationToken cancellationToken) { indexWriter.VolumeID = await db - .RegisterRemoteVolume(indexWriter.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Uploading, -1, TimeSpan.Zero) + .RegisterRemoteVolume(indexWriter.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Uploading, -1, TimeSpan.Zero, cancellationToken) .ConfigureAwait(false); var blockvolumeids = new List(); using var h = HashFactory.CreateHasher(options.BlockHashAlgorithm); - await foreach (var blockvolume in db.GetBlockVolumesFromIndexName(originalVolume.Name).ConfigureAwait(false)) + await foreach (var blockvolume in db.GetBlockVolumesFromIndexName(originalVolume.Name, cancellationToken).ConfigureAwait(false)) { indexWriter.StartVolume(blockvolume.Name); var volumeid = await db - .GetRemoteVolumeID(blockvolume.Name) + .GetRemoteVolumeID(blockvolume.Name, cancellationToken) .ConfigureAwait(false); - await foreach (var b in db.GetBlocks(volumeid).ConfigureAwait(false)) + await foreach (var b in db.GetBlocks(volumeid, cancellationToken).ConfigureAwait(false)) indexWriter.AddBlock(b.Hash, b.Size); indexWriter.FinishVolume(blockvolume.Hash, blockvolume.Size); blockvolumeids.Add(volumeid); if (options.IndexfilePolicy == Options.IndexFileStrategy.Full) - await foreach (var b in db.GetBlocklists(volumeid, options.Blocksize, options.BlockhashSize).ConfigureAwait(false)) + await foreach (var b in db.GetBlocklists(volumeid, options.Blocksize, options.BlockhashSize, cancellationToken).ConfigureAwait(false)) { var bh = Convert.ToBase64String(h.ComputeHash(b.Buffer, 0, b.Size)); if (bh != b.Hash) @@ -760,7 +760,7 @@ namespace Duplicati.Library.Main.Operation foreach (var blockvolumeid in blockvolumeids) await db - .AddIndexBlockLink(indexWriter.VolumeID, blockvolumeid) + .AddIndexBlockLink(indexWriter.VolumeID, blockvolumeid, cancellationToken) .ConfigureAwait(false); indexWriter.Close(); @@ -770,7 +770,7 @@ namespace Duplicati.Library.Main.Operation else { await db - .UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null) + .UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, cancellationToken) .ConfigureAwait(false); await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); await backendManager.PutAsync(indexWriter, null, null, false, null, cancellationToken).ConfigureAwait(false); @@ -779,7 +779,7 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!options.Dryrun) await db.Transaction - .CommitAsync("CommitRepairTransaction") + .CommitAsync("CommitRepairTransaction", true, cancellationToken) .ConfigureAwait(false); } @@ -835,12 +835,13 @@ namespace Duplicati.Library.Main.Operation /// /// The options for the current operation. /// The database to use for registering the initial volume. + /// Cancellation token to cancel the operation. /// A task that completes with the new instance of . - public static async Task CreateAsync(Options options, LocalDatabase db) + public static async Task CreateAsync(Options options, LocalDatabase db, CancellationToken cancellationToken) { var ipdv = new InProgressDblockVolumes(options, db); ipdv.m_activeWriter.VolumeID = await db - .RegisterRemoteVolume(ipdv.m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero) + .RegisterRemoteVolume(ipdv.m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, cancellationToken) .ConfigureAwait(false); return ipdv; } @@ -879,21 +880,24 @@ namespace Duplicati.Library.Main.Operation /// The offset in the buffer where the block data starts. /// The size of the block data. /// The compression hint for the block. - public async Task AddBlock(string hash, byte[] buffer, int offset, int size, CompressionHint hint) + /// Cancellation token to cancel the operation. + /// A task that completes when the block is added. + public async Task AddBlock(string hash, byte[] buffer, int offset, int size, CompressionHint hint, CancellationToken cancellationToken) { await m_activeWriter .AddBlock(hash, buffer, offset, size, hint) .ConfigureAwait(false); m_anyData = true; if (IsFull) - await StartNewVolume().ConfigureAwait(false); + await StartNewVolume(cancellationToken).ConfigureAwait(false); } /// /// Starts a new volume for the current operation. /// + /// Cancellation token to cancel the operation. /// A task that completes when the new volume is started. - public async Task StartNewVolume() + public async Task StartNewVolume(CancellationToken cancellationToken) { if (!m_anyData) return; @@ -902,7 +906,7 @@ namespace Duplicati.Library.Main.Operation m_activeWriter = new BlockVolumeWriter(m_options); m_activeWriter.VolumeID = await m_database - .RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero) + .RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, cancellationToken) .ConfigureAwait(false); m_anyData = false; } @@ -920,7 +924,7 @@ namespace Duplicati.Library.Main.Operation private async Task RunRepairDblocks(IBackendManager backendManager, LocalRepairDatabase db, IEnumerable missingDblockFiles, Action incrementProgress, CancellationToken cancellationToken) { var currentVolume = - await InProgressDblockVolumes.CreateAsync(m_options, db) + await InProgressDblockVolumes.CreateAsync(m_options, db, cancellationToken) .ConfigureAwait(false); foreach (var n in missingDblockFiles) @@ -932,7 +936,7 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) await db.Transaction - .CommitAsync("CommitEarlyExit", false) + .CommitAsync("CommitEarlyExit", false, cancellationToken) .ConfigureAwait(false); return; } @@ -953,13 +957,13 @@ namespace Duplicati.Library.Main.Operation // If we have any data in the volume, complete it so it will be uploaded if (currentVolume.AnyData) - await currentVolume.StartNewVolume().ConfigureAwait(false); + await currentVolume.StartNewVolume(cancellationToken).ConfigureAwait(false); // Complete any pending uploads await UploadCompletedBlockVolumes(backendManager, currentVolume, db, cancellationToken).ConfigureAwait(false); // After reset, we need to remove the temporary volume from the database await db - .RemoveRemoteVolume(currentVolume.RemoteFilename) + .RemoveRemoteVolume(currentVolume.RemoteFilename, cancellationToken) .ConfigureAwait(false); } @@ -976,7 +980,7 @@ namespace Duplicati.Library.Main.Operation foreach (var completedVolume in activeVolume.CompletedWriters) { await db - .UpdateRemoteVolume(completedVolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null) + .UpdateRemoteVolume(completedVolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, cancellationToken) .ConfigureAwait(false); // Create a new index file that points to the new volume @@ -986,19 +990,19 @@ namespace Duplicati.Library.Main.Operation { newvolindex = new IndexVolumeWriter(m_options); newvolindex.VolumeID = await db - .RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary) + .RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, cancellationToken) .ConfigureAwait(false); newvolindex.StartVolume(completedVolume.RemoteFilename); - await foreach (var b in db.GetBlocks(completedVolume.VolumeID).ConfigureAwait(false)) + await foreach (var b in db.GetBlocks(completedVolume.VolumeID, cancellationToken).ConfigureAwait(false)) newvolindex.AddBlock(b.Hash, b.Size); await db - .AddIndexBlockLink(newvolindex.VolumeID, completedVolume.VolumeID) + .AddIndexBlockLink(newvolindex.VolumeID, completedVolume.VolumeID, cancellationToken) .ConfigureAwait(false); if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) indexVolumeFinished = async () => { - await foreach (var blocklist in db.GetBlocklists(completedVolume.VolumeID, m_options.Blocksize, m_options.BlockhashSize).ConfigureAwait(false)) + await foreach (var blocklist in db.GetBlocklists(completedVolume.VolumeID, m_options.Blocksize, m_options.BlockhashSize, cancellationToken).ConfigureAwait(false)) newvolindex.WriteBlocklist(blocklist.Item1, blocklist.Item2, 0, blocklist.Item3); }; } @@ -1010,7 +1014,7 @@ namespace Duplicati.Library.Main.Operation else { await db.Transaction - .CommitAsync("PostRepairPreUploadBlockVolume") + .CommitAsync("PostRepairPreUploadBlockVolume", token: cancellationToken) .ConfigureAwait(false); await backendManager.PutAsync(completedVolume, newvolindex, indexVolumeFinished, false, null, cancellationToken).ConfigureAwait(false); } @@ -1022,13 +1026,13 @@ namespace Duplicati.Library.Main.Operation // Prepare for deleting the old stuff foreach (var vol in activeVolume.VolumesToDelete) await db - .UpdateRemoteVolume(vol.Name, RemoteVolumeState.Deleting, vol.Size, vol.Hash, false, TimeSpan.FromHours(2), null) + .UpdateRemoteVolume(vol.Name, RemoteVolumeState.Deleting, vol.Size, vol.Hash, false, TimeSpan.FromHours(2), null, cancellationToken) .ConfigureAwait(false); // Persist desired state prior to deleting the old files if (!m_options.Dryrun) await db.Transaction - .CommitAsync("PostRepairBlockVolume") + .CommitAsync("PostRepairBlockVolume", token: cancellationToken) .ConfigureAwait(false); // Delete the old files @@ -1042,7 +1046,7 @@ namespace Duplicati.Library.Main.Operation await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) await db.Transaction - .CommitAsync("PostRepairBlockVolume") + .CommitAsync("PostRepairBlockVolume", token: cancellationToken) .ConfigureAwait(false); activeVolume.CompletedWriters.Clear(); @@ -1064,18 +1068,18 @@ namespace Duplicati.Library.Main.Operation // as data can be either file contents, metadata or blocklist hashes // We attempt to recover all three source parts in the steps below await using var mbl = await db - .CreateBlockList(originalVolume.Name) + .CreateBlockList(originalVolume.Name, cancellationToken) .ConfigureAwait(false); var originalMissingBlockCount = await mbl - .GetMissingBlockCount() + .GetMissingBlockCount(cancellationToken) .ConfigureAwait(false); var recoveredSourceBlocks = 0L; // First we grab all known blocks from local files string lastRestoredHash = null; long lastRestoredSize = -1; - await foreach (var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize).ConfigureAwait(false)) + await foreach (var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize, cancellationToken).ConfigureAwait(false)) { if (block.Hash == lastRestoredHash && block.Size == lastRestoredSize) continue; @@ -1111,11 +1115,11 @@ namespace Duplicati.Library.Main.Operation // Found it, no need to look again lastRestoredHash = block.Hash; lastRestoredSize = block.Size; - if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID).ConfigureAwait(false)) + if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID, cancellationToken).ConfigureAwait(false)) { recoveredSourceBlocks++; await pendingVolume - .AddBlock(block.Hash, buffer, 0, size, CompressionHint.Default) + .AddBlock(block.Hash, buffer, 0, size, CompressionHint.Default, cancellationToken) .ConfigureAwait(false); } } @@ -1126,7 +1130,7 @@ namespace Duplicati.Library.Main.Operation } // Then grab all blocks with metadata - await foreach (var block in mbl.GetSourceItemsWithMetadataBlocks().ConfigureAwait(false)) + await foreach (var block in mbl.GetSourceItemsWithMetadataBlocks(cancellationToken).ConfigureAwait(false)) { if (block.Hash == lastRestoredHash && block.Size == lastRestoredSize) continue; @@ -1162,11 +1166,11 @@ namespace Duplicati.Library.Main.Operation // Found it, no need to look again lastRestoredHash = block.Hash; lastRestoredSize = block.Size; - if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID).ConfigureAwait(false)) + if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID, cancellationToken).ConfigureAwait(false)) { recoveredSourceBlocks++; await pendingVolume - .AddBlock(block.Hash, metahash.Blob, 0, metahash.Blob.Length, CompressionHint.Default) + .AddBlock(block.Hash, metahash.Blob, 0, metahash.Blob.Length, CompressionHint.Default, cancellationToken) .ConfigureAwait(false); } } @@ -1201,9 +1205,9 @@ namespace Duplicati.Library.Main.Operation if (resulthash == lastBlocklist.BlocklistHash && targetsize == blocklistHash.Length) { - if (await mbl.SetBlockRestored(resulthash, blocklistHash.Length, pendingVolume.VolumeID).ConfigureAwait(false)) + if (await mbl.SetBlockRestored(resulthash, blocklistHash.Length, pendingVolume.VolumeID, cancellationToken).ConfigureAwait(false)) await pendingVolume - .AddBlock(resulthash, blocklistHash.ToArray(), 0, (int)blocklistHash.Length, CompressionHint.Default) + .AddBlock(resulthash, blocklistHash.ToArray(), 0, (int)blocklistHash.Length, CompressionHint.Default, cancellationToken) .ConfigureAwait(false); } else @@ -1214,7 +1218,7 @@ namespace Duplicati.Library.Main.Operation blocklistHash.SetLength(0); } - await foreach (var blocklist in mbl.GetBlocklistHashes(hashesPerBlock).ConfigureAwait(false)) + await foreach (var blocklist in mbl.GetBlocklistHashes(hashesPerBlock, cancellationToken).ConfigureAwait(false)) { lastBlocklist ??= blocklist; @@ -1231,7 +1235,7 @@ namespace Duplicati.Library.Main.Operation await EmitBlockListBlock().ConfigureAwait(false); //Then we grab all remote volumes that have the missing blocks - await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(await mbl.GetMissingBlockSources().ToListAsync(cancellationToken: cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false)) + await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(await mbl.GetMissingBlockSources(cancellationToken).ToListAsync(cancellationToken: cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false)) { try { @@ -1239,10 +1243,10 @@ namespace Duplicati.Library.Main.Operation using (tmpfile) using (var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options)) foreach (var b in f.Blocks) - if (await mbl.SetBlockRestored(b.Key, b.Value, pendingVolume.VolumeID).ConfigureAwait(false)) + if (await mbl.SetBlockRestored(b.Key, b.Value, pendingVolume.VolumeID, cancellationToken).ConfigureAwait(false)) if (f.ReadBlock(b.Key, buffer) == b.Value) await pendingVolume - .AddBlock(b.Key, buffer, 0, (int)b.Value, CompressionHint.Default) + .AddBlock(b.Key, buffer, 0, (int)b.Value, CompressionHint.Default, cancellationToken) .ConfigureAwait(false); } catch (Exception ex) @@ -1253,13 +1257,13 @@ namespace Duplicati.Library.Main.Operation } var missingBlocks = await mbl - .GetMissingBlockCount() + .GetMissingBlockCount(cancellationToken) .ConfigureAwait(false); var recoveredBlocks = originalMissingBlockCount - missingBlocks; if (recoveredBlocks == 0 || (m_options.DisablePartialDblockRecovery && missingBlocks > 0)) { Logging.Log.WriteInformationMessage(LOGTAG, "RepairMissingBlocks", "Repair cannot acquire {0} required blocks for volume {1}, which are required by the following filesets: ", missingBlocks, originalVolume.Name); - await foreach (var f in mbl.GetFilesetsUsingMissingBlocks().ConfigureAwait(false)) + await foreach (var f in mbl.GetFilesetsUsingMissingBlocks(cancellationToken).ConfigureAwait(false)) Logging.Log.WriteInformationMessage(LOGTAG, "AffectedFilesetName", f.Name); var recoverymsg = string.Format("If you want to continue working with the database, you can use the \"{0}\" and \"{1}\" commands to purge the missing data from the database and the remote storage.", "list-broken-files", "purge-broken-files"); @@ -1280,7 +1284,7 @@ namespace Duplicati.Library.Main.Operation { pendingVolume.VolumesToDelete.Add(originalVolume); var oldIndexFiles = await db - .GetIndexFilesReferencingBlockFile(originalVolume.ID) + .GetIndexFilesReferencingBlockFile(originalVolume.ID, cancellationToken) .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); @@ -1288,12 +1292,12 @@ namespace Duplicati.Library.Main.Operation foreach (var oldIndexFile in oldIndexFiles) { var oldVolume = await db - .GetRemoteVolume(oldIndexFile) + .GetRemoteVolume(oldIndexFile, cancellationToken) .ConfigureAwait(false); if (oldVolume.State == RemoteVolumeState.Uploading || oldVolume.State == RemoteVolumeState.Uploaded || oldVolume.State == RemoteVolumeState.Verified) { var blockVolumesReferenced = await db - .GetBlockVolumesFromIndexName(oldIndexFile) + .GetBlockVolumesFromIndexName(oldIndexFile, cancellationToken) .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); if (blockVolumesReferenced.Any(x => x.Name != originalVolume.Name)) @@ -1322,12 +1326,12 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); await using var db = - await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache) + await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var sets = await db - .GetFilesetsWithMissingFiles() - .ToListAsync() + .GetFilesetsWithMissingFiles(m_result.TaskControl.ProgressToken) + .ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (sets.Count == 0) @@ -1340,7 +1344,7 @@ namespace Duplicati.Library.Main.Operation ix++; Logging.Log.WriteInformationMessage(LOGTAG, "RepairingBrokenFileset", "Repairing broken fileset {0} of {1}: {2}", ix, sets.Count, entry.Value); var volume = await db - .GetRemoteVolumeFromFilesetID(entry.Key) + .GetRemoteVolumeFromFilesetID(entry.Key, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var parsed = VolumeBase.ParseFilename(volume.Name); using var tmpfile = await backendManager.GetAsync(volume.Name, volume.Hash, volume.Size, CancellationToken.None).ConfigureAwait(false); @@ -1350,15 +1354,17 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); // Clear out the old fileset - await db.DeleteFilesetEntries(entry.Key).ConfigureAwait(false); - await using (var rdb = await LocalRecreateDatabase.CreateAsync(db, m_options).ConfigureAwait(false)) + await db + .DeleteFilesetEntries(entry.Key, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + await using (var rdb = await LocalRecreateDatabase.CreateAsync(db, m_options, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { - await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(rdb, compressor, entry.Key, m_options, new FilterExpression()) + await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(rdb, compressor, entry.Key, m_options, new FilterExpression(), m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } await db.Transaction - .CommitAsync("PostRepairFileset") + .CommitAsync("PostRepairFileset", true, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } } @@ -1377,22 +1383,22 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(0); await using var db = - await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache) + await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - await Utility.UpdateOptionsFromDb(db, m_options) + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - if (await db.RepairInProgress().ConfigureAwait(false) || await db.PartiallyRecreated().ConfigureAwait(false)) + if (await db.RepairInProgress(m_result.TaskControl.ProgressToken).ConfigureAwait(false) || await db.PartiallyRecreated(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) Logging.Log.WriteWarningMessage(LOGTAG, "InProgressDatabase", null, "The database is marked as \"in-progress\" and may be incomplete."); - await db.FixDuplicateMetahash().ConfigureAwait(false); - await db.FixDuplicateFileentries().ConfigureAwait(false); + await db.FixDuplicateMetahash(m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await db.FixDuplicateFileentries(m_result.TaskControl.ProgressToken).ConfigureAwait(false); await db - .FixDuplicateBlocklistHashes(m_options.Blocksize, m_options.BlockhashSize) + .FixDuplicateBlocklistHashes(m_options.Blocksize, m_options.BlockhashSize, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await db - .FixMissingBlocklistHashes(m_options.BlockHashAlgorithm, m_options.Blocksize) + .FixMissingBlocklistHashes(m_options.BlockHashAlgorithm, m_options.Blocksize, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } } diff --git a/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs b/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs index d3d18639e..80658a033 100644 --- a/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs @@ -46,7 +46,14 @@ namespace Duplicati.Library.Main.Operation Directory.CreateDirectory(m_options.Restorepath); using var tmpdb = new TempFile(); - await using var db = await Database.LocalDatabase.CreateLocalDatabaseAsync(File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles", true, m_options.SqlitePageCache).ConfigureAwait(false); + await using var db = await Database.LocalDatabase.CreateLocalDatabaseAsync( + File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles", + true, + m_options.SqlitePageCache, + null, + m_result.TaskControl.ProgressToken + ).ConfigureAwait(false); + var filter = JoinedFilterExpression.Join(new FilterExpression(filterstrings), compositefilter); try @@ -68,7 +75,7 @@ namespace Duplicati.Library.Main.Operation var file = fileversion.Value.File; var entry = await db - .GetRemoteVolume(file.Name) + .GetRemoteVolume(file.Name, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var res = new List(); diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index c6c9af53e..6d3b26311 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -115,7 +115,7 @@ namespace Duplicati.Library.Main.Operation { if (!m_options.NoLocalDb && SystemIO.IO_OS.FileExists(m_options.Dbpath)) { - db = await LocalRestoreDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache) + db = await LocalRestoreDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } else @@ -123,7 +123,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, building a temporary database"); tmpdb = new TempFile(); RecreateDatabaseHandler.NumberedFilterFilelistDelegate filelistfilter = FilterNumberedFilelist(m_options.Time, m_options.Version); - db = await LocalRestoreDatabase.CreateAsync(tmpdb, m_options.SqlitePageCache) + db = await LocalRestoreDatabase.CreateAsync(tmpdb, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); m_result.RecreateDatabaseResults = new RecreateDatabaseResults(m_result); using (new Logging.Timer(LOGTAG, "RecreateTempDbForRestore", "Recreate temporary database for restore")) @@ -152,16 +152,16 @@ namespace Duplicati.Library.Main.Operation } } - private static async Task PatchWithBlocklist(LocalRestoreDatabase database, BlockVolumeReader blocks, Options options, RestoreResults result, byte[] blockbuffer, RestoreHandlerMetadataStorage metadatastorage) + private static async Task PatchWithBlocklist(LocalRestoreDatabase database, BlockVolumeReader blocks, Options options, RestoreResults result, byte[] blockbuffer, RestoreHandlerMetadataStorage metadatastorage, CancellationToken cancellationToken) { var blocksize = options.Blocksize; var updateCounter = 0L; var fullblockverification = options.FullBlockVerification; using var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm); - await using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false); - await using var volumekeeper = await database.GetMissingBlockData(blocks, options.Blocksize).ConfigureAwait(false); - await foreach (var restorelist in volumekeeper.FilesWithMissingBlocks().ConfigureAwait(false)) + await using var blockmarker = await database.CreateBlockMarkerAsync(cancellationToken).ConfigureAwait(false); + await using var volumekeeper = await database.GetMissingBlockData(blocks, options.Blocksize, cancellationToken).ConfigureAwait(false); + await foreach (var restorelist in volumekeeper.FilesWithMissingBlocks(cancellationToken).ConfigureAwait(false)) { var targetpath = restorelist.Path; @@ -185,7 +185,7 @@ namespace Duplicati.Library.Main.Operation // TODO: Much faster if we iterate the volume and checks what blocks are used, // because the compressors usually like sequential reading using (var file = SystemIO.IO_OS.FileOpenWrite(targetpath)) - await foreach (var targetblock in restorelist.Blocks().ConfigureAwait(false)) + await foreach (var targetblock in restorelist.Blocks(cancellationToken).ConfigureAwait(false)) { file.Position = targetblock.Offset; var size = blocks.ReadBlock(targetblock.Key, blockbuffer); @@ -205,7 +205,7 @@ namespace Duplicati.Library.Main.Operation { file.Write(blockbuffer, 0, size); await blockmarker - .SetBlockRestored(restorelist.FileID, targetblock.Offset / blocksize, targetblock.Key, size, false) + .SetBlockRestored(restorelist.FileID, targetblock.Offset / blocksize, targetblock.Key, size, false, cancellationToken) .ConfigureAwait(false); } } @@ -217,7 +217,7 @@ namespace Duplicati.Library.Main.Operation if ((++updateCounter) % 20 == 0) await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) + .UpdateProcessed(result.OperationProgressUpdater, cancellationToken) .ConfigureAwait(false); } catch (Exception ex) @@ -231,7 +231,7 @@ namespace Duplicati.Library.Main.Operation if (!options.SkipMetadata) { - await foreach (var restoremetadata in volumekeeper.MetadataWithMissingBlocks().ConfigureAwait(false)) + await foreach (var restoremetadata in volumekeeper.MetadataWithMissingBlocks(cancellationToken).ConfigureAwait(false)) { var targetpath = restoremetadata.Path; Logging.Log.WriteVerboseMessage(LOGTAG, "RecordingMetadata", "Recording metadata from remote data: {0}", targetpath); @@ -241,7 +241,7 @@ namespace Duplicati.Library.Main.Operation // TODO: When we support multi-block metadata this needs to deal with it using (var ms = new System.IO.MemoryStream()) { - await foreach (var targetblock in restoremetadata.Blocks().ConfigureAwait(false)) + await foreach (var targetblock in restoremetadata.Blocks(cancellationToken).ConfigureAwait(false)) { ms.Position = targetblock.Offset; var size = blocks.ReadBlock(targetblock.Key, blockbuffer); @@ -249,7 +249,7 @@ namespace Duplicati.Library.Main.Operation { ms.Write(blockbuffer, 0, size); await blockmarker - .SetBlockRestored(restoremetadata.FileID, targetblock.Offset / blocksize, targetblock.Key, size, true) + .SetBlockRestored(restoremetadata.FileID, targetblock.Offset / blocksize, targetblock.Key, size, true, cancellationToken) .ConfigureAwait(false); } } @@ -268,9 +268,9 @@ namespace Duplicati.Library.Main.Operation } } await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) + .UpdateProcessed(result.OperationProgressUpdater, cancellationToken) .ConfigureAwait(false); - await blockmarker.CommitAsync().ConfigureAwait(false); + await blockmarker.CommitAsync(cancellationToken).ConfigureAwait(false); } private static void ApplyStoredMetadata(Options options, RestoreHandlerMetadataStorage metadatastorage) @@ -316,16 +316,16 @@ namespace Duplicati.Library.Main.Operation private async Task DoRunNewAsync(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter filter, CancellationToken cancellationToken) { // Perform initial setup - await Utility.UpdateOptionsFromDb(database, m_options) + await Utility.UpdateOptionsFromDb(database, m_options, cancellationToken) .ConfigureAwait(false); - await Utility.VerifyOptionsAndUpdateDatabase(database, m_options) + await Utility.VerifyOptionsAndUpdateDatabase(database, m_options, cancellationToken) .ConfigureAwait(false); // Verify the backend if necessary if (!m_options.NoBackendverification) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify); - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, cancellationToken).ConfigureAwait(false); } // Prepare the block and file list and create the directory structure @@ -339,7 +339,7 @@ namespace Duplicati.Library.Main.Operation // At this point, there should be no more writes to the database, so we have to unlock the database: await database.Transaction - .CommitAsync("CommitBeforeRestore") + .CommitAsync("CommitBeforeRestore", token: cancellationToken) .ConfigureAwait(false); using var setup_log_timer = new Logging.Timer(LOGTAG, "RestoreNetworkSetup", "RestoreNetworkSetup"); @@ -391,7 +391,7 @@ namespace Duplicati.Library.Main.Operation } await database.Transaction - .CommitAsync("CommitAfterRestore") + .CommitAsync("CommitAfterRestore", token: cancellationToken) .ConfigureAwait(false); await database.DisposePoolAsync().ConfigureAwait(false); @@ -422,7 +422,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesRestored", null, "Restore completed without errors but no files were restored"); // Drop the temp tables - await database.DropRestoreTable().ConfigureAwait(false); + await database.DropRestoreTable(cancellationToken).ConfigureAwait(false); await backendManager.WaitForEmptyAsync(database, m_result.TaskControl.ProgressToken).ConfigureAwait(false); // Report that the restore is complete @@ -443,16 +443,16 @@ namespace Duplicati.Library.Main.Operation //using (var database = new LocalRestoreDatabase(dbparent)) using (var metadatastorage = new RestoreHandlerMetadataStorage()) { - await Utility.UpdateOptionsFromDb(database, m_options) + await Utility.UpdateOptionsFromDb(database, m_options, cancellationToken) .ConfigureAwait(false); - await Utility.VerifyOptionsAndUpdateDatabase(database, m_options) + await Utility.VerifyOptionsAndUpdateDatabase(database, m_options, cancellationToken) .ConfigureAwait(false); m_blockbuffer = new byte[m_options.Blocksize]; if (!m_options.NoBackendverification) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify); - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, cancellationToken).ConfigureAwait(false); } //Figure out what files are to be patched, and what blocks are needed @@ -510,8 +510,8 @@ namespace Duplicati.Library.Main.Operation List volumes; using (new Logging.Timer(LOGTAG, "GetMissingVolumes", "GetMissingVolumes")) volumes = await database - .GetMissingVolumes() - .ToListAsync() + .GetMissingVolumes(cancellationToken) + .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); if (volumes.Count > 0) @@ -535,7 +535,7 @@ namespace Duplicati.Library.Main.Operation using (tmpfile) using (var blocks = new BlockVolumeReader(GetCompressionModule(name), tmpfile, m_options)) - await PatchWithBlocklist(database, blocks, m_options, m_result, m_blockbuffer, metadatastorage) + await PatchWithBlocklist(database, blocks, m_options, m_result, m_blockbuffer, metadatastorage, cancellationToken) .ConfigureAwait(false); } catch (Exception ex) @@ -550,7 +550,7 @@ namespace Duplicati.Library.Main.Operation var fileErrors = 0L; // Restore empty files. They might not have any blocks so don't appear in any volume. - await foreach (var file in database.GetFilesToRestore(true).Where(item => item.Length == 0).ConfigureAwait(false)) + await foreach (var file in database.GetFilesToRestore(true, cancellationToken).Where(item => item.Length == 0).ConfigureAwait(false)) { Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreEmptyFile", "Restoring empty file \"{0}\"", file.Path); @@ -589,7 +589,7 @@ namespace Duplicati.Library.Main.Operation // After all blocks in the files are restored, verify the file hash using (var filehasher = HashFactory.CreateHasher(m_options.FileHashAlgorithm)) using (new Logging.Timer(LOGTAG, "RestoreVerification", "RestoreVerification")) - await foreach (var file in database.GetFilesToRestore(true).ConfigureAwait(false)) + await foreach (var file in database.GetFilesToRestore(true, cancellationToken).ConfigureAwait(false)) { try { @@ -632,7 +632,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesRestored", null, "Restore completed without errors but no files were restored"); // Drop the temp tables - await database.DropRestoreTable().ConfigureAwait(false); + await database.DropRestoreTable(cancellationToken).ConfigureAwait(false); await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false); } @@ -712,9 +712,9 @@ namespace Duplicati.Library.Main.Operation private static async Task ScanForExistingSourceBlocksFast(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result) { // Fill BLOCKS with data from known local source files - await using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false); + await using var blockmarker = await database.CreateBlockMarkerAsync(result.TaskControl.ProgressToken).ConfigureAwait(false); var updateCount = 0L; - await foreach (var entry in database.GetFilesAndSourceBlocksFast(options.Blocksize).ConfigureAwait(false)) + await foreach (var entry in database.GetFilesAndSourceBlocksFast(options.Blocksize, result.TaskControl.ProgressToken).ConfigureAwait(false)) { var targetpath = entry.TargetPath; var targetfileid = entry.TargetFileID; @@ -737,7 +737,7 @@ namespace Duplicati.Library.Main.Operation try { using var sourcestream = SystemIO.IO_OS.FileOpenRead(sourcepath); - await foreach (var block in entry.Blocks().ConfigureAwait(false)) + await foreach (var block in entry.Blocks(result.TaskControl.ProgressToken).ConfigureAwait(false)) { if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; @@ -761,7 +761,7 @@ namespace Duplicati.Library.Main.Operation targetstream.Write(blockbuffer, 0, size); } - await blockmarker.SetBlockRestored(targetfileid, block.Index, key, block.Size, false) + await blockmarker.SetBlockRestored(targetfileid, block.Index, key, block.Size, false, result.TaskControl.ProgressToken) .ConfigureAwait(false); } } @@ -779,7 +779,7 @@ namespace Duplicati.Library.Main.Operation if ((++updateCount) % 20 == 0) { await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) + .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken) .ConfigureAwait(false); if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; @@ -810,17 +810,17 @@ namespace Duplicati.Library.Main.Operation } await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) + .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken) .ConfigureAwait(false); - await blockmarker.CommitAsync().ConfigureAwait(false); + await blockmarker.CommitAsync(result.TaskControl.ProgressToken).ConfigureAwait(false); } private static async Task ScanForExistingSourceBlocks(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result, RestoreHandlerMetadataStorage metadatastorage) { // Fill BLOCKS with data from known local source files - await using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false); + await using var blockmarker = await database.CreateBlockMarkerAsync(result.TaskControl.ProgressToken).ConfigureAwait(false); var updateCount = 0L; - await foreach (var restorelist in database.GetFilesAndSourceBlocks(options.SkipMetadata, options.Blocksize).ConfigureAwait(false)) + await foreach (var restorelist in database.GetFilesAndSourceBlocks(options.SkipMetadata, options.Blocksize, result.TaskControl.ProgressToken).ConfigureAwait(false)) { var targetpath = restorelist.TargetPath; var targetfileid = restorelist.TargetFileID; @@ -838,9 +838,9 @@ namespace Duplicati.Library.Main.Operation } using (var file = options.Dryrun ? null : SystemIO.IO_OS.FileOpenWrite(targetpath)) - await foreach (var targetblock in restorelist.Blocks().ConfigureAwait(false)) + await foreach (var targetblock in restorelist.Blocks(result.TaskControl.ProgressToken).ConfigureAwait(false)) { - await foreach (var source in targetblock.BlockSources().ConfigureAwait(false)) + await foreach (var source in targetblock.BlockSources(result.TaskControl.ProgressToken).ConfigureAwait(false)) { try { @@ -878,7 +878,7 @@ namespace Duplicati.Library.Main.Operation } await blockmarker - .SetBlockRestored(targetfileid, targetblock.Index, key, targetblock.Size, false) + .SetBlockRestored(targetfileid, targetblock.Index, key, targetblock.Size, false, result.TaskControl.ProgressToken) .ConfigureAwait(false); patched = true; break; @@ -898,7 +898,7 @@ namespace Duplicati.Library.Main.Operation if ((++updateCount) % 20 == 0) await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) + .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken) .ConfigureAwait(false); } catch (Exception ex) @@ -918,9 +918,9 @@ namespace Duplicati.Library.Main.Operation } await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) + .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken) .ConfigureAwait(false); - await blockmarker.CommitAsync().ConfigureAwait(false); + await blockmarker.CommitAsync(result.TaskControl.ProgressToken).ConfigureAwait(false); } private static async Task PrepareBlockAndFileList(LocalRestoreDatabase database, Options options, Library.Utility.IFilter filter, RestoreResults result) @@ -930,7 +930,7 @@ namespace Duplicati.Library.Main.Operation using (new Logging.Timer(LOGTAG, "PrepareRestoreFileList", "PrepareRestoreFileList")) { var c = await database - .PrepareRestoreFilelist(options.Time, options.Version, filter) + .PrepareRestoreFilelist(options.Time, options.Version, filter, result.TaskControl.ProgressToken) .ConfigureAwait(false); result.OperationProgressUpdater.UpdatefileCount(c.Item1, c.Item2, true); } @@ -941,30 +941,30 @@ namespace Duplicati.Library.Main.Operation // Find the largest common prefix var largest_prefix = options.DontCompressRestorePaths ? "" : - await database.GetLargestPrefix().ConfigureAwait(false); + await database.GetLargestPrefix(result.TaskControl.ProgressToken).ConfigureAwait(false); Logging.Log.WriteVerboseMessage(LOGTAG, "MappingRestorePath", "Mapping restore path prefix to \"{0}\" to \"{1}\"", largest_prefix, Util.AppendDirSeparator(options.Restorepath)); // Set the target paths, special care with C:\ and / await database - .SetTargetPaths(largest_prefix, Util.AppendDirSeparator(options.Restorepath)) + .SetTargetPaths(largest_prefix, Util.AppendDirSeparator(options.Restorepath), result.TaskControl.ProgressToken) .ConfigureAwait(false); } else { - await database.SetTargetPaths("", "").ConfigureAwait(false); + await database.SetTargetPaths("", "", result.TaskControl.ProgressToken).ConfigureAwait(false); } // Create a temporary table BLOCKS that lists all blocks that needs to be recovered using (new Logging.Timer(LOGTAG, "FindMissingBlocks", "FindMissingBlocks")) await database - .FindMissingBlocks(options.SkipMetadata) + .FindMissingBlocks(options.SkipMetadata, result.TaskControl.ProgressToken) .ConfigureAwait(false); // Create temporary tables and triggers that automatically track progress using (new Logging.Timer(LOGTAG, "CreateProgressTracker", "CreateProgressTracker")) await database - .CreateProgressTracker(false) + .CreateProgressTracker(false, result.TaskControl.ProgressToken) .ConfigureAwait(false); } @@ -983,7 +983,7 @@ namespace Duplicati.Library.Main.Operation SystemIO.IO_OS.DirectoryCreate(options.Restorepath); } - await foreach (var folder in database.GetTargetFolders().ConfigureAwait(false)) + await foreach (var folder in database.GetTargetFolders(result.TaskControl.ProgressToken).ConfigureAwait(false)) { try { @@ -1014,9 +1014,9 @@ namespace Duplicati.Library.Main.Operation private static async Task ScanForExistingTargetBlocks(LocalRestoreDatabase database, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm blockhasher, System.Security.Cryptography.HashAlgorithm filehasher, Options options, RestoreResults result) { // Scan existing files for existing BLOCKS - await using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false); + await using var blockmarker = await database.CreateBlockMarkerAsync(result.TaskControl.ProgressToken).ConfigureAwait(false); var updateCount = 0L; - await foreach (var restorelist in database.GetExistingFilesWithBlocks().ConfigureAwait(false)) + await foreach (var restorelist in database.GetExistingFilesWithBlocks(result.TaskControl.ProgressToken).ConfigureAwait(false)) { var rename = !options.Overwrite; var targetpath = restorelist.TargetPath; @@ -1069,7 +1069,7 @@ namespace Duplicati.Library.Main.Operation using (var file = SystemIO.IO_OS.FileOpenRead(targetpath)) using (var block = new Blockprocessor(file, blockbuffer)) - await foreach (var targetblock in restorelist.Blocks().ConfigureAwait(false)) + await foreach (var targetblock in restorelist.Blocks(result.TaskControl.ProgressToken).ConfigureAwait(false)) { var size = block.Readblock(); if (size <= 0) @@ -1095,7 +1095,7 @@ namespace Duplicati.Library.Main.Operation if (key == targetblock.Hash) { await blockmarker - .SetBlockRestored(targetfileid, targetblock.Index, key, size, false) + .SetBlockRestored(targetfileid, targetblock.Index, key, size, false, result.TaskControl.ProgressToken) .ConfigureAwait(false); blockhashmatch = true; } @@ -1132,7 +1132,7 @@ namespace Duplicati.Library.Main.Operation { //TODO: Check metadata to trigger rename? If metadata changed, it will still be restored for the file in-place. await blockmarker - .SetFileDataVerified(targetfileid) + .SetFileDataVerified(targetfileid, result.TaskControl.ProgressToken) .ConfigureAwait(false); Logging.Log.WriteVerboseMessage(LOGTAG, "TargetExistsInCorrectVersion", "Target file exists{1} and is correct version: {0}", targetpath, wasTruncated ? " (but was truncated)" : ""); rename = false; @@ -1142,7 +1142,7 @@ namespace Duplicati.Library.Main.Operation // The new file will have none of the correct blocks, // even if the scanned file had some await blockmarker - .SetAllBlocksMissing(targetfileid) + .SetAllBlocksMissing(targetfileid, result.TaskControl.ProgressToken) .ConfigureAwait(false); } } @@ -1150,7 +1150,7 @@ namespace Duplicati.Library.Main.Operation if ((++updateCount) % 20 == 0) { await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) + .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken) .ConfigureAwait(false); if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; @@ -1199,10 +1199,10 @@ namespace Duplicati.Library.Main.Operation //TODO: Also needs metadata check to make correct decision. // We stick to the policy to restore metadata in place, if data ok. So, metadata block may be restored. await blockmarker - .SetAllBlocksRestored(targetfileid, false) + .SetAllBlocksRestored(targetfileid, false, result.TaskControl.ProgressToken) .ConfigureAwait(false); await blockmarker - .SetFileDataVerified(targetfileid) + .SetFileDataVerified(targetfileid, result.TaskControl.ProgressToken) .ConfigureAwait(false); break; } @@ -1220,17 +1220,17 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteVerboseMessage(LOGTAG, "TargetFileRetargeted", "Target file exists and will be restored to: {0}", newname); await database - .UpdateTargetPath(targetfileid, newname) + .UpdateTargetPath(targetfileid, newname, result.TaskControl.ProgressToken) .ConfigureAwait(false); } } await blockmarker - .UpdateProcessed(result.OperationProgressUpdater) + .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken) .ConfigureAwait(false); await blockmarker - .CommitAsync() + .CommitAsync(result.TaskControl.ProgressToken) .ConfigureAwait(false); } } diff --git a/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs b/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs index 6f09c175b..fe66207c2 100644 --- a/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs +++ b/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs @@ -50,14 +50,14 @@ internal static class SearchEntriesHandler throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase"); await using var db = - await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache) + await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache, null, result.TaskControl.ProgressToken) .ConfigureAwait(false); long[]? filesetIds = null; if (!options.AllVersions) { filesetIds = await db - .GetFilesetIDs(options.Time, options.Version) - .ToArrayAsync() + .GetFilesetIDs(options.Time, options.Version, false, result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken) .ConfigureAwait(false); if (filesetIds.Length == 0) @@ -68,7 +68,7 @@ internal static class SearchEntriesHandler paths = paths.Select(path => Util.AppendDirSeparator(path)).ToArray(); result.FileVersions = await db - .SearchEntries(paths, filter, filesetIds, offset, limit) + .SearchEntries(paths, filter, filesetIds, offset, limit, result.TaskControl.ProgressToken) .ConfigureAwait(false); } } diff --git a/Duplicati/Library/Main/Operation/TestHandler.cs b/Duplicati/Library/Main/Operation/TestHandler.cs index 58852c7a6..523398465 100644 --- a/Duplicati/Library/Main/Operation/TestHandler.cs +++ b/Duplicati/Library/Main/Operation/TestHandler.cs @@ -39,12 +39,12 @@ namespace Duplicati.Library.Main.Operation private static readonly string LOGTAG = Logging.Log.LogTagFromType(); private readonly Options m_options; - private readonly TestResults m_results; + private readonly TestResults m_result; public TestHandler(Options options, TestResults results) { m_options = options; - m_results = results; + m_result = results; } public async Task RunAsync(long samples, IBackendManager backendManager) @@ -52,17 +52,17 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - await using var db = await LocalTestDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false); - await Utility.UpdateOptionsFromDb(db, m_options) + await using var db = await LocalTestDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - await Utility.VerifyOptionsAndUpdateDatabase(db, m_options) + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); await db - .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks) + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_results.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await DoRunAsync(samples, db, backendManager).ConfigureAwait(false); await db.Transaction .CommitAsync("TestHandlerComplete") @@ -72,35 +72,35 @@ namespace Duplicati.Library.Main.Operation public async Task DoRunAsync(long samples, LocalTestDatabase db, IBackendManager backend) { var files = await db - .SelectTestTargets(samples, m_options) - .ToListAsync() + .SelectTestTargets(samples, m_options, m_result.TaskControl.ProgressToken) + .ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - m_results.OperationProgressUpdater.UpdatePhase(OperationPhase.Verify_Running); - m_results.OperationProgressUpdater.UpdateProgress(0); + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Verify_Running); + m_result.OperationProgressUpdater.UpdateProgress(0); var progress = 0L; if (m_options.FullRemoteVerification != Options.RemoteTestStrategy.False) { var faultyIndexFiles = new List(); - await foreach (var (tf, hash, size, name) in backend.GetFilesOverlappedAsync(files, m_results.TaskControl.ProgressToken).ConfigureAwait(false)) + await foreach (var (tf, hash, size, name) in backend.GetFilesOverlappedAsync(files, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { var vol = new RemoteVolume(name, hash, size); try { - if (!await m_results.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backend.WaitForEmptyAsync(db, m_results.TaskControl.ProgressToken).ConfigureAwait(false); - m_results.EndTime = DateTime.UtcNow; + await backend.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + m_result.EndTime = DateTime.UtcNow; return; } progress++; - m_results.OperationProgressUpdater.UpdateProgress((float)progress / files.Count); + m_result.OperationProgressUpdater.UpdateProgress((float)progress / files.Count); KeyValuePair>> res; using (tf) - res = await TestVolumeInternals(db, vol, tf, m_options, m_options.FullBlockVerification ? 1.0 : 0.2) + res = await TestVolumeInternals(db, vol, tf, m_options, m_options.FullBlockVerification ? 1.0 : 0.2, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); var parsedInfo = VolumeBase.ParseFilename(vol.Name); @@ -119,14 +119,14 @@ namespace Duplicati.Library.Main.Operation faultyIndexFiles.Add(vol); } - m_results.AddResult(res.Key, res.Value); + m_result.AddResult(res.Key, res.Value); if (!string.IsNullOrWhiteSpace(vol.Hash) && vol.Size > 0) { if (res.Value == null || !res.Value.Any()) { var rv = await db - .GetRemoteVolume(vol.Name) + .GetRemoteVolume(vol.Name, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); if (rv.ID < 0) @@ -141,7 +141,7 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "CaptureHashAndSize", "Successfully captured hash and size for {0}, updating database", vol.Name); await db - .UpdateRemoteVolume(vol.Name, RemoteVolumeState.Verified, vol.Size, vol.Hash) + .UpdateRemoteVolume(vol.Name, RemoteVolumeState.Verified, vol.Size, vol.Hash, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } } @@ -150,16 +150,16 @@ namespace Duplicati.Library.Main.Operation } await db - .UpdateVerificationCount(vol.Name) + .UpdateVerificationCount(vol.Name, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } catch (Exception ex) { - m_results.AddResult(vol.Name, [new KeyValuePair(TestEntryStatus.Error, ex.Message)]); + m_result.AddResult(vol.Name, [new KeyValuePair(TestEntryStatus.Error, ex.Message)]); Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileProcessingFailed", ex, "Failed to process file {0}", vol.Name); if (ex.IsAbortException()) { - m_results.EndTime = DateTime.UtcNow; + m_result.EndTime = DateTime.UtcNow; throw; } } @@ -170,7 +170,7 @@ namespace Duplicati.Library.Main.Operation if (m_options.ReplaceFaultyIndexFiles) { Logging.Log.WriteWarningMessage(LOGTAG, "FaultyIndexFiles", null, "Found {0} faulty index files, repairing now", faultyIndexFiles.Count); - await ReplaceFaultyIndexFilesAsync(faultyIndexFiles, backend, db, m_results.TaskControl.ProgressToken).ConfigureAwait(false); + await ReplaceFaultyIndexFilesAsync(faultyIndexFiles, backend, db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } else Logging.Log.WriteWarningMessage(LOGTAG, "FaultyIndexFiles", null, "Found {0} faulty index files, use the option {1} to repair them", faultyIndexFiles.Count, "--replace-faulty-index-files"); @@ -183,26 +183,26 @@ namespace Duplicati.Library.Main.Operation { try { - if (!await m_results.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - m_results.EndTime = DateTime.UtcNow; + m_result.EndTime = DateTime.UtcNow; return; } progress++; - m_results.OperationProgressUpdater.UpdateProgress((float)progress / files.Count); + m_result.OperationProgressUpdater.UpdateProgress((float)progress / files.Count); if (f.Size <= 0 || string.IsNullOrWhiteSpace(f.Hash)) { Logging.Log.WriteInformationMessage(LOGTAG, "MissingRemoteHash", "No hash or size recorded for {0}, performing full verification", f.Name); KeyValuePair>> res; - (var tf, var hash, var size) = await backend.GetWithInfoAsync(f.Name, f.Hash, f.Size, m_results.TaskControl.ProgressToken).ConfigureAwait(false); + (var tf, var hash, var size) = await backend.GetWithInfoAsync(f.Name, f.Hash, f.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false); using (tf) - res = await TestVolumeInternals(db, f, tf, m_options, 1) + res = await TestVolumeInternals(db, f, tf, m_options, 1, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - m_results.AddResult(res.Key, res.Value); + m_result.AddResult(res.Key, res.Value); if (!string.IsNullOrWhiteSpace(hash) && size > 0) { @@ -216,7 +216,7 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "CapturedHashAndSize", "Successfully captured hash and size for {0}, updating database", f.Name); await db - .UpdateRemoteVolume(f.Name, RemoteVolumeState.Verified, size, hash) + .UpdateRemoteVolume(f.Name, RemoteVolumeState.Verified, size, hash, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); } } @@ -224,39 +224,39 @@ namespace Duplicati.Library.Main.Operation } else { - using (var tf = await backend.GetAsync(f.Name, f.Hash, f.Size, m_results.TaskControl.ProgressToken).ConfigureAwait(false)) + using (var tf = await backend.GetAsync(f.Name, f.Hash, f.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { } } await db - .UpdateVerificationCount(f.Name) + .UpdateVerificationCount(f.Name, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); - m_results.AddResult(f.Name, []); + m_result.AddResult(f.Name, []); } catch (Exception ex) { - m_results.AddResult(f.Name, [new KeyValuePair(TestEntryStatus.Error, ex.Message)]); + m_result.AddResult(f.Name, [new KeyValuePair(TestEntryStatus.Error, ex.Message)]); Logging.Log.WriteErrorMessage(LOGTAG, "FailedToProcessFile", ex, "Failed to process file {0}", f.Name); if (ex.IsAbortOrCancelException()) { - m_results.EndTime = DateTime.UtcNow; + m_result.EndTime = DateTime.UtcNow; throw; } } } } - m_results.EndTime = DateTime.UtcNow; + m_result.EndTime = DateTime.UtcNow; // generate a backup error status when any test is failing - except for 'extra' status // because these problems don't block database rebuilding. - var filtered = from n in m_results.Verifications where n.Value.Any(x => x.Key != TestEntryStatus.Extra) select n; + var filtered = from n in m_result.Verifications where n.Value.Any(x => x.Key != TestEntryStatus.Extra) select n; if (!filtered.Any()) { - Logging.Log.WriteInformationMessage(LOGTAG, "Test results", "Successfully verified {0} remote files", m_results.VerificationsActualLength); + Logging.Log.WriteInformationMessage(LOGTAG, "Test results", "Successfully verified {0} remote files", m_result.VerificationsActualLength); } else { - Logging.Log.WriteErrorMessage(LOGTAG, "Test results", null, "Verified {0} remote files with {1} problem(s)", m_results.VerificationsActualLength, filtered.Count()); + Logging.Log.WriteErrorMessage(LOGTAG, "Test results", null, "Verified {0} remote files with {1} problem(s)", m_result.VerificationsActualLength, filtered.Count()); } } @@ -268,8 +268,9 @@ namespace Duplicati.Library.Main.Operation /// The path to the downloaded copy of the file. /// The options to use for the test. /// A value between 0 and 1 that indicates how many blocks are tested in a dblock file. + /// A cancellation token to cancel the operation. /// A task that when awaited returns a key-value pair where the key is the volume name and the value is a list of test results. - public static async Task>>> TestVolumeInternals(LocalTestDatabase db, IRemoteVolume vol, string tf, Options options, double sample_percent) + public static async Task>>> TestVolumeInternals(LocalTestDatabase db, IRemoteVolume vol, string tf, Options options, double sample_percent, CancellationToken cancellationToken) { var hashsize = HashFactory.HashSizeBytes(options.BlockHashAlgorithm); var parsedInfo = Volumes.VolumeBase.ParseFilename(vol.Name); @@ -280,15 +281,15 @@ namespace Duplicati.Library.Main.Operation case RemoteVolumeType.Files: //Compare with db and see if all files are accounted for // with correct file hashes and blocklist hashes - await using (var fl = await db.CreateFilelist(vol.Name).ConfigureAwait(false)) + await using (var fl = await db.CreateFilelist(vol.Name, cancellationToken).ConfigureAwait(false)) { using (var rd = new Volumes.FilesetVolumeReader(parsedInfo.CompressionModule, tf, options)) foreach (var f in rd.Files) await fl - .Add(f.Path, f.Size, f.Hash, f.Metasize, f.Metahash, f.BlocklistHashes, f.Type, f.Time) + .Add(f.Path, f.Size, f.Hash, f.Metasize, f.Metahash, f.BlocklistHashes, f.Type, f.Time, cancellationToken) .ConfigureAwait(false); - return new KeyValuePair>>(vol.Name, await fl.Compare().ToListAsync().ConfigureAwait(false)); + return new KeyValuePair>>(vol.Name, await fl.Compare(cancellationToken).ToListAsync(cancellationToken: cancellationToken).ConfigureAwait(false)); } case RemoteVolumeType.Index: @@ -301,17 +302,17 @@ namespace Duplicati.Library.Main.Operation foreach (var v in rd.Volumes) { blocklinks.Add(new Tuple(v.Filename, v.Hash, v.Length)); - await using (var bl = await db.CreateBlocklist(v.Filename).ConfigureAwait(false)) + await using (var bl = await db.CreateBlocklist(v.Filename, cancellationToken).ConfigureAwait(false)) { foreach (var h in v.Blocks) await bl - .AddBlock(h.Key, h.Value) + .AddBlock(h.Key, h.Value, cancellationToken) .ConfigureAwait(false); combined.AddRange( await bl - .Compare() - .ToListAsync() + .Compare(cancellationToken) + .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false) ); } @@ -320,17 +321,17 @@ namespace Duplicati.Library.Main.Operation if (options.IndexfilePolicy == Options.IndexFileStrategy.Full) { var hashesPerBlock = options.Blocksize / options.BlockhashSize; - await using (var bl = await db.CreateBlocklistHashList(vol.Name).ConfigureAwait(false)) + await using (var bl = await db.CreateBlocklistHashList(vol.Name, cancellationToken).ConfigureAwait(false)) { foreach (var b in rd.BlockLists) await bl - .AddBlockHash(b.Hash, b.Length) + .AddBlockHash(b.Hash, b.Length, cancellationToken) .ConfigureAwait(false); combined.AddRange( await bl - .Compare(hashesPerBlock, options.BlockhashSize, options.Blocksize) - .ToListAsync() + .Compare(hashesPerBlock, options.BlockhashSize, options.Blocksize, cancellationToken) + .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false) ); } @@ -338,17 +339,17 @@ namespace Duplicati.Library.Main.Operation } // Compare with db and see that all blocklists are listed - await using (var il = await db.CreateIndexlist(vol.Name).ConfigureAwait(false)) + await using (var il = await db.CreateIndexlist(vol.Name, cancellationToken).ConfigureAwait(false)) { foreach (var t in blocklinks) await il - .AddBlockLink(t.Item1, t.Item2, t.Item3) + .AddBlockLink(t.Item1, t.Item2, t.Item3, cancellationToken) .ConfigureAwait(false); combined.AddRange( await il - .Compare() - .ToArrayAsync() + .Compare(cancellationToken) + .ToArrayAsync(cancellationToken: cancellationToken) .ConfigureAwait(false) ); } @@ -356,13 +357,13 @@ namespace Duplicati.Library.Main.Operation return new KeyValuePair>>(vol.Name, combined); case RemoteVolumeType.Blocks: using (var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm)) - await using (var bl = await db.CreateBlocklist(vol.Name).ConfigureAwait(false)) + await using (var bl = await db.CreateBlocklist(vol.Name, cancellationToken).ConfigureAwait(false)) using (var rd = new Volumes.BlockVolumeReader(parsedInfo.CompressionModule, tf, options)) { //Verify that all blocks are in the file foreach (var b in rd.Blocks) await bl - .AddBlock(b.Key, b.Value) + .AddBlock(b.Key, b.Value, cancellationToken) .ConfigureAwait(false); //Select random blocks and verify their hashes match the filename and size @@ -392,8 +393,8 @@ namespace Duplicati.Library.Main.Operation vol.Name, changes.Union( await bl - .Compare() - .ToListAsync() + .Compare(cancellationToken) + .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false) ) ); @@ -407,14 +408,14 @@ namespace Duplicati.Library.Main.Operation private async Task ReplaceFaultyIndexFilesAsync(List faultyIndexFiles, IBackendManager backendManager, LocalTestDatabase db, CancellationToken cancellationToken) { await using var repairdb = - await LocalRepairDatabase.CreateAsync(db) + await LocalRepairDatabase.CreateAsync(db, null, cancellationToken) .ConfigureAwait(false); foreach (var vol in faultyIndexFiles) { - if (!await m_results.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - m_results.EndTime = DateTime.UtcNow; + m_result.EndTime = DateTime.UtcNow; return; } @@ -429,8 +430,8 @@ namespace Duplicati.Library.Main.Operation } else { - await backendManager.DeleteAsync(vol.Name, vol.Size, true, m_results.TaskControl.ProgressToken).ConfigureAwait(false); - await backendManager.WaitForEmptyAsync(repairdb, m_results.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.DeleteAsync(vol.Name, vol.Size, true, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(repairdb, m_result.TaskControl.ProgressToken).ConfigureAwait(false); await repairdb.Transaction .CommitAsync("ReplaceFaultyIndexFileCommit") .ConfigureAwait(false); diff --git a/Duplicati/Library/Main/Operation/VacuumHandler.cs b/Duplicati/Library/Main/Operation/VacuumHandler.cs index dfc62247c..9556363fa 100644 --- a/Duplicati/Library/Main/Operation/VacuumHandler.cs +++ b/Duplicati/Library/Main/Operation/VacuumHandler.cs @@ -38,10 +38,10 @@ namespace Duplicati.Library.Main.Operation public virtual async Task RunAsync() { await using var db = - await Database.LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Vacuum", false, m_options.SqlitePageCache) + await Database.LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Vacuum", false, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken) .ConfigureAwait(false); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Vacuum_Running); - await db.Vacuum().ConfigureAwait(false); + await db.Vacuum(m_result.TaskControl.ProgressToken).ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; } } diff --git a/Duplicati/Library/Main/Utility.cs b/Duplicati/Library/Main/Utility.cs index feaf14dbc..0d01d04b7 100644 --- a/Duplicati/Library/Main/Utility.cs +++ b/Duplicati/Library/Main/Utility.cs @@ -28,6 +28,7 @@ using System.Threading.Tasks; using Newtonsoft.Json; using Duplicati.Library.Main.Database; using Duplicati.Library.Utility; +using System.Threading; namespace Duplicati.Library.Main { @@ -103,11 +104,12 @@ namespace Duplicati.Library.Main /// /// The database to read from. /// The options to update. + /// A cancellation token to cancel the operation. /// A task that completes when the options have been updated. - internal static async Task UpdateOptionsFromDb(LocalDatabase db, Options options) + internal static async Task UpdateOptionsFromDb(LocalDatabase db, Options options, CancellationToken cancellationToken) { string n = null; - var opts = await db.GetDbOptions().ConfigureAwait(false); + var opts = await db.GetDbOptions(cancellationToken).ConfigureAwait(false); if (opts.ContainsKey("blocksize") && (!options.RawOptions.TryGetValue("blocksize", out n) || string.IsNullOrEmpty(n))) options.RawOptions["blocksize"] = opts["blocksize"] + "b"; @@ -121,11 +123,12 @@ namespace Duplicati.Library.Main /// Checks if the database contains options that need to be verified, such as the blocksize. /// /// The database to check. + /// A cancellation token to cancel the operation. /// A task that when awaited returns true if the database contains options that need to be verified; false otherwise. - internal static async Task ContainsOptionsForVerification(LocalDatabase db) + internal static async Task ContainsOptionsForVerification(LocalDatabase db, CancellationToken cancellationToken) { - var opts = await db.GetDbOptions().ConfigureAwait(false); - await db.Transaction.CommitAsync().ConfigureAwait(false); + var opts = await db.GetDbOptions(cancellationToken).ConfigureAwait(false); + await db.Transaction.CommitAsync(cancellationToken).ConfigureAwait(false); return new[] { "blocksize", "blockhash", @@ -140,8 +143,9 @@ namespace Duplicati.Library.Main /// /// The database to check. /// The options to verify. + /// A cancellation token to cancel the operation. /// A task that completes when the options have been verified and the database has been updated if needed. - internal static async Task VerifyOptionsAndUpdateDatabase(LocalDatabase db, Options options) + internal static async Task VerifyOptionsAndUpdateDatabase(LocalDatabase db, Options options, CancellationToken cancellationToken) { var newDict = new Dictionary { @@ -149,7 +153,7 @@ namespace Duplicati.Library.Main { "blockhash", options.BlockHashAlgorithm }, { "filehash", options.FileHashAlgorithm } }; - var opts = await db.GetDbOptions().ConfigureAwait(false); + var opts = await db.GetDbOptions(cancellationToken).ConfigureAwait(false); if (options.NoEncryption) { @@ -199,7 +203,7 @@ namespace Duplicati.Library.Main } //Extra sanity check - if (await db.GetBlocksLargerThan(options.Blocksize).ConfigureAwait(false) > 0) + if (await db.GetBlocksLargerThan(options.Blocksize, cancellationToken).ConfigureAwait(false) > 0) throw new Duplicati.Library.Interface.UserInformationException("You have attempted to change the block-size on an existing backup, which is not supported. Please configure a new clean backup if you want to change the block-size.", "BlockSizeChangeNotSupported"); if (needsUpdate) @@ -209,7 +213,7 @@ namespace Duplicati.Library.Main if (!newDict.ContainsKey(k.Key)) newDict[k.Key] = k.Value; - await db.SetDbOptions(newDict).ConfigureAwait(false); + await db.SetDbOptions(newDict, cancellationToken).ConfigureAwait(false); } } }