haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updated all of the handler calls to database methods to forward their cancellation token

Browse Source
This commit is contained in:
Carl Johnsen
2025-06-19 09:47:33 +02:00
parent 7c425c4649
commit 32efa4b845
24 changed files with 638 additions and 584 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Main/Database/LocalListDatabase.cs
+9 -4
View File
@@ -71,7 +71,9 @@ namespace Duplicati.Library.Main.Database
/// <summary>
/// Asynchronously retrieves the sizes of the file versions.
/// </summary>
IAsyncEnumerable<long> Sizes();
/// <param name="token">A cancellation token to cancel the operation.</param>
/// <returns>An asynchronous enumerable of file sizes.</returns>
IAsyncEnumerable<long> Sizes(CancellationToken token);
}
/// <summary>
@@ -301,12 +303,13 @@ namespace Duplicati.Library.Main.Database
More = true;
}
public async IAsyncEnumerable<long> Sizes()
/// <inheritdoc />
public async IAsyncEnumerable<long> Sizes([EnumeratorCancellation] CancellationToken token)
{
while (More && Path == m_reader.ConvertValueToString(0))
{
yield return m_reader.ConvertValueToInt64(1, -1);
More = await m_reader.ReadAsync().ConfigureAwait(false);
More = await m_reader.ReadAsync(token).ConfigureAwait(false);
}
}
}
@@ -316,8 +319,10 @@ namespace Duplicati.Library.Main.Database
/// </summary>
private class FileversionFixed : IFileversion
{
/// <inheritdoc/>
public string? Path { get; internal set; }
public IAsyncEnumerable<long> Sizes() => AsyncEnumerable.Empty<long>();
/// <inheritdoc/>
public IAsyncEnumerable<long> Sizes(CancellationToken token) => AsyncEnumerable.Empty<long>();
}
/// <inheritdoc />
Duplicati/Library/Main/Operation/BackupHandler.cs
+37 -37
View File
@@ -214,7 +214,7 @@ namespace Duplicati.Library.Main.Operation
// all of the local database classes handle this), as the constructor
// is not async and shouldn't be executing "heavy" code, such as
// database queries.
var journalData = m_database.GetChangeJournalData(lastfilesetid).ToEnumerable();
var journalData = m_database.GetChangeJournalData(lastfilesetid, m_taskReader.ProgressToken).ToEnumerable();
var service = new UsnJournalService(fileProvider.SnapshotService, filter, m_options.FileAttributeFilter, m_options.SkipFilesLargerThan,
journalData, cancellationTokenSource.Token);
@@ -300,9 +300,9 @@ namespace Duplicati.Library.Main.Operation
{
try
{
database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options)
database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
var lastFilestTime = await database.FilesetTimes()
var lastFilestTime = await database.FilesetTimes(result.TaskControl.ProgressToken)
.Select(x => x.Value)
.Append(DateTime.UnixEpoch)
.Select(Library.Utility.Utility.NormalizeDateTimeToEpochSeconds)
@@ -319,40 +319,40 @@ namespace Duplicati.Library.Main.Operation
Log.WriteVerboseMessage(LOGTAG, "DatabaseTimestampError", "The database has a timestamp of {0}, but the last fileset has a timestamp of {1}. Sleeping for {2} seconds to allow the clock to catch up.", database.OperationTimestamp.ToLocalTime(), Library.Utility.Utility.EPOCH.AddSeconds(lastFilestTime).ToLocalTime(), sleepSeconds);
Thread.Sleep(TimeSpan.FromSeconds(sleepSeconds));
database.Dispose();
database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options)
database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
result.Dryrun = options.Dryrun;
// Check the database integrity
await Utility.UpdateOptionsFromDb(database, options)
await Utility.UpdateOptionsFromDb(database, options, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await Utility.VerifyOptionsAndUpdateDatabase(database, options)
await Utility.VerifyOptionsAndUpdateDatabase(database, options, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
var probe_path = await database
.GetFirstPath()
.GetFirstPath(result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (probe_path != null && Util.GuessDirSeparator(probe_path) != Util.DirectorySeparatorString)
throw new UserInformationException(string.Format("The backup contains files that belong to another operating system. Proceeding with a backup would cause the database to contain paths from two different operation systems, which is not supported. To proceed without losing remote data, delete all filesets and make sure the --{0} option is set, then run the backup again to re-use the existing data on the remote store.", "no-auto-compact"), "CrossOsDatabaseReuseNotSupported");
if (await database.PartiallyRecreated().ConfigureAwait(false))
if (await database.PartiallyRecreated(result.TaskControl.ProgressToken).ConfigureAwait(false))
throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated");
if (await database.RepairInProgress().ConfigureAwait(false))
if (await database.RepairInProgress(result.TaskControl.ProgressToken).ConfigureAwait(false))
throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the backup process cannot continue. You may delete the local database and attempt to repair it again.", "DatabaseRepairInProgress");
// Make sure the database is sane
await database
.VerifyConsistency(options.Blocksize, options.BlockhashSize, !options.DisableFilelistConsistencyChecks)
.VerifyConsistency(options.Blocksize, options.BlockhashSize, !options.DisableFilelistConsistencyChecks, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
// Guard the last filelist
if (!options.DisableSyntheticFilelist)
lastTempFilelist = await database
.GetLastIncompleteFilesetVolume()
.GetLastIncompleteFilesetVolume(result.TaskControl.ProgressToken)
.ConfigureAwait(false);
try
@@ -364,8 +364,8 @@ namespace Duplicati.Library.Main.Operation
}
else
{
await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: false, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean).ConfigureAwait(false);
var updatedLastTemp = string.IsNullOrWhiteSpace(lastTempFilelist.Name) ? default : await database.GetRemoteVolume(lastTempFilelist.Name).ConfigureAwait(false);
await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: false, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean, result.TaskControl.ProgressToken).ConfigureAwait(false);
var updatedLastTemp = string.IsNullOrWhiteSpace(lastTempFilelist.Name) ? default : await database.GetRemoteVolume(lastTempFilelist.Name, result.TaskControl.ProgressToken).ConfigureAwait(false);
if (!string.IsNullOrWhiteSpace(updatedLastTemp.Name) && updatedLastTemp.State == RemoteVolumeState.Deleting)
{
// The last temporary filelist was emptied, and scheduled for deletion, so we need to delete it
@@ -381,7 +381,7 @@ namespace Duplicati.Library.Main.Operation
await backendManager.WaitForEmptyAsync(database, result.TaskControl.ProgressToken).ConfigureAwait(false);
// In case the backend delete failed, manually set the state to deleted
await database
.UpdateRemoteVolume(updatedLastTemp.Name, RemoteVolumeState.Deleted, updatedLastTemp.Size, updatedLastTemp.Hash, false, TimeSpan.FromHours(2), null)
.UpdateRemoteVolume(updatedLastTemp.Name, RemoteVolumeState.Deleted, updatedLastTemp.Size, updatedLastTemp.Hash, false, TimeSpan.FromHours(2), null, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
// The last temporary filelist is no more
lastTempFilelist = default;
@@ -402,18 +402,18 @@ namespace Duplicati.Library.Main.Operation
.ConfigureAwait(false);
// Re-open the database and backend manager
database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options)
database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
Log.WriteInformationMessage(LOGTAG, "BackendCleanupFinished", "Backend cleanup finished, retrying verification");
await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false);
await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, result.TaskControl.ProgressToken).ConfigureAwait(false);
}
else
throw;
}
// Return the updated version of the fileset, in case it was actually complete
return new PreBackupVerifyResult(database, await database.GetLastIncompleteFilesetVolume().ConfigureAwait(false));
return new PreBackupVerifyResult(database, await database.GetLastIncompleteFilesetVolume(result.TaskControl.ProgressToken).ConfigureAwait(false));
}
catch
{
@@ -540,7 +540,7 @@ namespace Duplicati.Library.Main.Operation
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Delete);
m_result.DeleteResults = new DeleteResults(m_result);
await using var db = await LocalDeleteDatabase.CreateAsync(m_database).ConfigureAwait(false);
await using var db = await LocalDeleteDatabase.CreateAsync(m_database, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
await new DeleteHandler(m_options, (DeleteResults)m_result.DeleteResults)
.DoRunAsync(db, true, currentIsSmall, backendManager)
.ConfigureAwait(false);
@@ -550,7 +550,7 @@ namespace Duplicati.Library.Main.Operation
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Compact);
m_result.CompactResults = new CompactResults(m_result);
await using var db = await LocalDeleteDatabase.CreateAsync(m_database).ConfigureAwait(false);
await using var db = await LocalDeleteDatabase.CreateAsync(m_database, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
await new CompactHandler(m_options, (CompactResults)m_result.CompactResults)
.DoCompactAsync(db, true, backendManager)
.ConfigureAwait(false);
@@ -561,7 +561,7 @@ namespace Duplicati.Library.Main.Operation
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupVerify);
using (new Logging.Timer(LOGTAG, "AfterBackupVerify", "AfterBackupVerify"))
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, m_result.BackendWriter, [currentFilelistVolume], [previousTemporaryFilelist], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false);
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, m_result.BackendWriter, [currentFilelistVolume], [previousTemporaryFilelist], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
await backendManager
.WaitForEmptyAsync(m_database, m_taskReader.ProgressToken)
@@ -569,7 +569,7 @@ namespace Duplicati.Library.Main.Operation
// Calculate the number of samples to test, using the largest number of file of a given type
long remoteVolumeCount = await m_database
.GetRemoteVolumes()
.GetRemoteVolumes(m_result.TaskControl.ProgressToken)
.Where(x => x.State == RemoteVolumeState.Verified)
.GroupBy(x => x.Type)
.SelectAwait(async x => await x.LongCountAsync())
@@ -582,7 +582,7 @@ namespace Duplicati.Library.Main.Operation
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupTest);
m_result.TestResults = new TestResults(m_result);
await using var testdb = await LocalTestDatabase.CreateAsync(m_database).ConfigureAwait(false);
await using var testdb = await LocalTestDatabase.CreateAsync(m_database, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
await new TestHandler(m_options, (TestResults)m_result.TestResults)
.DoRunAsync(samplesToTest, testdb, backendManager)
.ConfigureAwait(false);
@@ -597,33 +597,33 @@ namespace Duplicati.Library.Main.Operation
if (result.BackendWriter != null)
{
result.BackendWriter.KnownFileCount = await database
.GetRemoteVolumes()
.CountAsync()
.GetRemoteVolumes(cancelToken)
.CountAsync(cancelToken)
.ConfigureAwait(false);
result.BackendWriter.KnownFilesets = await database
.GetRemoteVolumes()
.GetRemoteVolumes(cancelToken)
.CountAsync(x => x.Type == RemoteVolumeType.Files)
.ConfigureAwait(false);
result.BackendWriter.KnownFileSize = await database
.GetRemoteVolumes()
.GetRemoteVolumes(cancelToken)
.Select(x => Math.Max(0, x.Size))
.SumAsync()
.SumAsync(cancelToken)
.ConfigureAwait(false);
result.BackendWriter.UnknownFileCount = 0;
result.BackendWriter.UnknownFileSize = 0;
result.BackendWriter.BackupListCount = await database
.FilesetTimes()
.CountAsync()
.FilesetTimes(cancelToken)
.CountAsync(cancelToken)
.ConfigureAwait(false);
result.BackendWriter.LastBackupDate = (
await database
.FilesetTimes()
.FirstOrDefaultAsync()
.FilesetTimes(cancelToken)
.FirstOrDefaultAsync(cancelToken)
.ConfigureAwait(false)
)
.Value.ToLocalTime();
@@ -711,7 +711,7 @@ namespace Duplicati.Library.Main.Operation
// If we crash now, there is a chance we leave behind partial remote files
if (!m_options.Dryrun)
await database
.TerminatedWithActiveUploads(true)
.TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, true)
.ConfigureAwait(false);
await using (m_database = database)
@@ -731,7 +731,7 @@ namespace Duplicati.Library.Main.Operation
// Grab the previous backup ID, if any
var prevfileset = await m_database
.FilesetTimes()
.FilesetTimes(m_result.TaskControl.ProgressToken)
.FirstOrDefaultAsync()
.ConfigureAwait(false);
@@ -768,7 +768,7 @@ namespace Duplicati.Library.Main.Operation
if (m_options.DisableFileScanner)
{
var d = await m_database
.GetLastBackupFileCountAndSize()
.GetLastBackupFileCountAndSize(m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
m_result.OperationProgressUpdater.UpdatefileCount(d.Item1, d.Item2, true);
@@ -815,7 +815,7 @@ namespace Duplicati.Library.Main.Operation
if (!m_options.Dryrun)
await database
.TerminatedWithActiveUploads(false)
.TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, false)
.ConfigureAwait(false);
// Make sure we have the database up-to-date
@@ -826,7 +826,7 @@ namespace Duplicati.Library.Main.Operation
if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
{
var lastVolumeSize = await m_database
.GetLastWrittenDBlockVolumeSize()
.GetLastWrittenDBlockVolumeSize(m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await CompactIfRequired(backendManager, lastVolumeSize)
@@ -836,7 +836,7 @@ namespace Duplicati.Library.Main.Operation
if (m_options.UploadVerificationFile && await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload);
await FilelistProcessor.UploadVerificationFile(backendManager, m_options, m_database)
await FilelistProcessor.UploadVerificationFile(backendManager, m_options, m_database, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
Duplicati/Library/Main/Operation/CompactHandler.cs
+34 -33
View File
@@ -52,17 +52,17 @@ namespace Duplicati.Library.Main.Operation
if (!System.IO.File.Exists(m_options.Dbpath))
throw new Exception(string.Format("Database file does not exist: {0}", m_options.Dbpath));
await using (var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Compact", m_options.SqlitePageCache).ConfigureAwait(false))
await using (var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Compact", m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
{
await Utility.UpdateOptionsFromDb(db, m_options)
await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await Utility.VerifyOptionsAndUpdateDatabase(db, m_options)
await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
var changed = await DoCompactAsync(db, false, backendManager).ConfigureAwait(false);
if (changed && m_options.UploadVerificationFile)
await FilelistProcessor.UploadVerificationFile(backendManager, m_options, db);
await FilelistProcessor.UploadVerificationFile(backendManager, m_options, db, m_result.TaskControl.ProgressToken);
if (!m_options.Dryrun)
{
@@ -72,7 +72,8 @@ namespace Duplicati.Library.Main.Operation
if (changed)
{
await db.WriteResults(m_result).ConfigureAwait(false);
await db.WriteResults(m_result, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (m_options.AutoVacuum)
{
m_result.VacuumResults = new VacuumResults(m_result);
@@ -86,7 +87,7 @@ namespace Duplicati.Library.Main.Operation
internal async Task<bool> DoCompactAsync(LocalDeleteDatabase db, bool hasVerifiedBackend, IBackendManager backendManager)
{
var report = await db
.GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount)
.GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
report.ReportCompactData();
@@ -95,11 +96,11 @@ namespace Duplicati.Library.Main.Operation
{
// Workaround where we allow a running backendmanager to be used
if (!hasVerifiedBackend)
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false);
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, FilelistProcessor.VerifyMode.VerifyStrict, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
var newvol = new BlockVolumeWriter(m_options);
newvol.VolumeID = await db
.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary)
.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
IndexVolumeWriter newvolindex = null;
@@ -107,20 +108,20 @@ namespace Duplicati.Library.Main.Operation
{
newvolindex = new IndexVolumeWriter(m_options);
newvolindex.VolumeID = await db
.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary)
.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await db
.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID)
.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
var blocksInVolume = 0L;
var buffer = new byte[m_options.Blocksize];
var remoteList = await db
.GetRemoteVolumes()
.GetRemoteVolumes(m_result.TaskControl.ProgressToken)
.Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified)
.ToArrayAsync()
.ToArrayAsync(m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
//These are for bookkeeping
@@ -152,7 +153,7 @@ namespace Duplicati.Library.Main.Operation
// If we crash now, we may leave partial files
if (!m_options.Dryrun)
await db
.TerminatedWithActiveUploads(true)
.TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, true)
.ConfigureAwait(false);
newvolindex?.StartVolume(newvol.RemoteFilename);
@@ -167,7 +168,7 @@ namespace Duplicati.Library.Main.Operation
.ToList();
}
using (var q = await db.CreateBlockQueryHelper().ConfigureAwait(false))
using (var q = await db.CreateBlockQueryHelper(m_result.TaskControl.ProgressToken).ConfigureAwait(false))
{
await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(volumesToDownload, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
{
@@ -182,7 +183,7 @@ namespace Duplicati.Library.Main.Operation
downloadedVolumes.Add(new KeyValuePair<string, long>(entry.Name, entry.Size));
var volumeid = await db
.GetRemoteVolumeID(entry.Name)
.GetRemoteVolumeID(entry.Name, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
var inst = VolumeBase.ParseFilename(entry.Name);
@@ -190,7 +191,7 @@ namespace Duplicati.Library.Main.Operation
{
foreach (var e in f.Blocks)
{
if (await q.UseBlock(e.Key, e.Value, volumeid).ConfigureAwait(false))
if (await q.UseBlock(e.Key, e.Value, volumeid, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
{
//TODO: How do we get the compression hint? Reverse query for filename in db?
var s = f.ReadBlock(e.Key, buffer);
@@ -205,7 +206,7 @@ namespace Duplicati.Library.Main.Operation
newvolindex.AddBlock(e.Key, e.Value);
await db
.RegisterDuplicatedBlock(e.Key, e.Value, newvol.VolumeID)
.RegisterDuplicatedBlock(e.Key, e.Value, newvol.VolumeID, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
blocksInVolume++;
@@ -217,18 +218,18 @@ namespace Duplicati.Library.Main.Operation
newvol = new BlockVolumeWriter(m_options);
newvol.VolumeID = await db
.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary)
.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
{
newvolindex = new IndexVolumeWriter(m_options);
newvolindex.VolumeID = await db
.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary)
.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await db
.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID)
.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
newvolindex.StartVolume(newvol.RemoteFilename);
@@ -242,7 +243,7 @@ namespace Duplicati.Library.Main.Operation
// Commit as we have uploaded a volume
if (!m_options.Dryrun)
await db.Transaction
.CommitAsync("CommitCompact")
.CommitAsync("CommitCompact", true, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (deleteableVolumes.Any())
@@ -268,13 +269,13 @@ namespace Duplicati.Library.Main.Operation
else
{
await db
.RemoveRemoteVolume(newvol.RemoteFilename)
.RemoveRemoteVolume(newvol.RemoteFilename, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (newvolindex != null)
{
await db
.RemoveRemoteVolume(newvolindex.RemoteFilename)
.RemoveRemoteVolume(newvolindex.RemoteFilename, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
newvolindex.FinishVolume(null, 0);
@@ -285,7 +286,7 @@ namespace Duplicati.Library.Main.Operation
// The remainder of the operation cannot leave partial files
if (!m_options.Dryrun)
await db
.TerminatedWithActiveUploads(false)
.TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, false)
.ConfigureAwait(false);
}
else
@@ -355,27 +356,27 @@ namespace Duplicati.Library.Main.Operation
{
// Find volumes that can be deleted
var remoteFilesToRemove = await db
.ReOrderDeleteableVolumes(deleteableVolumes)
.ToListAsync()
.ReOrderDeleteableVolumes(deleteableVolumes, cancellationToken)
.ToListAsync(cancellationToken: cancellationToken)
.ConfigureAwait(false);
// Make sure we do not re-assign blocks to any of the volumes we are about to delete
var toRemoveVolumeIds = await db
.GetRemoteVolumeIDs(remoteFilesToRemove.Select(x => x.Name))
.GetRemoteVolumeIDs(remoteFilesToRemove.Select(x => x.Name), cancellationToken)
.Select(x => x.Value)
.Distinct()
.ToListAsync()
.ToListAsync(cancellationToken: cancellationToken)
.ConfigureAwait(false);
// Mark all volumes and relevant index files as disposable
foreach (var f in remoteFilesToRemove)
{
await db
.PrepareForDelete(f.Name, toRemoveVolumeIds)
.PrepareForDelete(f.Name, toRemoveVolumeIds, cancellationToken)
.ConfigureAwait(false);
await db
.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash)
.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, cancellationToken)
.ConfigureAwait(false);
}
@@ -384,7 +385,7 @@ namespace Duplicati.Library.Main.Operation
if (!m_options.Dryrun)
await db.Transaction
.CommitAsync("CommitDelete")
.CommitAsync("CommitDelete", token: cancellationToken)
.ConfigureAwait(false);
await foreach (var d in PerformDelete(backend, remoteFilesToRemove, cancellationToken).ConfigureAwait(false))
@@ -397,7 +398,7 @@ namespace Duplicati.Library.Main.Operation
if (newvolindex != null && m_options.IndexfilePolicy == Options.IndexFileStrategy.Full)
indexVolumeFinished = async () =>
{
await foreach (var blocklist in db.GetBlocklists(newvol.VolumeID, m_options.Blocksize, m_options.BlockhashSize).ConfigureAwait(false))
await foreach (var blocklist in db.GetBlocklists(newvol.VolumeID, m_options.Blocksize, m_options.BlockhashSize, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
newvolindex.WriteBlocklist(blocklist.Hash, blocklist.Buffer, 0, blocklist.Size);
};
@@ -409,7 +410,7 @@ namespace Duplicati.Library.Main.Operation
// because the transaction is not thread-safe, and shared with the upload
await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
await db
.UpdateRemoteVolume(newvol.RemoteFilename, RemoteVolumeState.Uploading, -1, null)
.UpdateRemoteVolume(newvol.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
// TODO: The upload here does not flush the database messages,
Duplicati/Library/Main/Operation/CreateBugReportHandler.cs
+3 -3
View File
@@ -68,11 +68,11 @@ namespace Duplicati.Library.Main.Operation
using (var tmp = new TempFile())
{
File.Copy(m_options.Dbpath, tmp, true);
await using (var db = await LocalBugReportDatabase.CreateAsync(tmp, m_options.SqlitePageCache).ConfigureAwait(false))
await using (var db = await LocalBugReportDatabase.CreateAsync(tmp, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
{
await db.Fix().ConfigureAwait(false);
await db.Fix(m_result.TaskControl.ProgressToken).ConfigureAwait(false);
if (m_options.AutoVacuum)
await db.Vacuum().ConfigureAwait(false);
await db.Vacuum(m_result.TaskControl.ProgressToken).ConfigureAwait(false);
}
// Apply zip64 option if not already set
Duplicati/Library/Main/Operation/DeleteHandler.cs
+11 -10
View File
@@ -51,11 +51,11 @@ namespace Duplicati.Library.Main.Operation
if (!System.IO.File.Exists(m_options.Dbpath))
throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseFileMissing");
await using var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Delete", m_options.SqlitePageCache).ConfigureAwait(false);
await Utility.UpdateOptionsFromDb(db, m_options)
await using var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Delete", m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await Utility.VerifyOptionsAndUpdateDatabase(db, m_options)
await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await DoRunAsync(db, false, false, backendManager).ConfigureAwait(false);
@@ -69,13 +69,13 @@ namespace Duplicati.Library.Main.Operation
public async Task DoRunAsync(LocalDeleteDatabase db, bool hasVerifiedBackend, bool forceCompact, IBackendManager backendManager)
{
if (!hasVerifiedBackend)
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false);
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
// We collapse the async enumerablo into a array to avoid multiple
// enumerations (and thus multiple database queries)
var filesets = await db
.FilesetsWithBackupVersion()
.ToArrayAsync()
.FilesetsWithBackupVersion(m_result.TaskControl.ProgressToken)
.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
List<IListResultFileset> versionsToDelete =
@@ -109,19 +109,20 @@ namespace Duplicati.Library.Main.Operation
.DropFilesetsFromTable(
versionsToDelete
.Select(x => x.Time)
.ToArray()
.ToArray(),
m_result.TaskControl.ProgressToken
)
.ToArrayAsync()
.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
foreach (var f in lst)
await db
.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null)
.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (!m_options.Dryrun)
await db.Transaction
.CommitAsync("CommitBeforeDelete")
.CommitAsync("CommitBeforeDelete", true, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
foreach (var f in lst)
Duplicati/Library/Main/Operation/FilelistProcessor.cs
+42 -30
View File
@@ -71,7 +71,7 @@ namespace Duplicati.Library.Main.Operation
/// <returns>A task that completes when the verification is done.</returns>
public static async Task VerifyLocalList(IBackendManager backendManager, LocalDatabase database, CancellationToken cancellationToken)
{
var locallist = database.GetRemoteVolumes();
var locallist = database.GetRemoteVolumes(cancellationToken);
await foreach (var i in locallist.ConfigureAwait(false))
{
switch (i.State)
@@ -115,20 +115,21 @@ namespace Duplicati.Library.Main.Operation
/// <param name="backendWriter">The backend writer to use.</param>
/// <param name="latestVolumesOnly">True if only the latest volumes should be verified.</param>
/// <param name="verifyMode">The mode to use for verification.</param>
/// <param name="cancellationToken">The cancellation token to use.</param>
/// <returns>An awaitable task.</returns>
public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter backendWriter, bool latestVolumesOnly, VerifyMode verifyMode)
public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter backendWriter, bool latestVolumesOnly, VerifyMode verifyMode, CancellationToken cancellationToken)
{
if (!options.NoBackendverification)
{
LocalBackupDatabase backupDatabase =
await LocalBackupDatabase.CreateAsync(database, options)
await LocalBackupDatabase.CreateAsync(database, options, null, cancellationToken)
.ConfigureAwait(false);
IEnumerable<string> protectedFiles = await backupDatabase
.GetTemporaryFilelistVolumeNames(latestVolumesOnly)
.GetTemporaryFilelistVolumeNames(latestVolumesOnly, cancellationToken)
.ConfigureAwait(false);
await VerifyRemoteList(backend, options, database, backendWriter, protectedFiles, [], logErrors: true, verifyMode: verifyMode).ConfigureAwait(false);
await VerifyRemoteList(backend, options, database, backendWriter, protectedFiles, [], logErrors: true, verifyMode: verifyMode, cancellationToken).ConfigureAwait(false);
}
}
@@ -144,10 +145,11 @@ namespace Duplicati.Library.Main.Operation
/// <param name="strictExcemptFiles">Filenames that should be exempted from strict verification.</param>
/// <param name="logErrors">Disable the logging of errors to prevent spamming the log; exceptions will be thrown regardless.</param>
/// <param name="verifyMode">The mode to use for verification.</param>
/// <param name="cancellationToken">The cancellation token to use.</param>
/// <returns>An awaitable task.</returns>
public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable<string> protectedFiles, IEnumerable<string> strictExcemptFiles, bool logErrors, VerifyMode verifyMode)
public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable<string> protectedFiles, IEnumerable<string> strictExcemptFiles, bool logErrors, VerifyMode verifyMode, CancellationToken cancellationToken)
{
var tp = await RemoteListAnalysis(backend, options, database, log, protectedFiles, strictExcemptFiles, verifyMode).ConfigureAwait(false);
var tp = await RemoteListAnalysis(backend, options, database, log, protectedFiles, strictExcemptFiles, verifyMode, cancellationToken).ConfigureAwait(false);
long extraCount = 0;
long missingCount = 0;
@@ -213,10 +215,17 @@ namespace Duplicati.Library.Main.Operation
/// <returns>The verification file.</returns>
/// <param name="db">The database instance</param>
/// <param name="stream">The stream to write to</param>
public static void CreateVerificationFile(LocalDatabase db, System.IO.StreamWriter stream)
/// <param name="cancellationToken">The cancellation token to use.</param>
public static void CreateVerificationFile(LocalDatabase db, System.IO.StreamWriter stream, CancellationToken cancellationToken)
{
var s = new Newtonsoft.Json.JsonSerializer();
s.Serialize(stream, db.GetRemoteVolumes().Where(x => x.State != RemoteVolumeState.Temporary).ToEnumerable().Cast<IRemoteVolume>().ToArray());
s.Serialize(stream,
db.GetRemoteVolumes(cancellationToken)
.Where(x => x.State != RemoteVolumeState.Temporary)
.ToEnumerable()
.Cast<IRemoteVolume>()
.ToArray()
);
}
/// <summary>
@@ -225,14 +234,15 @@ namespace Duplicati.Library.Main.Operation
/// <param name="backendManager">The backend manager instance to use.</param>
/// <param name="options">The options to use.</param>
/// <param name="db">The attached database.</param>
/// <param name="cancellationToken">The cancellation token to use.</param>
/// <returns>A task that completes when the upload is done.</returns>
public static async Task UploadVerificationFile(IBackendManager backendManager, Options options, LocalDatabase db)
public static async Task UploadVerificationFile(IBackendManager backendManager, Options options, LocalDatabase db, CancellationToken cancellationToken)
{
using (var tempfile = new Library.Utility.TempFile())
{
var remotename = options.Prefix + "-verification.json";
await using (var stream = new System.IO.StreamWriter(tempfile, false, System.Text.Encoding.UTF8))
CreateVerificationFile(db, stream);
CreateVerificationFile(db, stream, cancellationToken);
if (options.Dryrun)
{
@@ -240,8 +250,8 @@ namespace Duplicati.Library.Main.Operation
}
else
{
await backendManager.PutVerificationFileAsync(remotename, tempfile, CancellationToken.None).ConfigureAwait(false);
await backendManager.WaitForEmptyAsync(db, CancellationToken.None).ConfigureAwait(false);
await backendManager.PutVerificationFileAsync(remotename, tempfile, cancellationToken).ConfigureAwait(false);
await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false);
}
}
}
@@ -257,18 +267,19 @@ namespace Duplicati.Library.Main.Operation
/// <param name="protectedFiles">Filenames that should be exempted from deletion.</param>
/// <param name="strictExcemptFiles">Filenames that should be exempted from strict verification.</param>
/// <param name="verifyMode">The mode to use for verification.</param>
/// <param name="cancellationToken">The cancellation token to use.</param>
/// <returns>A task that when awaited contains the analysis result.</returns>
public static async Task<RemoteAnalysisResult> RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IBackendWriter log, IEnumerable<string> protectedFiles, IEnumerable<string> strictExcemptFiles, VerifyMode verifyMode)
public static async Task<RemoteAnalysisResult> RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IBackendWriter log, IEnumerable<string> protectedFiles, IEnumerable<string> strictExcemptFiles, VerifyMode verifyMode, CancellationToken cancellationToken)
{
// If the last operation completed, no cleanup should be required
if (verifyMode == VerifyMode.VerifyAndClean && !await database.TerminatedWithActiveUploads().ConfigureAwait(false))
if (verifyMode == VerifyMode.VerifyAndClean && !await database.TerminatedWithActiveUploads(cancellationToken).ConfigureAwait(false))
verifyMode = VerifyMode.VerifyStrict;
// Force cleanup should set the mode to cleanup
if (verifyMode == VerifyMode.VerifyAndCleanForced)
verifyMode = VerifyMode.VerifyAndClean;
var rawlist = await backendManager.ListAsync(CancellationToken.None).ConfigureAwait(false);
var rawlist = await backendManager.ListAsync(cancellationToken).ConfigureAwait(false);
var lookup = new Dictionary<string, Volumes.IParsedVolume>();
protectedFiles = protectedFiles ?? [];
strictExcemptFiles = strictExcemptFiles ?? [];
@@ -300,8 +311,8 @@ namespace Duplicati.Library.Main.Operation
log.UnknownFileCount = unknownlist.Count;
log.UnknownFileSize = unknownlist.Select(x => Math.Max(0, x.Size)).Sum();
log.BackupListCount = await database
.FilesetTimes()
.CountAsync()
.FilesetTimes(cancellationToken)
.CountAsync(cancellationToken: cancellationToken)
.ConfigureAwait(false);
log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime();
@@ -316,17 +327,17 @@ namespace Duplicati.Library.Main.Operation
var missingUploadingVolumes = new HashSet<string>();
var protectedVolumes = new HashSet<string>();
await foreach (var e in database.DuplicateRemoteVolumes().ConfigureAwait(false))
await foreach (var e in database.DuplicateRemoteVolumes(cancellationToken).ConfigureAwait(false))
{
if (e.Value == RemoteVolumeState.Uploading || e.Value == RemoteVolumeState.Temporary)
await database
.UnlinkRemoteVolume(e.Key, e.Value)
.UnlinkRemoteVolume(e.Key, e.Value, cancellationToken)
.ConfigureAwait(false);
else
throw new RemoteListVerificationException(string.Format("The remote volume {0} appears in the database with state {1} and a deleted state, cannot continue", e.Key, e.Value.ToString()), "AmbiguousStateRemoteFiles");
}
var locallist = database.GetRemoteVolumes();
var locallist = database.GetRemoteVolumes(cancellationToken);
await foreach (var i in locallist.ConfigureAwait(false))
{
var remoteFound = lookup.TryGetValue(i.Name, out var r);
@@ -336,11 +347,11 @@ namespace Duplicati.Library.Main.Operation
correctSize = true;
if (archived && i.ArchiveTime <= DateTime.UnixEpoch)
await database
.UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, true)
.UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, true, cancellationToken)
.ConfigureAwait(false);
else if (!archived && i.ArchiveTime > DateTime.UnixEpoch)
await database
.UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, false)
.UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, false, cancellationToken)
.ConfigureAwait(false);
lookup.Remove(i.Name);
@@ -397,7 +408,7 @@ namespace Duplicati.Library.Main.Operation
{
Logging.Log.WriteInformationMessage(LOGTAG, "PromotingCompleteFile", "Promoting uploaded complete file from {0} to {2}: {1}", i.State, i.Name, RemoteVolumeState.Uploaded);
await database
.UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash)
.UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash, cancellationToken)
.ConfigureAwait(false);
}
else if (!remoteFound)
@@ -407,7 +418,7 @@ namespace Duplicati.Library.Main.Operation
protectedVolumes.Add(i.Name);
Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "Keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name);
await database
.UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null)
.UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null, cancellationToken)
.ConfigureAwait(false);
}
else if (verifyMode == VerifyMode.VerifyStrict && !strictExcemptFiles.Any(pf => pf == i.Name))
@@ -420,7 +431,7 @@ namespace Duplicati.Library.Main.Operation
Logging.Log.WriteInformationMessage(LOGTAG, "SchedulingMissingFileForDelete", "Scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name);
missingUploadingVolumes.Add(i.Name);
await database
.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null)
.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null, cancellationToken)
.ConfigureAwait(false);
}
}
@@ -460,7 +471,7 @@ namespace Duplicati.Library.Main.Operation
missing.Add(i);
else if (correctSize)
await database
.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash)
.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash, cancellationToken)
.ConfigureAwait(false);
else
missingHash.Add(new Tuple<long, RemoteVolumeEntry>(r.File.Size, i));
@@ -491,17 +502,18 @@ namespace Duplicati.Library.Main.Operation
await database
.RemoveRemoteVolumes(
missingUploadingVolumes
.Concat(temporaryAndDeletedVolumes)
.Concat(temporaryAndDeletedVolumes),
cancellationToken
)
.ConfigureAwait(false);
// Clear the flag after we have cleaned up
if (!options.Dryrun && protectedVolumes.Count == 0)
await database
.TerminatedWithActiveUploads(false)
.TerminatedWithActiveUploads(cancellationToken, false)
.ConfigureAwait(false);
}
else if (verifyMode == VerifyMode.VerifyOnly && await database.TerminatedWithActiveUploads().ConfigureAwait(false))
else if (verifyMode == VerifyMode.VerifyOnly && await database.TerminatedWithActiveUploads(cancellationToken).ConfigureAwait(false))
{
Logging.Log.WriteWarningMessage(LOGTAG, "ActiveUploadsDetected", null, "Active uploads detected, but no cleanup was performed. Run the \"repair\" command to clean up the incomplete files");
}
Duplicati/Library/Main/Operation/ListAffected.cs
+9 -9
View File
@@ -44,21 +44,21 @@ namespace Duplicati.Library.Main.Operation
if (!File.Exists(m_options.Dbpath))
throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist");
await using var db = await Database.LocalListAffectedDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false);
await using var db = await Database.LocalListAffectedDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
if (callback == null)
{
m_result.SetResult(
await db.GetFilesets(args)
await db.GetFilesets(args, m_result.TaskControl.ProgressToken)
.OrderByDescending(x => x.Time)
.ToArrayAsync()
.ConfigureAwait(false),
await db.GetFiles(args)
await db.GetFiles(args, m_result.TaskControl.ProgressToken)
.ToArrayAsync()
.ConfigureAwait(false),
await db.GetLogLines(args)
await db.GetLogLines(args, m_result.TaskControl.ProgressToken)
.ToArrayAsync()
.ConfigureAwait(false),
await db.GetVolumes(args)
await db.GetVolumes(args, m_result.TaskControl.ProgressToken)
.ToArrayAsync()
.ConfigureAwait(false)
);
@@ -66,17 +66,17 @@ namespace Duplicati.Library.Main.Operation
else
{
m_result.SetResult(
await db.GetFilesets(args)
await db.GetFilesets(args, m_result.TaskControl.ProgressToken)
.OrderByDescending(x => x.Time)
.ToArrayAsync()
.ConfigureAwait(false),
await db.GetFiles(args)
await db.GetFiles(args, m_result.TaskControl.ProgressToken)
.ToArrayAsync()
.ConfigureAwait(false),
await db.GetLogLines(args)
await db.GetLogLines(args, m_result.TaskControl.ProgressToken)
.ToArrayAsync()
.ConfigureAwait(false),
await db.GetVolumes(args)
await db.GetVolumes(args, m_result.TaskControl.ProgressToken)
.ToArrayAsync()
.ConfigureAwait(false)
);
Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs
+18 -18
View File
@@ -52,10 +52,10 @@ namespace Duplicati.Library.Main.Operation
if (!File.Exists(m_options.Dbpath))
throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist");
await using var db = await Database.LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false);
await Utility.UpdateOptionsFromDb(db, m_options)
await using var db = await Database.LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await Utility.VerifyOptionsAndUpdateDatabase(db, m_options)
await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await DoRunAsync(backendManager, db, filter, callbackhandler).ConfigureAwait(false);
await db.Transaction.RollBackAsync().ConfigureAwait(false);
@@ -65,18 +65,18 @@ namespace Duplicati.Library.Main.Operation
{
List<Database.RemoteVolumeEntry>? missing = null;
var brokensets = await db
.GetBrokenFilesets(options.Time, options.Version)
.ToArrayAsync()
.GetBrokenFilesets(options.Time, options.Version, result.TaskControl.ProgressToken)
.ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (brokensets.Length == 0)
{
if (await db.RepairInProgress().ConfigureAwait(false))
if (await db.RepairInProgress(result.TaskControl.ProgressToken).ConfigureAwait(false))
throw new UserInformationException("Cannot continue because the database is marked as being under repair, but does not have broken files.", "CannotListOnDatabaseInRepair");
Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsInDatabase", "No broken filesets found in database, checking for missing remote files");
var remotestate = await FilelistProcessor.RemoteListAnalysis(backendManager, options, db, result.BackendWriter, null, null, FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false);
var remotestate = await FilelistProcessor.RemoteListAnalysis(backendManager, options, db, result.BackendWriter, null, null, FilelistProcessor.VerifyMode.VerifyOnly, result.TaskControl.ProgressToken).ConfigureAwait(false);
if (!remotestate.ParsedVolumes.Any())
throw new UserInformationException("No remote volumes were found, refusing purge", "CannotPurgeWithNoRemoteVolumes");
@@ -90,24 +90,24 @@ namespace Duplicati.Library.Main.Operation
// Mark all volumes as disposable
foreach (var f in missing)
await db
.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash)
.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
Logging.Log.WriteInformationMessage(LOGTAG, "MarkedRemoteFilesForDeletion", "Marked {0} remote files for deletion", missing.Count);
// Drop all content from tables
await db
.RemoveMissingBlocks(missing.Select(x => x.Name))
.RemoveMissingBlocks(missing.Select(x => x.Name), result.TaskControl.ProgressToken)
.ConfigureAwait(false);
// Mark all orphaned index files as disposable after removing the missing block files
await foreach (var f in db.GetOrphanedIndexFiles().ConfigureAwait(false))
await foreach (var f in db.GetOrphanedIndexFiles(result.TaskControl.ProgressToken).ConfigureAwait(false))
await db
.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash)
.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
brokensets = await db
.GetBrokenFilesets(options.Time, options.Version)
.GetBrokenFilesets(options.Time, options.Version, result.TaskControl.ProgressToken)
.ToArrayAsync()
.ConfigureAwait(false);
}
@@ -120,7 +120,7 @@ namespace Duplicati.Library.Main.Operation
if (filter != null && !filter.Empty)
throw new UserInformationException("Filters are not supported for this operation", "FiltersAreNotSupportedForListBrokenFiles");
if (await db.PartiallyRecreated().ConfigureAwait(false))
if (await db.PartiallyRecreated(m_result.TaskControl.ProgressToken).ConfigureAwait(false))
throw new UserInformationException("The command does not work on partially recreated databases", "ListBrokenFilesDoesNotWorkOnPartialDatabase");
(var brokensets, var missing) = await GetBrokenFilesetsFromRemote(backendManager, m_result, db, m_options).ConfigureAwait(false);
@@ -142,8 +142,8 @@ namespace Duplicati.Library.Main.Operation
}
var fstimes = await db
.FilesetTimes()
.ToListAsync()
.FilesetTimes(m_result.TaskControl.ProgressToken)
.ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
var brokenfilesets =
@@ -165,8 +165,8 @@ namespace Duplicati.Library.Main.Operation
x.Timestamp,
callbackhandler == null && !m_options.ListSetsOnly
? await db
.GetBrokenFilenames(x.FilesetID)
.ToArrayAsync()
.GetBrokenFilenames(x.FilesetID, m_result.TaskControl.ProgressToken)
.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken)
.ConfigureAwait(false)
: new MockList<Tuple<string, long>>((int)x.BrokenCount)
)))
@@ -175,7 +175,7 @@ namespace Duplicati.Library.Main.Operation
if (callbackhandler != null)
foreach (var bs in brokenfilesets)
await foreach (var fe in db.GetBrokenFilenames(bs.FilesetID).ConfigureAwait(false))
await foreach (var fe in db.GetBrokenFilenames(bs.FilesetID, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
if (!callbackhandler(bs.Version, bs.Timestamp, bs.BrokenCount, fe.Item1, fe.Item2))
break;
}
Duplicati/Library/Main/Operation/ListChangesHandler.cs
+12 -12
View File
@@ -80,13 +80,13 @@ namespace Duplicati.Library.Main.Operation
DateTime compareVersionTime;
using var tmpdb = useLocalDb ? null : new TempFile();
await using var db = await Database.LocalListChangesDatabase.CreateAsync(useLocalDb ? m_options.Dbpath : (string)tmpdb, m_options.SqlitePageCache).ConfigureAwait(false);
await using var storageKeeper = await db.CreateStorageHelper().ConfigureAwait(false);
await using var db = await Database.LocalListChangesDatabase.CreateAsync(useLocalDb ? m_options.Dbpath : (string)tmpdb, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
await using var storageKeeper = await db.CreateStorageHelper(m_result.TaskControl.ProgressToken).ConfigureAwait(false);
if (useLocalDb)
{
var dbtimes = await db
.FilesetTimes()
.ToListAsync()
.FilesetTimes(m_result.TaskControl.ProgressToken)
.ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (dbtimes.Count < 2)
@@ -101,10 +101,10 @@ namespace Duplicati.Library.Main.Operation
SelectTime(compareVersion, times, out compareVersionIndex, out compareVersionTime, out compareVersionId);
await storageKeeper
.AddFromDb(baseVersionId, false, filter)
.AddFromDb(baseVersionId, false, filter, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await storageKeeper
.AddFromDb(compareVersionId, true, filter)
.AddFromDb(compareVersionId, true, filter, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
else
@@ -151,7 +151,7 @@ namespace Duplicati.Library.Main.Operation
foreach (var f in rd.Files)
if (FilterExpression.Matches(filter, f.Path))
await storageKeeper
.AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), false)
.AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), false, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
@@ -162,20 +162,20 @@ namespace Duplicati.Library.Main.Operation
foreach (var f in rd.Files)
if (FilterExpression.Matches(filter, f.Path))
await storageKeeper
.AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), true)
.AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), true, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
var changes = await storageKeeper
.CreateChangeCountReport()
.CreateChangeCountReport(m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
var sizes = await storageKeeper
.CreateChangeSizeReport()
.CreateChangeSizeReport(m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
var lst = (m_options.FullResult || callback != null) ?
(from n in storageKeeper.CreateChangedFileReport()
(from n in storageKeeper.CreateChangedFileReport(m_result.TaskControl.ProgressToken)
select n) : null;
m_result.SetResult(
@@ -184,7 +184,7 @@ namespace Duplicati.Library.Main.Operation
changes.DeletedFolders, changes.DeletedSymlinks, changes.DeletedFiles,
changes.ModifiedFolders, changes.ModifiedSymlinks, changes.ModifiedFiles,
sizes.AddedSize, sizes.DeletedSize, sizes.PreviousSize, sizes.CurrentSize,
(lst == null || callback == null) ? null : await lst.ToArrayAsync().ConfigureAwait(false)
(lst == null || callback == null) ? null : await lst.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken).ConfigureAwait(false)
);
if (callback != null)
Duplicati/Library/Main/Operation/ListControlFilesHandler.cs
+2 -2
View File
@@ -43,7 +43,7 @@ namespace Duplicati.Library.Main.Operation
{
var cancellationToken = CancellationToken.None;
using var tmpdb = new TempFile();
await using var db = await LocalDatabase.CreateLocalDatabaseAsync(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true, m_options.SqlitePageCache).ConfigureAwait(false);
await using var db = await LocalDatabase.CreateLocalDatabaseAsync(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
var filter = JoinedFilterExpression.Join(new Library.Utility.FilterExpression(filterstrings), compositefilter);
try
@@ -62,7 +62,7 @@ namespace Duplicati.Library.Main.Operation
var file = fileversion.Value.File;
var entry = await db
.GetRemoteVolume(file.Name)
.GetRemoteVolume(file.Name, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
var files = new List<Library.Interface.IListResultFile>();
Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs
+4 -4
View File
@@ -48,14 +48,14 @@ internal static class ListFileVersionsHandler
throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase");
await using var db =
await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache)
await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache, null, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
long[]? filesetIds = null;
if (!options.AllVersions)
{
filesetIds = await db
.GetFilesetIDs(options.Time, options.Version)
.ToArrayAsync()
.GetFilesetIDs(options.Time, options.Version, false, result.TaskControl.ProgressToken)
.ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (filesetIds.Length == 0)
@@ -67,7 +67,7 @@ internal static class ListFileVersionsHandler
paths = paths.Select(path => Util.AppendDirSeparator(path)).ToArray();
result.FileVersions = await db
.ListFileVersions(paths, filesetIds, offset, limit)
.ListFileVersions(paths, filesetIds, offset, limit, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
}
Duplicati/Library/Main/Operation/ListFilesHandler.cs
+14 -14
View File
@@ -55,15 +55,15 @@ namespace Duplicati.Library.Main.Operation
//Use a speedy local query
if (!m_options.NoLocalDb && System.IO.File.Exists(m_options.Dbpath))
await using (var db = await Database.LocalListDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false))
await using (var db = await Database.LocalListDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
{
await using var filesets = await db.SelectFileSets(m_options.Time, m_options.Version).ConfigureAwait(false);
await using var filesets = await db.SelectFileSets(m_options.Time, m_options.Version, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
if (!filter.Empty)
{
if (simpleList || (m_options.ListFolderContents && !m_options.AllVersions))
{
await filesets
.TakeFirst()
.TakeFirst(m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
}
@@ -71,11 +71,11 @@ namespace Duplicati.Library.Main.Operation
IAsyncEnumerable<Database.LocalListDatabase.IFileversion> files;
if (m_options.ListFolderContents)
{
files = filesets.SelectFolderContents(filter);
files = filesets.SelectFolderContents(filter, m_result.TaskControl.ProgressToken);
}
else if (m_options.ListPrefixOnly)
{
files = filesets.GetLargestPrefix(filter);
files = filesets.GetLargestPrefix(filter, m_result.TaskControl.ProgressToken);
}
else if (filter.Empty)
{
@@ -83,16 +83,16 @@ namespace Duplicati.Library.Main.Operation
}
else
{
files = filesets.SelectFiles(filter);
files = filesets.SelectFiles(filter, m_result.TaskControl.ProgressToken);
}
if (m_options.ListSetsOnly)
{
m_result.SetResult(
await filesets
.QuickSets()
.QuickSets(m_result.TaskControl.ProgressToken)
.Select(x => new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes))
.ToArrayAsync()
.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken)
.ConfigureAwait(false),
null
);
@@ -101,9 +101,9 @@ namespace Duplicati.Library.Main.Operation
{
m_result.SetResult(
await filesets
.Sets()
.Sets(m_result.TaskControl.ProgressToken)
.Select(x => new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes))
.ToArrayAsync()
.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken)
.ConfigureAwait(false),
files == null
? null
@@ -112,14 +112,14 @@ namespace Duplicati.Library.Main.Operation
new ListResultFile(
n.Path,
await n
.Sizes()
.ToArrayAsync()
.Sizes(m_result.TaskControl.ProgressToken)
.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken)
.ConfigureAwait(false)
)
)
.Select(x => x.Result)
.Cast<IListResultFile>()
.ToArrayAsync()
.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken)
.ConfigureAwait(false)
);
}
@@ -137,7 +137,7 @@ namespace Duplicati.Library.Main.Operation
// Otherwise, grab info from remote location
using (var tmpdb = new TempFile())
await using (var db = await LocalDatabase.CreateLocalDatabaseAsync(tmpdb, "List", true, m_options.SqlitePageCache).ConfigureAwait(false))
await using (var db = await LocalDatabase.CreateLocalDatabaseAsync(tmpdb, "List", true, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
{
var filteredList = ParseAndFilterFilesets(await backendManager.ListAsync(cancellationToken).ConfigureAwait(false), m_options);
if (filteredList.Count == 0)
Duplicati/Library/Main/Operation/ListFilesetsHandler.cs
+4 -3
View File
@@ -22,6 +22,7 @@
#nullable enable
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Duplicati.Library.Interface;
@@ -49,12 +50,12 @@ internal static class ListFilesetsHandler
// Use a speedy local query
if (System.IO.File.Exists(options.Dbpath) && !options.NoLocalDb)
{
await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache)
await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache, null, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
result.Filesets = await db
.ListFilesetsExtended()
.ToArrayAsync()
.ListFilesetsExtended(result.TaskControl.ProgressToken)
.ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken)
.ConfigureAwait(false);
return;
Duplicati/Library/Main/Operation/ListFolderHandler.cs
+9 -7
View File
@@ -46,12 +46,12 @@ internal static class ListFolderHandler
if (!System.IO.File.Exists(options.Dbpath) || options.NoLocalDb)
throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase");
await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache)
await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache, null, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
var filesetIds = await db
.GetFilesetIDs(options.Time, options.Version, singleTimeMatch: true)
.ToArrayAsync()
.GetFilesetIDs(options.Time, options.Version, singleTimeMatch: true, result.TaskControl.ProgressToken)
.ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (filesetIds.Length == 0)
@@ -65,10 +65,11 @@ internal static class ListFolderHandler
throw new UserInformationException("When no folder is specified, only one folder can be listed", "MultipleFoldersFound");
result.Entries = await db
.ListFilesetEntries(
db.GetRootPrefixes(filesetIds[0]).ToEnumerable(),
db.GetRootPrefixes(filesetIds[0], result.TaskControl.ProgressToken).ToEnumerable(),
filesetIds[0],
offset,
limit
limit,
result.TaskControl.ProgressToken
)
.ConfigureAwait(false);
}
@@ -76,10 +77,11 @@ internal static class ListFolderHandler
{
result.Entries = await db
.ListFolder(
db.GetPrefixIds(folders).ToEnumerable(),
db.GetPrefixIds(folders, result.TaskControl.ProgressToken).ToEnumerable(),
filesetIds[0],
offset,
limit
limit,
result.TaskControl.ProgressToken
)
.ConfigureAwait(false);
}
Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs
+23 -22
View File
@@ -54,13 +54,13 @@ namespace Duplicati.Library.Main.Operation
if (filter != null && !filter.Empty)
throw new UserInformationException("Filters are not supported for this operation", "FiltersNotAllowedOnPurgeBrokenFiles");
await using var db = await LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false);
if (await db.PartiallyRecreated().ConfigureAwait(false))
await using var db = await LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
if (await db.PartiallyRecreated(m_result.TaskControl.ProgressToken).ConfigureAwait(false))
throw new UserInformationException("The command does not work on partially recreated databases", "CannotPurgeOnPartialDatabase");
await Utility.UpdateOptionsFromDb(db, m_options)
await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await Utility.VerifyOptionsAndUpdateDatabase(db, m_options)
await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
var (sets, missing) = await ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(backendManager, m_result, db, m_options).ConfigureAwait(false);
@@ -84,8 +84,8 @@ namespace Duplicati.Library.Main.Operation
var pgspan = 0.95f / sets.Length;
var filesets = await db
.FilesetTimes()
.ToListAsync()
.FilesetTimes(m_result.TaskControl.ProgressToken)
.ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
var compare_list = sets.Select(async x => new
@@ -95,7 +95,7 @@ namespace Duplicati.Library.Main.Operation
RemoveCount = x.RemoveCount,
Version = filesets.FindIndex(y => y.Key == x.FilesetID),
SetCount = await db
.GetFilesetFileCount(x.FilesetID)
.GetFilesetFileCount(x.FilesetID, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false)
})
.Select(x => x.Result)
@@ -109,7 +109,8 @@ namespace Duplicati.Library.Main.Operation
.GetEmptyMetadataBlocksetId(
(missing ?? []).Select(x => x.ID),
emptymetadata.FileHash,
emptymetadata.Blob.Length
emptymetadata.Blob.Length,
m_result.TaskControl.ProgressToken
)
.ConfigureAwait(false);
if (replacementMetadataBlocksetId < 0)
@@ -119,11 +120,11 @@ namespace Duplicati.Library.Main.Operation
var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray();
var to_purge = compare_list.Where(x => x.RemoveCount != x.SetCount).ToArray();
if (fully_emptied.Length == await db.FilesetTimes().CountAsync().ConfigureAwait(false))
if (fully_emptied.Length == await db.FilesetTimes(m_result.TaskControl.ProgressToken).CountAsync(cancellationToken: m_result.TaskControl.ProgressToken).ConfigureAwait(false))
throw new UserInformationException("All filesets are fully broken and needs to be removed. To avoid unexpected deletions, you must manually remove the remote files and delete the database.", "AllFilesetsBroken");
if (!m_options.Dryrun)
await db.Transaction.CommitAsync().ConfigureAwait(false);
await db.Transaction.CommitAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false);
if (fully_emptied.Length != 0)
@@ -134,7 +135,7 @@ namespace Duplicati.Library.Main.Operation
Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing {0} filesets where all file(s) are broken: {1}", fully_emptied.Length, string.Join(", ", fully_emptied.Select(x => x.Timestamp.ToLocalTime().ToString())));
m_result.DeleteResults = new DeleteResults(m_result);
await using (var rmdb = await LocalDeleteDatabase.CreateAsync(db).ConfigureAwait(false))
await using (var rmdb = await LocalDeleteDatabase.CreateAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
{
var opts = new Options(new Dictionary<string, string?>(m_options.RawOptions));
opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString()));
@@ -146,7 +147,7 @@ namespace Duplicati.Library.Main.Operation
if (!m_options.Dryrun)
await rmdb.Transaction
.CommitAsync("CommitDelete")
.CommitAsync("CommitDelete", true, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
@@ -163,12 +164,12 @@ namespace Duplicati.Library.Main.Operation
Logging.Log.WriteInformationMessage(LOGTAG, "PurgingFiles", "Purging {0} file(s) from fileset {1}", bs.RemoveCount, bs.Timestamp.ToLocalTime());
var opts = new Options(new Dictionary<string, string?>(m_options.RawOptions));
await using (var pgdb = await Database.LocalPurgeDatabase.CreateAsync(db).ConfigureAwait(false))
await using (var pgdb = await Database.LocalPurgeDatabase.CreateAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
{
// Recompute the version number after we deleted the versions before
filesets = await pgdb
.FilesetTimes()
.ToListAsync()
.FilesetTimes(m_result.TaskControl.ProgressToken)
.ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
var thisversion = filesets.FindIndex(y => y.Key == bs.FilesetID);
if (thisversion < 0)
@@ -186,9 +187,9 @@ namespace Duplicati.Library.Main.Operation
// Update entries that would be removed because of missing metadata
var updatedEntries = 0;
if (!m_options.DisableReplaceMissingMetadata)
updatedEntries = await db.ReplaceMetadata(filesetid, replacementMetadataBlocksetId);
updatedEntries = await db.ReplaceMetadata(filesetid, replacementMetadataBlocksetId, m_result.TaskControl.ProgressToken);
await db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID");
await db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID", m_result.TaskControl.ProgressToken);
return updatedEntries;
}).ConfigureAwait(false);
}
@@ -201,20 +202,20 @@ namespace Duplicati.Library.Main.Operation
m_result.OperationProgressUpdater.UpdateProgress(0.95f);
if (!m_options.Dryrun && await db.RepairInProgress().ConfigureAwait(false))
if (!m_options.Dryrun && await db.RepairInProgress(m_result.TaskControl.ProgressToken).ConfigureAwait(false))
{
Logging.Log.WriteInformationMessage(LOGTAG, "ValidatingDatabase", "Database was previously marked as in-progress, checking if it is valid after purging files");
await db
.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true)
.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
Logging.Log.WriteInformationMessage(LOGTAG, "UpdatingDatabase", "Purge completed, and consistency checks completed, marking database as complete");
await db.RepairInProgress(false).ConfigureAwait(false);
await db.RepairInProgress(m_result.TaskControl.ProgressToken, false).ConfigureAwait(false);
}
else
{
await db.Transaction.RollBackAsync().ConfigureAwait(false);
await db.Transaction.RollBackAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false);
await db
.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true)
.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
Duplicati/Library/Main/Operation/PurgeFilesHandler.cs
+31 -31
View File
@@ -52,10 +52,10 @@ namespace Duplicati.Library.Main.Operation
if (!System.IO.File.Exists(m_options.Dbpath))
throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist");
await using var db = await Database.LocalPurgeDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false);
await using var db = await Database.LocalPurgeDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
await DoRunAsync(backendManager, db, filter, null, 0, 1).ConfigureAwait(false);
await db
.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true)
.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
@@ -69,44 +69,44 @@ namespace Duplicati.Library.Main.Operation
var doCompactStep = !m_options.NoAutoCompact && filtercommand == null;
if (await db.PartiallyRecreated().ConfigureAwait(false))
if (await db.PartiallyRecreated(m_result.TaskControl.ProgressToken).ConfigureAwait(false))
throw new UserInformationException("The purge command does not work on partially recreated databases", "PurgeNotAllowedOnPartialDatabase");
if (await db.RepairInProgress().ConfigureAwait(false) && filtercommand == null)
if (await db.RepairInProgress(m_result.TaskControl.ProgressToken).ConfigureAwait(false) && filtercommand == null)
throw new UserInformationException(string.Format("The purge command does not work on an incomplete database, try the {0} operation.", "purge-broken-files"), "PurgeNotAllowedOnIncompleteDatabase");
var versions = await db
.GetFilesetIDs(m_options.Time, m_options.Version)
.GetFilesetIDs(m_options.Time, m_options.Version, false, m_result.TaskControl.ProgressToken)
.OrderByDescending(x => x)
.ToArrayAsync()
.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (versions.Length <= 0)
throw new UserInformationException("No filesets matched the supplied time or versions", "NoFilesetFoundForTimeOrVersion");
var orphans = await db.CountOrphanFiles().ConfigureAwait(false);
var orphans = await db.CountOrphanFiles(m_result.TaskControl.ProgressToken).ConfigureAwait(false);
if (orphans != 0)
throw new UserInformationException(string.Format("Unable to start the purge process as there are {0} orphan file(s)", orphans), "CannotPurgeWithOrphans");
await Utility.UpdateOptionsFromDb(db, m_options)
await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await Utility.VerifyOptionsAndUpdateDatabase(db, m_options)
await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (filtercommand == null)
{
await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, false)
await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, false, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (m_options.NoBackendverification)
await FilelistProcessor.VerifyLocalList(backendManager, db, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
else
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, null, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false);
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, null, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
}
var filesets = await db
.FilesetTimes()
.FilesetTimes(m_result.TaskControl.ProgressToken)
.OrderByDescending(x => x.Value)
.ToArrayAsync()
.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
var versionprogress = ((doCompactStep ? 0.75f : 1.0f) / versions.Length) * pgspan;
@@ -118,7 +118,7 @@ namespace Duplicati.Library.Main.Operation
// If we crash now, it is possible that the remote storage contains partial files
if (!m_options.Dryrun)
await db.TerminatedWithActiveUploads(true).ConfigureAwait(false);
await db.TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, true).ConfigureAwait(false);
// Reverse makes sure we re-write the old versions first
foreach (var versionid in versions.Reverse())
@@ -132,18 +132,18 @@ namespace Duplicati.Library.Main.Operation
var ts = await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, tsOriginal)
.ConfigureAwait(false);
var prevfilename = await db.GetRemoteVolumeNameForFileset(filesets[ix].Key)
var prevfilename = await db.GetRemoteVolumeNameForFileset(filesets[ix].Key, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (ix != 0 && filesets[ix - 1].Value <= ts)
throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is larger than the next timestamp {2}", prevfilename, ts, filesets[ix - 1].Value));
await using (var tempset = await db.CreateTemporaryFileset(versionid).ConfigureAwait(false))
await using (var tempset = await db.CreateTemporaryFileset(versionid, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
{
if (filtercommand == null)
await tempset.ApplyFilter(filter).ConfigureAwait(false);
await tempset.ApplyFilter(filter, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
else
await tempset.ApplyFilter(filtercommand)
await tempset.ApplyFilter(filtercommand, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (tempset.RemovedFileCount + tempset.UpdatedFileCount == 0)
@@ -161,10 +161,10 @@ namespace Duplicati.Library.Main.Operation
using (var vol = new FilesetVolumeWriter(m_options, ts))
{
var isOriginalFilesetFullBackup = await db
.IsFilesetFullBackup(tsOriginal)
.IsFilesetFullBackup(tsOriginal, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
var newids = await tempset
.ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup)
.ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
vol.VolumeID = newids.Item1;
vol.CreateFilesetFile(isOriginalFilesetFullBackup);
@@ -172,7 +172,7 @@ namespace Duplicati.Library.Main.Operation
Logging.Log.WriteInformationMessage(LOGTAG, "ReplacingFileset", "Replacing fileset {0} with {1} which has with {2} fewer file(s) ({3} reduction)", prevfilename, vol.RemoteFilename, tempset.RemovedFileCount, Library.Utility.Utility.FormatSizeString(tempset.RemovedFileSize));
await db
.WriteFileset(vol, newids.Item2)
.WriteFileset(vol, newids.Item2, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
m_result.RemovedFileSize += tempset.RemovedFileSize;
@@ -185,7 +185,7 @@ namespace Duplicati.Library.Main.Operation
if (m_options.Dryrun || m_options.FullResult)
{
await foreach (var fe in tempset.ListAllDeletedFiles().ConfigureAwait(false))
await foreach (var fe in tempset.ListAllDeletedFiles(m_result.TaskControl.ProgressToken).ConfigureAwait(false))
{
var msg = string.Format(" Purging file {0} ({1})", fe.Key, Library.Utility.Utility.FormatSizeString(fe.Value));
@@ -212,19 +212,19 @@ namespace Duplicati.Library.Main.Operation
else
{
await db
.UpdateRemoteVolume(vol.RemoteFilename, RemoteVolumeState.Uploading, -1, null)
.UpdateRemoteVolume(vol.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
var lst = await db
.DropFilesetsFromTable(new[] { tsOriginal })
.ToArrayAsync()
.DropFilesetsFromTable(new[] { tsOriginal }, m_result.TaskControl.ProgressToken)
.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
foreach (var f in lst)
await db
.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null)
.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await db.Transaction
.CommitAsync()
.CommitAsync(m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await backendManager.PutAsync(vol, null, null, true, async () =>
@@ -235,7 +235,7 @@ namespace Duplicati.Library.Main.Operation
// passing down a transaction, but maybe is
// not needed anymore.
await db.Transaction
.CommitAsync()
.CommitAsync(m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
await backendManager.DeleteAsync(prevfilename, -1, true, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
@@ -251,7 +251,7 @@ namespace Duplicati.Library.Main.Operation
if (!m_options.Dryrun)
await db
.TerminatedWithActiveUploads(false)
.TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, false)
.ConfigureAwait(false);
if (doCompactStep)
@@ -265,12 +265,12 @@ namespace Duplicati.Library.Main.Operation
m_result.OperationProgressUpdater.UpdateProgress(pgoffset + (0.75f * pgspan));
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Compact);
m_result.CompactResults = new CompactResults(m_result);
await using var cdb = await Database.LocalDeleteDatabase.CreateAsync(db).ConfigureAwait(false);
await using var cdb = await Database.LocalDeleteDatabase.CreateAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
await new CompactHandler(m_options, (CompactResults)m_result.CompactResults)
.DoCompactAsync(cdb, true, backendManager)
.ConfigureAwait(false);
await cdb.Transaction.CommitAsync("PostCompact");
await cdb.Transaction.CommitAsync("PostCompact", true, m_result.TaskControl.ProgressToken);
}
m_result.OperationProgressUpdater.UpdateProgress(pgoffset + pgspan);
Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs
+87 -74
View File
@@ -23,6 +23,7 @@ using System.Collections.Generic;
//using System.Data;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Duplicati.Library.Interface;
using Duplicati.Library.Main.Database;
@@ -65,14 +66,14 @@ namespace Duplicati.Library.Main.Operation
throw new UserInformationException(string.Format("Cannot recreate database because file already exists: {0}", path), "RecreateTargetDatabaseExists");
await using var db =
await LocalDatabase.CreateLocalDatabaseAsync(path, "Recreate", true, m_options.SqlitePageCache)
await LocalDatabase.CreateLocalDatabaseAsync(path, "Recreate", true, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await DoRunAsync(backendManager, db, false, filter, filelistfilter, blockprocessor).ConfigureAwait(false);
// Ensure database is consistent after the recreate
await db
.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true)
.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
@@ -89,9 +90,9 @@ namespace Duplicati.Library.Main.Operation
throw new UserInformationException(string.Format("Can only update with paths, try setting --{0}", "repair-only-paths"), "RepairUpdateRequiresPathsOnly");
await using var db =
await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Recreate", true, m_options.SqlitePageCache)
await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Recreate", true, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if ((await db.FindMatchingFilesets(m_options.Time, m_options.Version).ConfigureAwait(false)).Any())
if ((await db.FindMatchingFilesets(m_options.Time, m_options.Version, m_result.TaskControl.ProgressToken).ConfigureAwait(false)).Any())
{
if (m_options.IgnoreUpdateIfVersionExists)
{
@@ -103,17 +104,17 @@ namespace Duplicati.Library.Main.Operation
}
// Mark as incomplete
await db.PartiallyRecreated(true).ConfigureAwait(false);
await db.PartiallyRecreated(m_result.TaskControl.ProgressToken, true).ConfigureAwait(false);
var preexistingOptionsInDatabase =
await Utility.ContainsOptionsForVerification(db)
await Utility.ContainsOptionsForVerification(db, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await Utility.UpdateOptionsFromDb(db, m_options)
await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
// Make sure the options have not changed between calls, unless there are no previous options
if (preexistingOptionsInDatabase)
await Utility.VerifyOptionsAndUpdateDatabase(db, m_options)
await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await DoRunAsync(backendManager, db, true, filter, filelistfilter, blockprocessor).ConfigureAwait(false);
@@ -135,9 +136,9 @@ namespace Duplicati.Library.Main.Operation
//We build a local database in steps.
await using var restoredb =
await LocalRecreateDatabase.CreateAsync(dbparent, m_options)
await LocalRecreateDatabase.CreateAsync(dbparent, m_options, null, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await restoredb.RepairInProgress(true).ConfigureAwait(false);
await restoredb.RepairInProgress(m_result.TaskControl.ProgressToken, true).ConfigureAwait(false);
var expRecreateDb = false; // experimental recreate db code flag
var volumeIds = new Dictionary<string, long>();
@@ -205,9 +206,9 @@ namespace Duplicati.Library.Main.Operation
// If we are updating, all files should be accounted for
foreach (var fl in remotefiles)
volumeIds[fl.File.Name] = updating
? await restoredb.GetRemoteVolumeID(fl.File.Name)
? await restoredb.GetRemoteVolumeID(fl.File.Name, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false)
: await restoredb.RegisterRemoteVolume(fl.File.Name, fl.FileType, fl.File.Size, RemoteVolumeState.Uploaded)
: await restoredb.RegisterRemoteVolume(fl.File.Name, fl.FileType, fl.File.Size, RemoteVolumeState.Uploaded, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
var hasUpdatedOptions = false;
@@ -224,7 +225,7 @@ namespace Duplicati.Library.Main.Operation
foreach (var n in filelists)
if (volumeIds[n.File.Name] == -1)
volumeIds[n.File.Name] = await restoredb
.RegisterRemoteVolume(n.File.Name, n.FileType, RemoteVolumeState.Uploaded, n.File.Size, new TimeSpan(0))
.RegisterRemoteVolume(n.File.Name, n.FileType, RemoteVolumeState.Uploaded, n.File.Size, new TimeSpan(0), m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
@@ -240,7 +241,7 @@ namespace Duplicati.Library.Main.Operation
m_result.EndTime = DateTime.UtcNow;
// Implicit rollback
await restoredb.Transaction
.RollBackAsync()
.RollBackAsync(m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
return;
}
@@ -262,7 +263,7 @@ namespace Duplicati.Library.Main.Operation
if (!string.IsNullOrWhiteSpace(hash) && size > 0)
await restoredb
.UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, size, hash)
.UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, size, hash, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
var parsed = VolumeBase.ParseFilename(entry.Name);
@@ -280,10 +281,10 @@ namespace Duplicati.Library.Main.Operation
// Create timestamped operations based on the file timestamp
var filesetid = await restoredb
.CreateFileset(volumeIds[entry.Name], parsed.Time)
.CreateFileset(volumeIds[entry.Name], parsed.Time, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await RecreateFilesetFromRemoteList(restoredb, compressor, filesetid, m_options, filter)
await RecreateFilesetFromRemoteList(restoredb, compressor, filesetid, m_options, filter, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
}
@@ -295,7 +296,7 @@ namespace Duplicati.Library.Main.Operation
m_result.EndTime = DateTime.UtcNow;
// Implicit rollback
await restoredb.Transaction
.RollBackAsync()
.RollBackAsync(m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
throw;
}
@@ -305,7 +306,7 @@ namespace Duplicati.Library.Main.Operation
m_result.EndTime = DateTime.UtcNow;
// Implicit rollback
await restoredb.Transaction
.RollBackAsync()
.RollBackAsync(m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
throw;
}
@@ -314,7 +315,7 @@ namespace Duplicati.Library.Main.Operation
{
// Implicit rollback
await restoredb.Transaction
.RollBackAsync()
.RollBackAsync(m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
throw;
}
@@ -323,12 +324,12 @@ namespace Duplicati.Library.Main.Operation
//Make sure we write the config if it has been read from a manifest
if (hasUpdatedOptions)
await Utility.VerifyOptionsAndUpdateDatabase(restoredb, m_options)
await Utility.VerifyOptionsAndUpdateDatabase(restoredb, m_options, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
using (new Logging.Timer(LOGTAG, "CommitUpdateFilesetFromRemote", "CommitUpdateFilesetFromRemote"))
await restoredb.Transaction
.CommitAsync()
.CommitAsync(m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
// do we stop after just handling the dlist files ?
@@ -361,7 +362,7 @@ namespace Duplicati.Library.Main.Operation
m_result.EndTime = DateTime.UtcNow;
// Implicit rollback
await restoredb.Transaction
.RollBackAsync()
.RollBackAsync(m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
return;
}
@@ -374,7 +375,7 @@ namespace Duplicati.Library.Main.Operation
{
if (!string.IsNullOrWhiteSpace(hash) && size > 0)
await restoredb
.UpdateRemoteVolume(name, RemoteVolumeState.Verified, size, hash)
.UpdateRemoteVolume(name, RemoteVolumeState.Verified, size, hash, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
using (var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options, hashsize))
@@ -383,13 +384,13 @@ namespace Duplicati.Library.Main.Operation
{
var filename = a.Filename;
var volumeID = await restoredb
.GetRemoteVolumeID(filename)
.GetRemoteVolumeID(filename, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
// No such file
if (volumeID < 0)
(volumeID, filename) =
await ProbeForMatchingFilename(filename, restoredb)
await ProbeForMatchingFilename(filename, restoredb, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
var missing = false;
@@ -402,7 +403,7 @@ namespace Duplicati.Library.Main.Operation
Logging.Log.WriteWarningMessage(LOGTAG, "MissingFileDetected", null, "Remote file referenced as {0} by {1}, but not found in list, registering a missing remote file", filename, name);
missing = true;
volumeID = await restoredb
.RegisterRemoteVolume(filename, p.FileType, RemoteVolumeState.Temporary)
.RegisterRemoteVolume(filename, p.FileType, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
@@ -410,16 +411,17 @@ namespace Duplicati.Library.Main.Operation
//Add all block/volume mappings
foreach (var b in a.Blocks)
{
anyChange |= (await restoredb.UpdateBlock(b.Key, b.Value, volumeID).ConfigureAwait(false)).Item1;
anyChange |= (await restoredb.UpdateBlock(b.Key, b.Value, volumeID, m_result.TaskControl.ProgressToken).ConfigureAwait(false)).Item1;
}
await restoredb
.UpdateRemoteVolume(filename, missing ? RemoteVolumeState.Temporary : RemoteVolumeState.Verified, a.Length, a.Hash)
.UpdateRemoteVolume(filename, missing ? RemoteVolumeState.Temporary : RemoteVolumeState.Verified, a.Length, a.Hash, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await restoredb
.AddIndexBlockLink(
await restoredb.GetRemoteVolumeID(name).ConfigureAwait(false),
volumeID
await restoredb.GetRemoteVolumeID(name, m_result.TaskControl.ProgressToken).ConfigureAwait(false),
volumeID,
m_result.TaskControl.ProgressToken
)
.ConfigureAwait(false);
}
@@ -436,7 +438,7 @@ namespace Duplicati.Library.Main.Operation
// done before we add it to the database, since we do not have nested transactions
var list = b.Blocklist.ToList();
await restoredb
.AddTempBlockListHash(b.Hash, list)
.AddTempBlockListHash(b.Hash, list, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
catch (System.IO.InvalidDataException e)
@@ -461,7 +463,7 @@ namespace Duplicati.Library.Main.Operation
m_result.EndTime = DateTime.UtcNow;
// Implicit rollback
await restoredb.Transaction
.RollBackAsync()
.RollBackAsync(m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
throw;
}
@@ -470,7 +472,7 @@ namespace Duplicati.Library.Main.Operation
{
// Implicit rollback
await restoredb.Transaction
.RollBackAsync()
.RollBackAsync(m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
throw;
}
@@ -479,25 +481,25 @@ namespace Duplicati.Library.Main.Operation
using (new Logging.Timer(LOGTAG, "CommitRecreateDb", "CommitRecreatedDb"))
await restoredb.Transaction
.CommitAsync()
.CommitAsync(m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
// TODO: In some cases, we can avoid downloading all index files,
// if we are lucky and pick the right ones
}
await restoredb.CleanupMissingVolumes().ConfigureAwait(false);
await restoredb.CleanupMissingVolumes(m_result.TaskControl.ProgressToken).ConfigureAwait(false);
// Update the real tables from the temp tables
if (expRecreateDb)
// add missing blocks and blocksetentry data (at this point
// we have not yet anything in the blocksetentry table)
await restoredb
.AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize)
.AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, false, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
else
await restoredb
.FindMissingBlocklistHashes(hashsize, m_options.Blocksize)
.FindMissingBlocklistHashes(hashsize, m_options.Blocksize, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
// We have now grabbed as much information as possible,
@@ -507,7 +509,7 @@ namespace Duplicati.Library.Main.Operation
{
// Grab the list matching the pass type
var lst = await restoredb
.GetMissingBlockListVolumes(i, m_options.Blocksize, hashsize, m_options.RepairForceBlockUse)
.GetMissingBlockListVolumes(i, m_options.Blocksize, hashsize, m_options.RepairForceBlockUse, m_result.TaskControl.ProgressToken)
.ToListAsync()
.ConfigureAwait(false);
@@ -545,7 +547,7 @@ namespace Duplicati.Library.Main.Operation
m_result.EndTime = DateTime.UtcNow;
// Implicit rollback
await restoredb.Transaction
.RollBackAsync()
.RollBackAsync(m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
return;
}
@@ -555,26 +557,26 @@ namespace Duplicati.Library.Main.Operation
Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingBlocklistVolumes", "Pass {0} of 3, processing blocklist volume {1} of {2}", (i + 1), progress, lst.Count);
var volumeid = await restoredb
.GetRemoteVolumeID(name)
.GetRemoteVolumeID(name, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await restoredb
.UpdateRemoteVolume(name, RemoteVolumeState.Uploaded, size, hash)
.UpdateRemoteVolume(name, RemoteVolumeState.Uploaded, size, hash, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
bool anyChange = false;
// Update the block table so we know about the block/volume map
foreach (var h in rd.Blocks)
{
anyChange |= (await restoredb.UpdateBlock(h.Key, h.Value, volumeid).ConfigureAwait(false)).Item1;
anyChange |= (await restoredb.UpdateBlock(h.Key, h.Value, volumeid, m_result.TaskControl.ProgressToken).ConfigureAwait(false)).Item1;
}
// now that we have the blocks/volume relationships, we can go from the (already known from dlist step) blocklisthashes
// to the needed list blocks in the volume, so grab them from the database
// read the blocks list hashes from the volume data (the handled file) and insert them into the temp blocklisthash table
await foreach (var blocklisthash in restoredb.GetBlockLists(volumeid).ConfigureAwait(false))
await foreach (var blocklisthash in restoredb.GetBlockLists(volumeid, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
{
if (await restoredb.AddTempBlockListHash(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize)).ConfigureAwait(false))
if (await restoredb.AddTempBlockListHash(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize), m_result.TaskControl.ProgressToken).ConfigureAwait(false))
{
anyChange = true;
}
@@ -593,17 +595,17 @@ namespace Duplicati.Library.Main.Operation
}
if (expRecreateDb)
await restoredb
.AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, false)
.AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, false, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
else
await restoredb
.FindMissingBlocklistHashes(hashsize, m_options.Blocksize)
.FindMissingBlocklistHashes(hashsize, m_options.Blocksize, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
using (new Logging.Timer(LOGTAG, "CommitRestoredBlocklist", "CommitRestoredBlocklist"))
await restoredb.Transaction
.CommitAsync()
.CommitAsync(m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
//At this point we can patch files with data from the block volume
@@ -618,7 +620,7 @@ namespace Duplicati.Library.Main.Operation
{
// Implicit rollback
await restoredb.Transaction
.RollBackAsync()
.RollBackAsync(m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
throw;
}
@@ -634,10 +636,10 @@ namespace Duplicati.Library.Main.Operation
// All blocks are collected and added into the Block table
// Find out which blocks are deleted and move them into DeletedBlock,
// so that compact can calculate the unused space
await restoredb.CleanupDeletedBlocks().ConfigureAwait(false);
await restoredb.CleanupDeletedBlocks(m_result.TaskControl.ProgressToken).ConfigureAwait(false);
}
await restoredb.CleanupMissingVolumes().ConfigureAwait(false);
await restoredb.CleanupMissingVolumes(m_result.TaskControl.ProgressToken).ConfigureAwait(false);
if (m_options.RepairOnlyPaths)
{
@@ -652,11 +654,11 @@ namespace Duplicati.Library.Main.Operation
// except to continue a backup
m_result.EndTime = DateTime.UtcNow;
await using (var lbfdb = await LocalListBrokenFilesDatabase.CreateAsync(restoredb).ConfigureAwait(false))
await using (var lbfdb = await LocalListBrokenFilesDatabase.CreateAsync(restoredb, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
{
var broken = await lbfdb
.GetBrokenFilesets(new DateTime(0), null)
.CountAsync()
.GetBrokenFilesets(new DateTime(0), null, m_result.TaskControl.ProgressToken)
.CountAsync(cancellationToken: m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (broken != 0)
@@ -664,18 +666,28 @@ namespace Duplicati.Library.Main.Operation
}
await restoredb
.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true)
.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
Logging.Log.WriteInformationMessage(LOGTAG, "RecreateCompleted", "Recreate completed, and consistency checks completed, marking database as complete");
await restoredb.RepairInProgress(false).ConfigureAwait(false);
await restoredb.RepairInProgress(m_result.TaskControl.ProgressToken, false).ConfigureAwait(false);
}
m_result.EndTime = DateTime.UtcNow;
}
public static async Task RecreateFilesetFromRemoteList(LocalRecreateDatabase restoredb, ICompression compressor, long filesetid, Options options, IFilter filter)
/// <summary>
/// Recreate a fileset from the remote filelist.
/// </summary>
/// <param name="restoredb">The database to restore to.</param>
/// <param name="compressor">The compression module to use.</param>
/// <param name="filesetid">The ID of the fileset to recreate.</param>
/// <param name="options">The options to use for the operation.</param>
/// <param name="filter">The filter to apply to the files.</param>
/// <param name="cancellationToken">The cancellation token to monitor for cancellation requests.</param>
/// <returns>A task that completes when the fileset has been recreated.</returns>
public static async Task RecreateFilesetFromRemoteList(LocalRecreateDatabase restoredb, ICompression compressor, long filesetid, Options options, IFilter filter, CancellationToken cancellationToken)
{
var blocksize = options.Blocksize;
var hashes_pr_block = blocksize / options.BlockhashSize;
@@ -685,11 +697,11 @@ namespace Duplicati.Library.Main.Operation
// update fileset using filesetData
await restoredb
.UpdateFullBackupStateInFileset(filesetid, filesetData.IsFullBackup)
.UpdateFullBackupStateInFileset(filesetid, filesetData.IsFullBackup, cancellationToken)
.ConfigureAwait(false);
// clear any existing fileset entries
await restoredb.ClearFilesetEntries(filesetid).ConfigureAwait(false);
await restoredb.ClearFilesetEntries(filesetid, cancellationToken).ConfigureAwait(false);
using (var filelistreader = new FilesetVolumeReader(compressor, options))
foreach (var fe in filelistreader.Files.Where(x => Library.Utility.FilterExpression.Matches(filter, x.Path)))
@@ -703,17 +715,17 @@ namespace Duplicati.Library.Main.Operation
var metadataid = long.MinValue;
var split = LocalDatabase.SplitIntoPrefixAndName(fe.Path);
var prefixid = await restoredb
.GetOrCreatePathPrefix(split.Key)
.GetOrCreatePathPrefix(split.Key, cancellationToken)
.ConfigureAwait(false);
switch (fe.Type)
{
case FilelistEntryType.Folder:
metadataid = await restoredb
.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes)
.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, cancellationToken)
.ConfigureAwait(false);
await restoredb
.AddDirectoryEntry(filesetid, prefixid, split.Value, fe.Time, metadataid)
.AddDirectoryEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, cancellationToken)
.ConfigureAwait(false);
break;
@@ -723,24 +735,24 @@ namespace Duplicati.Library.Main.Operation
if (expectedblocks <= 1) expectedblocklisthashes = 0;
var blocksetid = await restoredb
.AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes)
.AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes, cancellationToken)
.ConfigureAwait(false);
metadataid = await restoredb
.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes)
.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, cancellationToken)
.ConfigureAwait(false);
await restoredb
.AddFileEntry(filesetid, prefixid, split.Value, fe.Time, blocksetid, metadataid)
.AddFileEntry(filesetid, prefixid, split.Value, fe.Time, blocksetid, metadataid, cancellationToken)
.ConfigureAwait(false);
if (fe.Size <= blocksize)
{
if (!string.IsNullOrWhiteSpace(fe.Blockhash))
await restoredb
.AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize)
.AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize, cancellationToken)
.ConfigureAwait(false);
else if (options.BlockHashAlgorithm == options.FileHashAlgorithm)
await restoredb
.AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size)
.AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size, cancellationToken)
.ConfigureAwait(false);
else if (fe.Size > 0)
Logging.Log.WriteWarningMessage(LOGTAG, "MissingBlockHash", null, "No block hash found for file: {0}", fe.Path);
@@ -749,10 +761,10 @@ namespace Duplicati.Library.Main.Operation
break;
case FilelistEntryType.Symlink:
metadataid = await restoredb
.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes)
.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, cancellationToken)
.ConfigureAwait(false);
await restoredb
.AddSymlinkEntry(filesetid, prefixid, split.Value, fe.Time, metadataid)
.AddSymlinkEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, cancellationToken)
.ConfigureAwait(false);
break;
default:
@@ -764,11 +776,11 @@ namespace Duplicati.Library.Main.Operation
{
if (!string.IsNullOrWhiteSpace(fe.Metablockhash))
await restoredb
.AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize)
.AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize, cancellationToken)
.ConfigureAwait(false);
else if (options.BlockHashAlgorithm == options.FileHashAlgorithm)
await restoredb
.AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize)
.AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize, cancellationToken)
.ConfigureAwait(false);
else
Logging.Log.WriteWarningMessage(LOGTAG, "MissingMetadataBlockHash", null, "No block hash found for file metadata: {0}", fe.Path);
@@ -780,7 +792,7 @@ namespace Duplicati.Library.Main.Operation
}
}
await restoredb.Transaction.CommitAsync().ConfigureAwait(false);
await restoredb.Transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
}
/// <summary>
@@ -788,8 +800,9 @@ namespace Duplicati.Library.Main.Operation
/// </summary>
/// <param name="filename">The filename read and written.</param>
/// <param name="restoredb">The database to query.</param>
/// <param name="cancellationToken">A cancellation token to cancel the operation.</param>
/// <returns>The volume id of the item.</returns>
public static async Task<(long, string)> ProbeForMatchingFilename(string filename, LocalRecreateDatabase restoredb)
public static async Task<(long, string)> ProbeForMatchingFilename(string filename, LocalRecreateDatabase restoredb, CancellationToken cancellationToken)
{
var p = VolumeBase.ParseFilename(filename);
if (p != null)
@@ -799,7 +812,7 @@ namespace Duplicati.Library.Main.Operation
{
var testfilename = VolumeBase.GenerateFilename(p.FileType, p.Prefix, p.Guid, p.Time, compmodule, encmodule);
var tvid = await restoredb
.GetRemoteVolumeID(testfilename)
.GetRemoteVolumeID(testfilename, cancellationToken)
.ConfigureAwait(false);
if (tvid >= 0)
Duplicati/Library/Main/Operation/RepairHandler.cs
+132 -126
View File
@@ -66,11 +66,11 @@ namespace Duplicati.Library.Main.Operation
try
{
await using var db =
await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache)
await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
knownRemotes = await db
.GetRemoteVolumes()
.CountAsync()
.GetRemoteVolumes(m_result.TaskControl.ProgressToken)
.CountAsync(cancellationToken: m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
catch (Exception ex)
@@ -134,50 +134,50 @@ namespace Duplicati.Library.Main.Operation
m_result.OperationProgressUpdater.UpdateProgress(0);
await using var db =
await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache)
await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache, null, cancellationToken)
.ConfigureAwait(false);
try
{
await Utility.UpdateOptionsFromDb(db, m_options)
await Utility.UpdateOptionsFromDb(db, m_options, cancellationToken)
.ConfigureAwait(false);
await Utility.VerifyOptionsAndUpdateDatabase(db, m_options)
await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, cancellationToken)
.ConfigureAwait(false);
if (await db.PartiallyRecreated().ConfigureAwait(false))
if (await db.PartiallyRecreated(cancellationToken).ConfigureAwait(false))
throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated");
if (await db.RepairInProgress().ConfigureAwait(false))
if (await db.RepairInProgress(cancellationToken).ConfigureAwait(false))
throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsInRepairState");
// Ensure the database is consistent before we start fixing the remote
await db
.VerifyConsistencyForRepair(m_options.Blocksize, m_options.BlockhashSize, true)
.VerifyConsistencyForRepair(m_options.Blocksize, m_options.BlockhashSize, true, cancellationToken)
.ConfigureAwait(false);
// If the last backup failed, guard the incomplete fileset, so we can create a synthetic filelist
var lastTempFilelist = await db
.GetLastIncompleteFilesetVolume()
.GetLastIncompleteFilesetVolume(cancellationToken)
.ConfigureAwait(false);
var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false);
var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced, cancellationToken).ConfigureAwait(false);
await db.Transaction
.CommitAsync("CommitRemoteListAnalysisTransaction")
.CommitAsync("CommitRemoteListAnalysisTransaction", token: cancellationToken)
.ConfigureAwait(false);
var buffer = new byte[m_options.Blocksize];
var hashsize = m_options.BlockhashSize;
var missingRemoteFilesets = await db
.MissingRemoteFilesets()
.MissingRemoteFilesets(cancellationToken)
.ToListAsync(cancellationToken: cancellationToken)
.ConfigureAwait(false);
var missingLocalFilesets = await db
.MissingLocalFilesets()
.MissingLocalFilesets(cancellationToken)
.ToListAsync(cancellationToken: cancellationToken)
.ConfigureAwait(false);
var emptyIndexFiles = await db
.EmptyIndexFiles()
.EmptyIndexFiles(cancellationToken)
.ToListAsync(cancellationToken: cancellationToken)
.ConfigureAwait(false);
@@ -192,12 +192,12 @@ namespace Duplicati.Library.Main.Operation
// Find the most recent timestamp from either a fileset or a remote volume
var mostRecentLocal = await db
.GetRemoteVolumes()
.GetRemoteVolumes(cancellationToken)
.Where(x => x.Type == RemoteVolumeType.Files)
.Select(x => VolumeBase.ParseFilename(x.Name).Time.ToLocalTime())
.Concat(db.FilesetTimes().Select(x => x.Value.ToLocalTime()))
.Concat(db.FilesetTimes(cancellationToken).Select(x => x.Value.ToLocalTime()))
.Append(DateTime.MinValue)
.MaxAsync()
.MaxAsync(cancellationToken: cancellationToken)
.ConfigureAwait(false);
var mostRecentRemote = tp.ParsedVolumes.Select(x => x.Time.ToLocalTime()).Append(DateTime.MinValue).Max();
@@ -231,7 +231,7 @@ namespace Duplicati.Library.Main.Operation
if (tp.VerificationRequiredVolumes.Any())
{
await using var testdb =
await LocalTestDatabase.CreateAsync(db)
await LocalTestDatabase.CreateAsync(db, null, cancellationToken)
.ConfigureAwait(false);
foreach (var n in tp.VerificationRequiredVolumes)
@@ -242,11 +242,11 @@ namespace Duplicati.Library.Main.Operation
await backendManager.WaitForEmptyAsync(testdb, cancellationToken).ConfigureAwait(false);
if (!m_options.Dryrun)
await testdb.Transaction
.CommitAsync("CommitEarlyExit", false)
.CommitAsync("CommitEarlyExit", false, cancellationToken)
.ConfigureAwait(false);
else
await testdb.Transaction
.RollBackAsync()
.RollBackAsync(cancellationToken)
.ConfigureAwait(false);
return;
}
@@ -258,7 +258,7 @@ namespace Duplicati.Library.Main.Operation
var (tf, hash, size) = await backendManager.GetWithInfoAsync(n.Name, n.Hash, n.Size, cancellationToken).ConfigureAwait(false);
using (tf)
res =
await TestHandler.TestVolumeInternals(testdb, n, tf, m_options, 1)
await TestHandler.TestVolumeInternals(testdb, n, tf, m_options, 1, cancellationToken)
.ConfigureAwait(false);
if (res.Value.Any())
@@ -268,7 +268,7 @@ namespace Duplicati.Library.Main.Operation
{
Logging.Log.WriteInformationMessage(LOGTAG, "CapturedRemoteFileHash", "Sucessfully captured hash for {0}, updating database", n.Name);
await db
.UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash)
.UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash, cancellationToken)
.ConfigureAwait(false);
}
@@ -281,7 +281,7 @@ namespace Duplicati.Library.Main.Operation
}
await db.Transaction
.CommitAsync("CommitVerificationTransaction")
.CommitAsync("CommitVerificationTransaction", true, cancellationToken)
.ConfigureAwait(false);
}
@@ -294,11 +294,11 @@ namespace Duplicati.Library.Main.Operation
await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false);
if (!m_options.Dryrun)
await db.Transaction
.CommitAsync("CommitEarlyExit", false)
.CommitAsync("CommitEarlyExit", false, cancellationToken)
.ConfigureAwait(false);
else
await db.Transaction
.RollBackAsync()
.RollBackAsync(cancellationToken)
.ConfigureAwait(false);
return;
}
@@ -319,7 +319,7 @@ namespace Duplicati.Library.Main.Operation
foreach (var rv in ifr.Volumes)
{
var entry = await db
.GetRemoteVolume(rv.Filename)
.GetRemoteVolume(rv.Filename, cancellationToken)
.ConfigureAwait(false);
if (entry.ID < 0)
throw new Exception(string.Format("Unknown remote file {0} detected", rv.Filename));
@@ -331,44 +331,44 @@ namespace Duplicati.Library.Main.Operation
throw new Exception(string.Format("Volume {0} hash/size mismatch ({1} - {2}) vs ({3} - {4})", rv.Filename, entry.Hash, entry.Size, rv.Hash, rv.Length));
await db
.CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks)
.CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks, cancellationToken)
.ConfigureAwait(false);
}
var blocksize = m_options.Blocksize;
foreach (var ixb in ifr.BlockLists)
await db
.CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize)
.CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize, cancellationToken)
.ConfigureAwait(false);
// Register the new index file and link it to the block files
var selfid = await db
.RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, RemoteVolumeState.Uploading, size, new TimeSpan(0))
.RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, RemoteVolumeState.Uploading, size, new TimeSpan(0), cancellationToken)
.ConfigureAwait(false);
foreach (var rv in ifr.Volumes)
{
// Guard against unknown block files
long id = await db
.GetRemoteVolumeID(rv.Filename)
.GetRemoteVolumeID(rv.Filename, cancellationToken)
.ConfigureAwait(false);
if (id == -1)
Logging.Log.WriteWarningMessage(LOGTAG, "UnknownBlockFile", null, "Index file {0} references unknown block file: {1}", n.File.Name, rv.Filename);
else
await db
.AddIndexBlockLink(selfid, id)
.AddIndexBlockLink(selfid, id, cancellationToken)
.ConfigureAwait(false);
}
if (!m_options.Dryrun)
await db.Transaction
.CommitAsync("CommitIndexFileTransaction")
.CommitAsync("CommitIndexFileTransaction", true, cancellationToken)
.ConfigureAwait(false);
}
// All checks fine, we accept the new index file
Logging.Log.WriteInformationMessage(LOGTAG, "AcceptNewIndexFile", "Accepting new index file {0}", n.File.Name);
await db
.UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash)
.UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash, cancellationToken)
.ConfigureAwait(false);
continue;
@@ -382,7 +382,7 @@ namespace Duplicati.Library.Main.Operation
if (!m_options.Dryrun)
{
await db
.RegisterRemoteVolume(n.File.Name, n.FileType, n.File.Size, RemoteVolumeState.Deleting)
.RegisterRemoteVolume(n.File.Name, n.FileType, n.File.Size, RemoteVolumeState.Deleting, cancellationToken)
.ConfigureAwait(false);
await backendManager.DeleteAsync(n.File.Name, n.File.Size, false, cancellationToken).ConfigureAwait(false);
}
@@ -408,11 +408,11 @@ namespace Duplicati.Library.Main.Operation
await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false);
if (!m_options.Dryrun)
await db.Transaction
.CommitAsync("CommitEarlyExit", false)
.CommitAsync("CommitEarlyExit", false, cancellationToken)
.ConfigureAwait(false);
else
await db.Transaction
.RollBackAsync()
.RollBackAsync(cancellationToken)
.ConfigureAwait(false);
return;
}
@@ -432,17 +432,17 @@ namespace Duplicati.Library.Main.Operation
fsw.CreateFilesetFile(isfull);
await db
.WriteFileset(fsw, filesetId)
.WriteFileset(fsw, filesetId, cancellationToken)
.ConfigureAwait(false);
fsw.Close();
fsw.VolumeID = await db
.RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, -1, RemoteVolumeState.Temporary)
.RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, -1, RemoteVolumeState.Temporary, cancellationToken)
.ConfigureAwait(false);
await db
.LinkFilesetToVolume(filesetId, fsw.VolumeID)
.LinkFilesetToVolume(filesetId, fsw.VolumeID, cancellationToken)
.ConfigureAwait(false);
if (m_options.Dryrun)
@@ -460,39 +460,39 @@ namespace Duplicati.Library.Main.Operation
await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false);
if (!m_options.Dryrun)
await db.Transaction
.CommitAsync("CommitFilesetTransaction")
.CommitAsync("CommitFilesetTransaction", token: cancellationToken)
.ConfigureAwait(false);
}
foreach (var volumename in missingLocalFilesets)
{
var remoteVolume = await db
.GetRemoteVolume(volumename)
.GetRemoteVolume(volumename, cancellationToken)
.ConfigureAwait(false);
using var tmpfile = await backendManager.GetAsync(remoteVolume.Name, remoteVolume.Hash, remoteVolume.Size, cancellationToken).ConfigureAwait(false);
var parsed = VolumeBase.ParseFilename(remoteVolume.Name);
using var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read);
using var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions);
await using var recreatedb = await LocalRecreateDatabase.CreateAsync(db, m_options).ConfigureAwait(false);
await using var recreatedb = await LocalRecreateDatabase.CreateAsync(db, m_options, null, cancellationToken).ConfigureAwait(false);
if (compressor == null)
throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule");
var filesetid = await db
.CreateFileset(remoteVolume.ID, parsed.Time)
.CreateFileset(remoteVolume.ID, parsed.Time, cancellationToken)
.ConfigureAwait(false);
await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, compressor, filesetid, m_options, new FilterExpression())
await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, compressor, filesetid, m_options, new FilterExpression(), cancellationToken)
.ConfigureAwait(false);
if (!m_options.Dryrun)
await recreatedb.Transaction
.CommitAsync("CommitRecreateFilesetTransaction")
.CommitAsync("CommitRecreateFilesetTransaction", token: cancellationToken)
.ConfigureAwait(false);
}
if (!m_options.Dryrun && tp.MissingVolumes.Any())
await db
.TerminatedWithActiveUploads(true)
.TerminatedWithActiveUploads(cancellationToken, true)
.ConfigureAwait(false);
if (tp.MissingVolumes.Any(x => x.Type != RemoteVolumeType.Index && x.Type != RemoteVolumeType.Files && x.Type != RemoteVolumeType.Blocks))
@@ -548,7 +548,7 @@ namespace Duplicati.Library.Main.Operation
try
{
// Check if the index file has already been deleted, beause the dblock was recreated
var currentState = (await db.GetRemoteVolume(n.Name).ConfigureAwait(false)).State;
var currentState = (await db.GetRemoteVolume(n.Name, cancellationToken).ConfigureAwait(false)).State;
if (currentState == RemoteVolumeState.Deleted || currentState == RemoteVolumeState.Temporary || currentState == RemoteVolumeState.Deleting)
continue;
@@ -577,7 +577,7 @@ namespace Duplicati.Library.Main.Operation
await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false);
// Implicit rollback
await db.Transaction
.RollBackAsync()
.RollBackAsync(cancellationToken)
.ConfigureAwait(false);
return;
}
@@ -604,7 +604,7 @@ namespace Duplicati.Library.Main.Operation
// FlushPendingMessagesAsync, but maybe it
// is not needed anymore.
await db.Transaction
.CommitAsync()
.CommitAsync(cancellationToken)
.ConfigureAwait(false);
}
}
@@ -628,23 +628,23 @@ namespace Duplicati.Library.Main.Operation
if (!m_options.Dryrun)
{
await db
.TerminatedWithActiveUploads(false)
.TerminatedWithActiveUploads(cancellationToken, false)
.ConfigureAwait(false);
await db.Transaction
.CommitAsync("CommitRepairTransaction")
.CommitAsync("CommitRepairTransaction", true, cancellationToken)
.ConfigureAwait(false);
}
else
{
await db.Transaction
.RollBackAsync("RollbackRepairTransaction")
.RollBackAsync("RollbackRepairTransaction", true, cancellationToken)
.ConfigureAwait(false);
}
}
catch (Exception)
{
// Implicit rollback
await db.Transaction.RollBackAsync().ConfigureAwait(false);
await db.Transaction.RollBackAsync(cancellationToken).ConfigureAwait(false);
throw;
}
}
@@ -662,10 +662,10 @@ namespace Duplicati.Library.Main.Operation
private async Task RunRepairDlist(IBackendManager backendManager, LocalRepairDatabase db, FilesetVolumeWriter volumeWriter, RemoteVolumeEntry originalVolume, DateTime filesetTime, CancellationToken cancellationToken)
{
volumeWriter.VolumeID = await db
.RegisterRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, -1, TimeSpan.Zero)
.RegisterRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, cancellationToken)
.ConfigureAwait(false);
(var prevFilesetId, var _, var isPrevFull) = await db
.GetFilesetFromRemotename(originalVolume.Name)
.GetFilesetFromRemotename(originalVolume.Name, cancellationToken)
.ConfigureAwait(false);
if (!string.IsNullOrEmpty(m_options.ControlFiles))
@@ -674,28 +674,28 @@ namespace Duplicati.Library.Main.Operation
volumeWriter.CreateFilesetFile(isPrevFull);
var newFilesetID = await db
.CreateFileset(volumeWriter.VolumeID, filesetTime)
.CreateFileset(volumeWriter.VolumeID, filesetTime, cancellationToken)
.ConfigureAwait(false);
await db
.UpdateFullBackupStateInFileset(newFilesetID, isPrevFull)
.UpdateFullBackupStateInFileset(newFilesetID, isPrevFull, cancellationToken)
.ConfigureAwait(false);
await db
.LinkFilesetToVolume(newFilesetID, volumeWriter.VolumeID)
.LinkFilesetToVolume(newFilesetID, volumeWriter.VolumeID, cancellationToken)
.ConfigureAwait(false);
await db
.MoveFilesFromFileset(newFilesetID, prevFilesetId)
.MoveFilesFromFileset(newFilesetID, prevFilesetId, cancellationToken)
.ConfigureAwait(false);
await db
.WriteFileset(volumeWriter, newFilesetID)
.WriteFileset(volumeWriter, newFilesetID, cancellationToken)
.ConfigureAwait(false);
volumeWriter.Close();
await db
.UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null)
.UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null, cancellationToken)
.ConfigureAwait(false);
await db
.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null)
.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, cancellationToken)
.ConfigureAwait(false);
await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false);
@@ -704,7 +704,7 @@ namespace Duplicati.Library.Main.Operation
else
{
await db.Transaction
.CommitAsync("CommitPriorToFilesetUpload")
.CommitAsync("CommitPriorToFilesetUpload", true, cancellationToken)
.ConfigureAwait(false);
await backendManager.PutAsync(volumeWriter, null, null, false, null, cancellationToken).ConfigureAwait(false);
}
@@ -712,7 +712,7 @@ namespace Duplicati.Library.Main.Operation
await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false);
if (!m_options.Dryrun)
await db.Transaction
.CommitAsync("CommitFilesetTransaction")
.CommitAsync("CommitFilesetTransaction", true, cancellationToken)
.ConfigureAwait(false);
}
@@ -729,26 +729,26 @@ namespace Duplicati.Library.Main.Operation
public static async Task RunRepairDindex(IBackendManager backendManager, LocalRepairDatabase db, IndexVolumeWriter indexWriter, IRemoteVolume originalVolume, Options options, CancellationToken cancellationToken)
{
indexWriter.VolumeID = await db
.RegisterRemoteVolume(indexWriter.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Uploading, -1, TimeSpan.Zero)
.RegisterRemoteVolume(indexWriter.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Uploading, -1, TimeSpan.Zero, cancellationToken)
.ConfigureAwait(false);
var blockvolumeids = new List<long>();
using var h = HashFactory.CreateHasher(options.BlockHashAlgorithm);
await foreach (var blockvolume in db.GetBlockVolumesFromIndexName(originalVolume.Name).ConfigureAwait(false))
await foreach (var blockvolume in db.GetBlockVolumesFromIndexName(originalVolume.Name, cancellationToken).ConfigureAwait(false))
{
indexWriter.StartVolume(blockvolume.Name);
var volumeid = await db
.GetRemoteVolumeID(blockvolume.Name)
.GetRemoteVolumeID(blockvolume.Name, cancellationToken)
.ConfigureAwait(false);
await foreach (var b in db.GetBlocks(volumeid).ConfigureAwait(false))
await foreach (var b in db.GetBlocks(volumeid, cancellationToken).ConfigureAwait(false))
indexWriter.AddBlock(b.Hash, b.Size);
indexWriter.FinishVolume(blockvolume.Hash, blockvolume.Size);
blockvolumeids.Add(volumeid);
if (options.IndexfilePolicy == Options.IndexFileStrategy.Full)
await foreach (var b in db.GetBlocklists(volumeid, options.Blocksize, options.BlockhashSize).ConfigureAwait(false))
await foreach (var b in db.GetBlocklists(volumeid, options.Blocksize, options.BlockhashSize, cancellationToken).ConfigureAwait(false))
{
var bh = Convert.ToBase64String(h.ComputeHash(b.Buffer, 0, b.Size));
if (bh != b.Hash)
@@ -760,7 +760,7 @@ namespace Duplicati.Library.Main.Operation
foreach (var blockvolumeid in blockvolumeids)
await db
.AddIndexBlockLink(indexWriter.VolumeID, blockvolumeid)
.AddIndexBlockLink(indexWriter.VolumeID, blockvolumeid, cancellationToken)
.ConfigureAwait(false);
indexWriter.Close();
@@ -770,7 +770,7 @@ namespace Duplicati.Library.Main.Operation
else
{
await db
.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null)
.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, cancellationToken)
.ConfigureAwait(false);
await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false);
await backendManager.PutAsync(indexWriter, null, null, false, null, cancellationToken).ConfigureAwait(false);
@@ -779,7 +779,7 @@ namespace Duplicati.Library.Main.Operation
await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false);
if (!options.Dryrun)
await db.Transaction
.CommitAsync("CommitRepairTransaction")
.CommitAsync("CommitRepairTransaction", true, cancellationToken)
.ConfigureAwait(false);
}
@@ -835,12 +835,13 @@ namespace Duplicati.Library.Main.Operation
/// </summary>
/// <param name="options">The options for the current operation.</param>
/// <param name="db">The database to use for registering the initial volume.</param>
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
/// <returns>A task that completes with the new instance of <see cref="InProgressDblockVolumes"/>.</returns>
public static async Task<InProgressDblockVolumes> CreateAsync(Options options, LocalDatabase db)
public static async Task<InProgressDblockVolumes> CreateAsync(Options options, LocalDatabase db, CancellationToken cancellationToken)
{
var ipdv = new InProgressDblockVolumes(options, db);
ipdv.m_activeWriter.VolumeID = await db
.RegisterRemoteVolume(ipdv.m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero)
.RegisterRemoteVolume(ipdv.m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, cancellationToken)
.ConfigureAwait(false);
return ipdv;
}
@@ -879,21 +880,24 @@ namespace Duplicati.Library.Main.Operation
/// <param name="offset">The offset in the buffer where the block data starts.</param>
/// <param name="size">The size of the block data.</param>
/// <param name="hint">The compression hint for the block.</param>
public async Task AddBlock(string hash, byte[] buffer, int offset, int size, CompressionHint hint)
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
/// <returns>A task that completes when the block is added.</returns>
public async Task AddBlock(string hash, byte[] buffer, int offset, int size, CompressionHint hint, CancellationToken cancellationToken)
{
await m_activeWriter
.AddBlock(hash, buffer, offset, size, hint)
.ConfigureAwait(false);
m_anyData = true;
if (IsFull)
await StartNewVolume().ConfigureAwait(false);
await StartNewVolume(cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Starts a new volume for the current operation.
/// </summary>
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
/// <returns>A task that completes when the new volume is started.</returns>
public async Task StartNewVolume()
public async Task StartNewVolume(CancellationToken cancellationToken)
{
if (!m_anyData)
return;
@@ -902,7 +906,7 @@ namespace Duplicati.Library.Main.Operation
m_activeWriter = new BlockVolumeWriter(m_options);
m_activeWriter.VolumeID = await m_database
.RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero)
.RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, cancellationToken)
.ConfigureAwait(false);
m_anyData = false;
}
@@ -920,7 +924,7 @@ namespace Duplicati.Library.Main.Operation
private async Task RunRepairDblocks(IBackendManager backendManager, LocalRepairDatabase db, IEnumerable<RemoteVolumeEntry> missingDblockFiles, Action incrementProgress, CancellationToken cancellationToken)
{
var currentVolume =
await InProgressDblockVolumes.CreateAsync(m_options, db)
await InProgressDblockVolumes.CreateAsync(m_options, db, cancellationToken)
.ConfigureAwait(false);
foreach (var n in missingDblockFiles)
@@ -932,7 +936,7 @@ namespace Duplicati.Library.Main.Operation
await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false);
if (!m_options.Dryrun)
await db.Transaction
.CommitAsync("CommitEarlyExit", false)
.CommitAsync("CommitEarlyExit", false, cancellationToken)
.ConfigureAwait(false);
return;
}
@@ -953,13 +957,13 @@ namespace Duplicati.Library.Main.Operation
// If we have any data in the volume, complete it so it will be uploaded
if (currentVolume.AnyData)
await currentVolume.StartNewVolume().ConfigureAwait(false);
await currentVolume.StartNewVolume(cancellationToken).ConfigureAwait(false);
// Complete any pending uploads
await UploadCompletedBlockVolumes(backendManager, currentVolume, db, cancellationToken).ConfigureAwait(false);
// After reset, we need to remove the temporary volume from the database
await db
.RemoveRemoteVolume(currentVolume.RemoteFilename)
.RemoveRemoteVolume(currentVolume.RemoteFilename, cancellationToken)
.ConfigureAwait(false);
}
@@ -976,7 +980,7 @@ namespace Duplicati.Library.Main.Operation
foreach (var completedVolume in activeVolume.CompletedWriters)
{
await db
.UpdateRemoteVolume(completedVolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null)
.UpdateRemoteVolume(completedVolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, cancellationToken)
.ConfigureAwait(false);
// Create a new index file that points to the new volume
@@ -986,19 +990,19 @@ namespace Duplicati.Library.Main.Operation
{
newvolindex = new IndexVolumeWriter(m_options);
newvolindex.VolumeID = await db
.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary)
.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, cancellationToken)
.ConfigureAwait(false);
newvolindex.StartVolume(completedVolume.RemoteFilename);
await foreach (var b in db.GetBlocks(completedVolume.VolumeID).ConfigureAwait(false))
await foreach (var b in db.GetBlocks(completedVolume.VolumeID, cancellationToken).ConfigureAwait(false))
newvolindex.AddBlock(b.Hash, b.Size);
await db
.AddIndexBlockLink(newvolindex.VolumeID, completedVolume.VolumeID)
.AddIndexBlockLink(newvolindex.VolumeID, completedVolume.VolumeID, cancellationToken)
.ConfigureAwait(false);
if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full)
indexVolumeFinished = async () =>
{
await foreach (var blocklist in db.GetBlocklists(completedVolume.VolumeID, m_options.Blocksize, m_options.BlockhashSize).ConfigureAwait(false))
await foreach (var blocklist in db.GetBlocklists(completedVolume.VolumeID, m_options.Blocksize, m_options.BlockhashSize, cancellationToken).ConfigureAwait(false))
newvolindex.WriteBlocklist(blocklist.Item1, blocklist.Item2, 0, blocklist.Item3);
};
}
@@ -1010,7 +1014,7 @@ namespace Duplicati.Library.Main.Operation
else
{
await db.Transaction
.CommitAsync("PostRepairPreUploadBlockVolume")
.CommitAsync("PostRepairPreUploadBlockVolume", token: cancellationToken)
.ConfigureAwait(false);
await backendManager.PutAsync(completedVolume, newvolindex, indexVolumeFinished, false, null, cancellationToken).ConfigureAwait(false);
}
@@ -1022,13 +1026,13 @@ namespace Duplicati.Library.Main.Operation
// Prepare for deleting the old stuff
foreach (var vol in activeVolume.VolumesToDelete)
await db
.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Deleting, vol.Size, vol.Hash, false, TimeSpan.FromHours(2), null)
.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Deleting, vol.Size, vol.Hash, false, TimeSpan.FromHours(2), null, cancellationToken)
.ConfigureAwait(false);
// Persist desired state prior to deleting the old files
if (!m_options.Dryrun)
await db.Transaction
.CommitAsync("PostRepairBlockVolume")
.CommitAsync("PostRepairBlockVolume", token: cancellationToken)
.ConfigureAwait(false);
// Delete the old files
@@ -1042,7 +1046,7 @@ namespace Duplicati.Library.Main.Operation
await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false);
if (!m_options.Dryrun)
await db.Transaction
.CommitAsync("PostRepairBlockVolume")
.CommitAsync("PostRepairBlockVolume", token: cancellationToken)
.ConfigureAwait(false);
activeVolume.CompletedWriters.Clear();
@@ -1064,18 +1068,18 @@ namespace Duplicati.Library.Main.Operation
// as data can be either file contents, metadata or blocklist hashes
// We attempt to recover all three source parts in the steps below
await using var mbl = await db
.CreateBlockList(originalVolume.Name)
.CreateBlockList(originalVolume.Name, cancellationToken)
.ConfigureAwait(false);
var originalMissingBlockCount = await mbl
.GetMissingBlockCount()
.GetMissingBlockCount(cancellationToken)
.ConfigureAwait(false);
var recoveredSourceBlocks = 0L;
// First we grab all known blocks from local files
string lastRestoredHash = null;
long lastRestoredSize = -1;
await foreach (var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize).ConfigureAwait(false))
await foreach (var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize, cancellationToken).ConfigureAwait(false))
{
if (block.Hash == lastRestoredHash && block.Size == lastRestoredSize)
continue;
@@ -1111,11 +1115,11 @@ namespace Duplicati.Library.Main.Operation
// Found it, no need to look again
lastRestoredHash = block.Hash;
lastRestoredSize = block.Size;
if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID).ConfigureAwait(false))
if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID, cancellationToken).ConfigureAwait(false))
{
recoveredSourceBlocks++;
await pendingVolume
.AddBlock(block.Hash, buffer, 0, size, CompressionHint.Default)
.AddBlock(block.Hash, buffer, 0, size, CompressionHint.Default, cancellationToken)
.ConfigureAwait(false);
}
}
@@ -1126,7 +1130,7 @@ namespace Duplicati.Library.Main.Operation
}
// Then grab all blocks with metadata
await foreach (var block in mbl.GetSourceItemsWithMetadataBlocks().ConfigureAwait(false))
await foreach (var block in mbl.GetSourceItemsWithMetadataBlocks(cancellationToken).ConfigureAwait(false))
{
if (block.Hash == lastRestoredHash && block.Size == lastRestoredSize)
continue;
@@ -1162,11 +1166,11 @@ namespace Duplicati.Library.Main.Operation
// Found it, no need to look again
lastRestoredHash = block.Hash;
lastRestoredSize = block.Size;
if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID).ConfigureAwait(false))
if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID, cancellationToken).ConfigureAwait(false))
{
recoveredSourceBlocks++;
await pendingVolume
.AddBlock(block.Hash, metahash.Blob, 0, metahash.Blob.Length, CompressionHint.Default)
.AddBlock(block.Hash, metahash.Blob, 0, metahash.Blob.Length, CompressionHint.Default, cancellationToken)
.ConfigureAwait(false);
}
}
@@ -1201,9 +1205,9 @@ namespace Duplicati.Library.Main.Operation
if (resulthash == lastBlocklist.BlocklistHash && targetsize == blocklistHash.Length)
{
if (await mbl.SetBlockRestored(resulthash, blocklistHash.Length, pendingVolume.VolumeID).ConfigureAwait(false))
if (await mbl.SetBlockRestored(resulthash, blocklistHash.Length, pendingVolume.VolumeID, cancellationToken).ConfigureAwait(false))
await pendingVolume
.AddBlock(resulthash, blocklistHash.ToArray(), 0, (int)blocklistHash.Length, CompressionHint.Default)
.AddBlock(resulthash, blocklistHash.ToArray(), 0, (int)blocklistHash.Length, CompressionHint.Default, cancellationToken)
.ConfigureAwait(false);
}
else
@@ -1214,7 +1218,7 @@ namespace Duplicati.Library.Main.Operation
blocklistHash.SetLength(0);
}
await foreach (var blocklist in mbl.GetBlocklistHashes(hashesPerBlock).ConfigureAwait(false))
await foreach (var blocklist in mbl.GetBlocklistHashes(hashesPerBlock, cancellationToken).ConfigureAwait(false))
{
lastBlocklist ??= blocklist;
@@ -1231,7 +1235,7 @@ namespace Duplicati.Library.Main.Operation
await EmitBlockListBlock().ConfigureAwait(false);
//Then we grab all remote volumes that have the missing blocks
await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(await mbl.GetMissingBlockSources().ToListAsync(cancellationToken: cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false))
await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(await mbl.GetMissingBlockSources(cancellationToken).ToListAsync(cancellationToken: cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false))
{
try
{
@@ -1239,10 +1243,10 @@ namespace Duplicati.Library.Main.Operation
using (tmpfile)
using (var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options))
foreach (var b in f.Blocks)
if (await mbl.SetBlockRestored(b.Key, b.Value, pendingVolume.VolumeID).ConfigureAwait(false))
if (await mbl.SetBlockRestored(b.Key, b.Value, pendingVolume.VolumeID, cancellationToken).ConfigureAwait(false))
if (f.ReadBlock(b.Key, buffer) == b.Value)
await pendingVolume
.AddBlock(b.Key, buffer, 0, (int)b.Value, CompressionHint.Default)
.AddBlock(b.Key, buffer, 0, (int)b.Value, CompressionHint.Default, cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex)
@@ -1253,13 +1257,13 @@ namespace Duplicati.Library.Main.Operation
}
var missingBlocks = await mbl
.GetMissingBlockCount()
.GetMissingBlockCount(cancellationToken)
.ConfigureAwait(false);
var recoveredBlocks = originalMissingBlockCount - missingBlocks;
if (recoveredBlocks == 0 || (m_options.DisablePartialDblockRecovery && missingBlocks > 0))
{
Logging.Log.WriteInformationMessage(LOGTAG, "RepairMissingBlocks", "Repair cannot acquire {0} required blocks for volume {1}, which are required by the following filesets: ", missingBlocks, originalVolume.Name);
await foreach (var f in mbl.GetFilesetsUsingMissingBlocks().ConfigureAwait(false))
await foreach (var f in mbl.GetFilesetsUsingMissingBlocks(cancellationToken).ConfigureAwait(false))
Logging.Log.WriteInformationMessage(LOGTAG, "AffectedFilesetName", f.Name);
var recoverymsg = string.Format("If you want to continue working with the database, you can use the \"{0}\" and \"{1}\" commands to purge the missing data from the database and the remote storage.", "list-broken-files", "purge-broken-files");
@@ -1280,7 +1284,7 @@ namespace Duplicati.Library.Main.Operation
{
pendingVolume.VolumesToDelete.Add(originalVolume);
var oldIndexFiles = await db
.GetIndexFilesReferencingBlockFile(originalVolume.ID)
.GetIndexFilesReferencingBlockFile(originalVolume.ID, cancellationToken)
.ToListAsync(cancellationToken: cancellationToken)
.ConfigureAwait(false);
@@ -1288,12 +1292,12 @@ namespace Duplicati.Library.Main.Operation
foreach (var oldIndexFile in oldIndexFiles)
{
var oldVolume = await db
.GetRemoteVolume(oldIndexFile)
.GetRemoteVolume(oldIndexFile, cancellationToken)
.ConfigureAwait(false);
if (oldVolume.State == RemoteVolumeState.Uploading || oldVolume.State == RemoteVolumeState.Uploaded || oldVolume.State == RemoteVolumeState.Verified)
{
var blockVolumesReferenced = await db
.GetBlockVolumesFromIndexName(oldIndexFile)
.GetBlockVolumesFromIndexName(oldIndexFile, cancellationToken)
.ToListAsync(cancellationToken: cancellationToken)
.ConfigureAwait(false);
if (blockVolumesReferenced.Any(x => x.Name != originalVolume.Name))
@@ -1322,12 +1326,12 @@ namespace Duplicati.Library.Main.Operation
throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist");
await using var db =
await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache)
await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
var sets = await db
.GetFilesetsWithMissingFiles()
.ToListAsync()
.GetFilesetsWithMissingFiles(m_result.TaskControl.ProgressToken)
.ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (sets.Count == 0)
@@ -1340,7 +1344,7 @@ namespace Duplicati.Library.Main.Operation
ix++;
Logging.Log.WriteInformationMessage(LOGTAG, "RepairingBrokenFileset", "Repairing broken fileset {0} of {1}: {2}", ix, sets.Count, entry.Value);
var volume = await db
.GetRemoteVolumeFromFilesetID(entry.Key)
.GetRemoteVolumeFromFilesetID(entry.Key, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
var parsed = VolumeBase.ParseFilename(volume.Name);
using var tmpfile = await backendManager.GetAsync(volume.Name, volume.Hash, volume.Size, CancellationToken.None).ConfigureAwait(false);
@@ -1350,15 +1354,17 @@ namespace Duplicati.Library.Main.Operation
throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule");
// Clear out the old fileset
await db.DeleteFilesetEntries(entry.Key).ConfigureAwait(false);
await using (var rdb = await LocalRecreateDatabase.CreateAsync(db, m_options).ConfigureAwait(false))
await db
.DeleteFilesetEntries(entry.Key, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await using (var rdb = await LocalRecreateDatabase.CreateAsync(db, m_options, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
{
await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(rdb, compressor, entry.Key, m_options, new FilterExpression())
await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(rdb, compressor, entry.Key, m_options, new FilterExpression(), m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
await db.Transaction
.CommitAsync("PostRepairFileset")
.CommitAsync("PostRepairFileset", true, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
}
@@ -1377,22 +1383,22 @@ namespace Duplicati.Library.Main.Operation
m_result.OperationProgressUpdater.UpdateProgress(0);
await using var db =
await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache)
await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await Utility.UpdateOptionsFromDb(db, m_options)
await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (await db.RepairInProgress().ConfigureAwait(false) || await db.PartiallyRecreated().ConfigureAwait(false))
if (await db.RepairInProgress(m_result.TaskControl.ProgressToken).ConfigureAwait(false) || await db.PartiallyRecreated(m_result.TaskControl.ProgressToken).ConfigureAwait(false))
Logging.Log.WriteWarningMessage(LOGTAG, "InProgressDatabase", null, "The database is marked as \"in-progress\" and may be incomplete.");
await db.FixDuplicateMetahash().ConfigureAwait(false);
await db.FixDuplicateFileentries().ConfigureAwait(false);
await db.FixDuplicateMetahash(m_result.TaskControl.ProgressToken).ConfigureAwait(false);
await db.FixDuplicateFileentries(m_result.TaskControl.ProgressToken).ConfigureAwait(false);
await db
.FixDuplicateBlocklistHashes(m_options.Blocksize, m_options.BlockhashSize)
.FixDuplicateBlocklistHashes(m_options.Blocksize, m_options.BlockhashSize, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await db
.FixMissingBlocklistHashes(m_options.BlockHashAlgorithm, m_options.Blocksize)
.FixMissingBlocklistHashes(m_options.BlockHashAlgorithm, m_options.Blocksize, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
}
Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs
+9 -2
View File
@@ -46,7 +46,14 @@ namespace Duplicati.Library.Main.Operation
Directory.CreateDirectory(m_options.Restorepath);
using var tmpdb = new TempFile();
await using var db = await Database.LocalDatabase.CreateLocalDatabaseAsync(File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles", true, m_options.SqlitePageCache).ConfigureAwait(false);
await using var db = await Database.LocalDatabase.CreateLocalDatabaseAsync(
File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles",
true,
m_options.SqlitePageCache,
null,
m_result.TaskControl.ProgressToken
).ConfigureAwait(false);
var filter = JoinedFilterExpression.Join(new FilterExpression(filterstrings), compositefilter);
try
@@ -68,7 +75,7 @@ namespace Duplicati.Library.Main.Operation
var file = fileversion.Value.File;
var entry = await db
.GetRemoteVolume(file.Name)
.GetRemoteVolume(file.Name, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
var res = new List<string>();
Duplicati/Library/Main/Operation/RestoreHandler.cs
+63 -63
View File
@@ -115,7 +115,7 @@ namespace Duplicati.Library.Main.Operation
{
if (!m_options.NoLocalDb && SystemIO.IO_OS.FileExists(m_options.Dbpath))
{
db = await LocalRestoreDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache)
db = await LocalRestoreDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
else
@@ -123,7 +123,7 @@ namespace Duplicati.Library.Main.Operation
Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, building a temporary database");
tmpdb = new TempFile();
RecreateDatabaseHandler.NumberedFilterFilelistDelegate filelistfilter = FilterNumberedFilelist(m_options.Time, m_options.Version);
db = await LocalRestoreDatabase.CreateAsync(tmpdb, m_options.SqlitePageCache)
db = await LocalRestoreDatabase.CreateAsync(tmpdb, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
m_result.RecreateDatabaseResults = new RecreateDatabaseResults(m_result);
using (new Logging.Timer(LOGTAG, "RecreateTempDbForRestore", "Recreate temporary database for restore"))
@@ -152,16 +152,16 @@ namespace Duplicati.Library.Main.Operation
}
}
private static async Task PatchWithBlocklist(LocalRestoreDatabase database, BlockVolumeReader blocks, Options options, RestoreResults result, byte[] blockbuffer, RestoreHandlerMetadataStorage metadatastorage)
private static async Task PatchWithBlocklist(LocalRestoreDatabase database, BlockVolumeReader blocks, Options options, RestoreResults result, byte[] blockbuffer, RestoreHandlerMetadataStorage metadatastorage, CancellationToken cancellationToken)
{
var blocksize = options.Blocksize;
var updateCounter = 0L;
var fullblockverification = options.FullBlockVerification;
using var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm);
await using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false);
await using var volumekeeper = await database.GetMissingBlockData(blocks, options.Blocksize).ConfigureAwait(false);
await foreach (var restorelist in volumekeeper.FilesWithMissingBlocks().ConfigureAwait(false))
await using var blockmarker = await database.CreateBlockMarkerAsync(cancellationToken).ConfigureAwait(false);
await using var volumekeeper = await database.GetMissingBlockData(blocks, options.Blocksize, cancellationToken).ConfigureAwait(false);
await foreach (var restorelist in volumekeeper.FilesWithMissingBlocks(cancellationToken).ConfigureAwait(false))
{
var targetpath = restorelist.Path;
@@ -185,7 +185,7 @@ namespace Duplicati.Library.Main.Operation
// TODO: Much faster if we iterate the volume and checks what blocks are used,
// because the compressors usually like sequential reading
using (var file = SystemIO.IO_OS.FileOpenWrite(targetpath))
await foreach (var targetblock in restorelist.Blocks().ConfigureAwait(false))
await foreach (var targetblock in restorelist.Blocks(cancellationToken).ConfigureAwait(false))
{
file.Position = targetblock.Offset;
var size = blocks.ReadBlock(targetblock.Key, blockbuffer);
@@ -205,7 +205,7 @@ namespace Duplicati.Library.Main.Operation
{
file.Write(blockbuffer, 0, size);
await blockmarker
.SetBlockRestored(restorelist.FileID, targetblock.Offset / blocksize, targetblock.Key, size, false)
.SetBlockRestored(restorelist.FileID, targetblock.Offset / blocksize, targetblock.Key, size, false, cancellationToken)
.ConfigureAwait(false);
}
}
@@ -217,7 +217,7 @@ namespace Duplicati.Library.Main.Operation
if ((++updateCounter) % 20 == 0)
await blockmarker
.UpdateProcessed(result.OperationProgressUpdater)
.UpdateProcessed(result.OperationProgressUpdater, cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex)
@@ -231,7 +231,7 @@ namespace Duplicati.Library.Main.Operation
if (!options.SkipMetadata)
{
await foreach (var restoremetadata in volumekeeper.MetadataWithMissingBlocks().ConfigureAwait(false))
await foreach (var restoremetadata in volumekeeper.MetadataWithMissingBlocks(cancellationToken).ConfigureAwait(false))
{
var targetpath = restoremetadata.Path;
Logging.Log.WriteVerboseMessage(LOGTAG, "RecordingMetadata", "Recording metadata from remote data: {0}", targetpath);
@@ -241,7 +241,7 @@ namespace Duplicati.Library.Main.Operation
// TODO: When we support multi-block metadata this needs to deal with it
using (var ms = new System.IO.MemoryStream())
{
await foreach (var targetblock in restoremetadata.Blocks().ConfigureAwait(false))
await foreach (var targetblock in restoremetadata.Blocks(cancellationToken).ConfigureAwait(false))
{
ms.Position = targetblock.Offset;
var size = blocks.ReadBlock(targetblock.Key, blockbuffer);
@@ -249,7 +249,7 @@ namespace Duplicati.Library.Main.Operation
{
ms.Write(blockbuffer, 0, size);
await blockmarker
.SetBlockRestored(restoremetadata.FileID, targetblock.Offset / blocksize, targetblock.Key, size, true)
.SetBlockRestored(restoremetadata.FileID, targetblock.Offset / blocksize, targetblock.Key, size, true, cancellationToken)
.ConfigureAwait(false);
}
}
@@ -268,9 +268,9 @@ namespace Duplicati.Library.Main.Operation
}
}
await blockmarker
.UpdateProcessed(result.OperationProgressUpdater)
.UpdateProcessed(result.OperationProgressUpdater, cancellationToken)
.ConfigureAwait(false);
await blockmarker.CommitAsync().ConfigureAwait(false);
await blockmarker.CommitAsync(cancellationToken).ConfigureAwait(false);
}
private static void ApplyStoredMetadata(Options options, RestoreHandlerMetadataStorage metadatastorage)
@@ -316,16 +316,16 @@ namespace Duplicati.Library.Main.Operation
private async Task DoRunNewAsync(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter filter, CancellationToken cancellationToken)
{
// Perform initial setup
await Utility.UpdateOptionsFromDb(database, m_options)
await Utility.UpdateOptionsFromDb(database, m_options, cancellationToken)
.ConfigureAwait(false);
await Utility.VerifyOptionsAndUpdateDatabase(database, m_options)
await Utility.VerifyOptionsAndUpdateDatabase(database, m_options, cancellationToken)
.ConfigureAwait(false);
// Verify the backend if necessary
if (!m_options.NoBackendverification)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify);
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false);
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, cancellationToken).ConfigureAwait(false);
}
// Prepare the block and file list and create the directory structure
@@ -339,7 +339,7 @@ namespace Duplicati.Library.Main.Operation
// At this point, there should be no more writes to the database, so we have to unlock the database:
await database.Transaction
.CommitAsync("CommitBeforeRestore")
.CommitAsync("CommitBeforeRestore", token: cancellationToken)
.ConfigureAwait(false);
using var setup_log_timer = new Logging.Timer(LOGTAG, "RestoreNetworkSetup", "RestoreNetworkSetup");
@@ -391,7 +391,7 @@ namespace Duplicati.Library.Main.Operation
}
await database.Transaction
.CommitAsync("CommitAfterRestore")
.CommitAsync("CommitAfterRestore", token: cancellationToken)
.ConfigureAwait(false);
await database.DisposePoolAsync().ConfigureAwait(false);
@@ -422,7 +422,7 @@ namespace Duplicati.Library.Main.Operation
Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesRestored", null, "Restore completed without errors but no files were restored");
// Drop the temp tables
await database.DropRestoreTable().ConfigureAwait(false);
await database.DropRestoreTable(cancellationToken).ConfigureAwait(false);
await backendManager.WaitForEmptyAsync(database, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
// Report that the restore is complete
@@ -443,16 +443,16 @@ namespace Duplicati.Library.Main.Operation
//using (var database = new LocalRestoreDatabase(dbparent))
using (var metadatastorage = new RestoreHandlerMetadataStorage())
{
await Utility.UpdateOptionsFromDb(database, m_options)
await Utility.UpdateOptionsFromDb(database, m_options, cancellationToken)
.ConfigureAwait(false);
await Utility.VerifyOptionsAndUpdateDatabase(database, m_options)
await Utility.VerifyOptionsAndUpdateDatabase(database, m_options, cancellationToken)
.ConfigureAwait(false);
m_blockbuffer = new byte[m_options.Blocksize];
if (!m_options.NoBackendverification)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify);
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false);
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, cancellationToken).ConfigureAwait(false);
}
//Figure out what files are to be patched, and what blocks are needed
@@ -510,8 +510,8 @@ namespace Duplicati.Library.Main.Operation
List<IRemoteVolume> volumes;
using (new Logging.Timer(LOGTAG, "GetMissingVolumes", "GetMissingVolumes"))
volumes = await database
.GetMissingVolumes()
.ToListAsync()
.GetMissingVolumes(cancellationToken)
.ToListAsync(cancellationToken: cancellationToken)
.ConfigureAwait(false);
if (volumes.Count > 0)
@@ -535,7 +535,7 @@ namespace Duplicati.Library.Main.Operation
using (tmpfile)
using (var blocks = new BlockVolumeReader(GetCompressionModule(name), tmpfile, m_options))
await PatchWithBlocklist(database, blocks, m_options, m_result, m_blockbuffer, metadatastorage)
await PatchWithBlocklist(database, blocks, m_options, m_result, m_blockbuffer, metadatastorage, cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex)
@@ -550,7 +550,7 @@ namespace Duplicati.Library.Main.Operation
var fileErrors = 0L;
// Restore empty files. They might not have any blocks so don't appear in any volume.
await foreach (var file in database.GetFilesToRestore(true).Where(item => item.Length == 0).ConfigureAwait(false))
await foreach (var file in database.GetFilesToRestore(true, cancellationToken).Where(item => item.Length == 0).ConfigureAwait(false))
{
Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreEmptyFile", "Restoring empty file \"{0}\"", file.Path);
@@ -589,7 +589,7 @@ namespace Duplicati.Library.Main.Operation
// After all blocks in the files are restored, verify the file hash
using (var filehasher = HashFactory.CreateHasher(m_options.FileHashAlgorithm))
using (new Logging.Timer(LOGTAG, "RestoreVerification", "RestoreVerification"))
await foreach (var file in database.GetFilesToRestore(true).ConfigureAwait(false))
await foreach (var file in database.GetFilesToRestore(true, cancellationToken).ConfigureAwait(false))
{
try
{
@@ -632,7 +632,7 @@ namespace Duplicati.Library.Main.Operation
Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesRestored", null, "Restore completed without errors but no files were restored");
// Drop the temp tables
await database.DropRestoreTable().ConfigureAwait(false);
await database.DropRestoreTable(cancellationToken).ConfigureAwait(false);
await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false);
}
@@ -712,9 +712,9 @@ namespace Duplicati.Library.Main.Operation
private static async Task ScanForExistingSourceBlocksFast(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result)
{
// Fill BLOCKS with data from known local source files
await using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false);
await using var blockmarker = await database.CreateBlockMarkerAsync(result.TaskControl.ProgressToken).ConfigureAwait(false);
var updateCount = 0L;
await foreach (var entry in database.GetFilesAndSourceBlocksFast(options.Blocksize).ConfigureAwait(false))
await foreach (var entry in database.GetFilesAndSourceBlocksFast(options.Blocksize, result.TaskControl.ProgressToken).ConfigureAwait(false))
{
var targetpath = entry.TargetPath;
var targetfileid = entry.TargetFileID;
@@ -737,7 +737,7 @@ namespace Duplicati.Library.Main.Operation
try
{
using var sourcestream = SystemIO.IO_OS.FileOpenRead(sourcepath);
await foreach (var block in entry.Blocks().ConfigureAwait(false))
await foreach (var block in entry.Blocks(result.TaskControl.ProgressToken).ConfigureAwait(false))
{
if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
return;
@@ -761,7 +761,7 @@ namespace Duplicati.Library.Main.Operation
targetstream.Write(blockbuffer, 0, size);
}
await blockmarker.SetBlockRestored(targetfileid, block.Index, key, block.Size, false)
await blockmarker.SetBlockRestored(targetfileid, block.Index, key, block.Size, false, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
}
@@ -779,7 +779,7 @@ namespace Duplicati.Library.Main.Operation
if ((++updateCount) % 20 == 0)
{
await blockmarker
.UpdateProcessed(result.OperationProgressUpdater)
.UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
return;
@@ -810,17 +810,17 @@ namespace Duplicati.Library.Main.Operation
}
await blockmarker
.UpdateProcessed(result.OperationProgressUpdater)
.UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await blockmarker.CommitAsync().ConfigureAwait(false);
await blockmarker.CommitAsync(result.TaskControl.ProgressToken).ConfigureAwait(false);
}
private static async Task ScanForExistingSourceBlocks(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result, RestoreHandlerMetadataStorage metadatastorage)
{
// Fill BLOCKS with data from known local source files
await using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false);
await using var blockmarker = await database.CreateBlockMarkerAsync(result.TaskControl.ProgressToken).ConfigureAwait(false);
var updateCount = 0L;
await foreach (var restorelist in database.GetFilesAndSourceBlocks(options.SkipMetadata, options.Blocksize).ConfigureAwait(false))
await foreach (var restorelist in database.GetFilesAndSourceBlocks(options.SkipMetadata, options.Blocksize, result.TaskControl.ProgressToken).ConfigureAwait(false))
{
var targetpath = restorelist.TargetPath;
var targetfileid = restorelist.TargetFileID;
@@ -838,9 +838,9 @@ namespace Duplicati.Library.Main.Operation
}
using (var file = options.Dryrun ? null : SystemIO.IO_OS.FileOpenWrite(targetpath))
await foreach (var targetblock in restorelist.Blocks().ConfigureAwait(false))
await foreach (var targetblock in restorelist.Blocks(result.TaskControl.ProgressToken).ConfigureAwait(false))
{
await foreach (var source in targetblock.BlockSources().ConfigureAwait(false))
await foreach (var source in targetblock.BlockSources(result.TaskControl.ProgressToken).ConfigureAwait(false))
{
try
{
@@ -878,7 +878,7 @@ namespace Duplicati.Library.Main.Operation
}
await blockmarker
.SetBlockRestored(targetfileid, targetblock.Index, key, targetblock.Size, false)
.SetBlockRestored(targetfileid, targetblock.Index, key, targetblock.Size, false, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
patched = true;
break;
@@ -898,7 +898,7 @@ namespace Duplicati.Library.Main.Operation
if ((++updateCount) % 20 == 0)
await blockmarker
.UpdateProcessed(result.OperationProgressUpdater)
.UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
catch (Exception ex)
@@ -918,9 +918,9 @@ namespace Duplicati.Library.Main.Operation
}
await blockmarker
.UpdateProcessed(result.OperationProgressUpdater)
.UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await blockmarker.CommitAsync().ConfigureAwait(false);
await blockmarker.CommitAsync(result.TaskControl.ProgressToken).ConfigureAwait(false);
}
private static async Task PrepareBlockAndFileList(LocalRestoreDatabase database, Options options, Library.Utility.IFilter filter, RestoreResults result)
@@ -930,7 +930,7 @@ namespace Duplicati.Library.Main.Operation
using (new Logging.Timer(LOGTAG, "PrepareRestoreFileList", "PrepareRestoreFileList"))
{
var c = await database
.PrepareRestoreFilelist(options.Time, options.Version, filter)
.PrepareRestoreFilelist(options.Time, options.Version, filter, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
result.OperationProgressUpdater.UpdatefileCount(c.Item1, c.Item2, true);
}
@@ -941,30 +941,30 @@ namespace Duplicati.Library.Main.Operation
// Find the largest common prefix
var largest_prefix = options.DontCompressRestorePaths
? "" :
await database.GetLargestPrefix().ConfigureAwait(false);
await database.GetLargestPrefix(result.TaskControl.ProgressToken).ConfigureAwait(false);
Logging.Log.WriteVerboseMessage(LOGTAG, "MappingRestorePath", "Mapping restore path prefix to \"{0}\" to \"{1}\"", largest_prefix, Util.AppendDirSeparator(options.Restorepath));
// Set the target paths, special care with C:\ and /
await database
.SetTargetPaths(largest_prefix, Util.AppendDirSeparator(options.Restorepath))
.SetTargetPaths(largest_prefix, Util.AppendDirSeparator(options.Restorepath), result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
else
{
await database.SetTargetPaths("", "").ConfigureAwait(false);
await database.SetTargetPaths("", "", result.TaskControl.ProgressToken).ConfigureAwait(false);
}
// Create a temporary table BLOCKS that lists all blocks that needs to be recovered
using (new Logging.Timer(LOGTAG, "FindMissingBlocks", "FindMissingBlocks"))
await database
.FindMissingBlocks(options.SkipMetadata)
.FindMissingBlocks(options.SkipMetadata, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
// Create temporary tables and triggers that automatically track progress
using (new Logging.Timer(LOGTAG, "CreateProgressTracker", "CreateProgressTracker"))
await database
.CreateProgressTracker(false)
.CreateProgressTracker(false, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
@@ -983,7 +983,7 @@ namespace Duplicati.Library.Main.Operation
SystemIO.IO_OS.DirectoryCreate(options.Restorepath);
}
await foreach (var folder in database.GetTargetFolders().ConfigureAwait(false))
await foreach (var folder in database.GetTargetFolders(result.TaskControl.ProgressToken).ConfigureAwait(false))
{
try
{
@@ -1014,9 +1014,9 @@ namespace Duplicati.Library.Main.Operation
private static async Task ScanForExistingTargetBlocks(LocalRestoreDatabase database, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm blockhasher, System.Security.Cryptography.HashAlgorithm filehasher, Options options, RestoreResults result)
{
// Scan existing files for existing BLOCKS
await using var blockmarker = await database.CreateBlockMarkerAsync().ConfigureAwait(false);
await using var blockmarker = await database.CreateBlockMarkerAsync(result.TaskControl.ProgressToken).ConfigureAwait(false);
var updateCount = 0L;
await foreach (var restorelist in database.GetExistingFilesWithBlocks().ConfigureAwait(false))
await foreach (var restorelist in database.GetExistingFilesWithBlocks(result.TaskControl.ProgressToken).ConfigureAwait(false))
{
var rename = !options.Overwrite;
var targetpath = restorelist.TargetPath;
@@ -1069,7 +1069,7 @@ namespace Duplicati.Library.Main.Operation
using (var file = SystemIO.IO_OS.FileOpenRead(targetpath))
using (var block = new Blockprocessor(file, blockbuffer))
await foreach (var targetblock in restorelist.Blocks().ConfigureAwait(false))
await foreach (var targetblock in restorelist.Blocks(result.TaskControl.ProgressToken).ConfigureAwait(false))
{
var size = block.Readblock();
if (size <= 0)
@@ -1095,7 +1095,7 @@ namespace Duplicati.Library.Main.Operation
if (key == targetblock.Hash)
{
await blockmarker
.SetBlockRestored(targetfileid, targetblock.Index, key, size, false)
.SetBlockRestored(targetfileid, targetblock.Index, key, size, false, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
blockhashmatch = true;
}
@@ -1132,7 +1132,7 @@ namespace Duplicati.Library.Main.Operation
{
//TODO: Check metadata to trigger rename? If metadata changed, it will still be restored for the file in-place.
await blockmarker
.SetFileDataVerified(targetfileid)
.SetFileDataVerified(targetfileid, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
Logging.Log.WriteVerboseMessage(LOGTAG, "TargetExistsInCorrectVersion", "Target file exists{1} and is correct version: {0}", targetpath, wasTruncated ? " (but was truncated)" : "");
rename = false;
@@ -1142,7 +1142,7 @@ namespace Duplicati.Library.Main.Operation
// The new file will have none of the correct blocks,
// even if the scanned file had some
await blockmarker
.SetAllBlocksMissing(targetfileid)
.SetAllBlocksMissing(targetfileid, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
}
@@ -1150,7 +1150,7 @@ namespace Duplicati.Library.Main.Operation
if ((++updateCount) % 20 == 0)
{
await blockmarker
.UpdateProcessed(result.OperationProgressUpdater)
.UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
return;
@@ -1199,10 +1199,10 @@ namespace Duplicati.Library.Main.Operation
//TODO: Also needs metadata check to make correct decision.
// We stick to the policy to restore metadata in place, if data ok. So, metadata block may be restored.
await blockmarker
.SetAllBlocksRestored(targetfileid, false)
.SetAllBlocksRestored(targetfileid, false, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await blockmarker
.SetFileDataVerified(targetfileid)
.SetFileDataVerified(targetfileid, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
break;
}
@@ -1220,17 +1220,17 @@ namespace Duplicati.Library.Main.Operation
Logging.Log.WriteVerboseMessage(LOGTAG, "TargetFileRetargeted", "Target file exists and will be restored to: {0}", newname);
await database
.UpdateTargetPath(targetfileid, newname)
.UpdateTargetPath(targetfileid, newname, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
}
await blockmarker
.UpdateProcessed(result.OperationProgressUpdater)
.UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await blockmarker
.CommitAsync()
.CommitAsync(result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
}
Duplicati/Library/Main/Operation/SearchEntriesHandler.cs
+4 -4
View File
@@ -50,14 +50,14 @@ internal static class SearchEntriesHandler
throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase");
await using var db =
await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache)
await Database.LocalListDatabase.CreateAsync(options.Dbpath, options.SqlitePageCache, null, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
long[]? filesetIds = null;
if (!options.AllVersions)
{
filesetIds = await db
.GetFilesetIDs(options.Time, options.Version)
.ToArrayAsync()
.GetFilesetIDs(options.Time, options.Version, false, result.TaskControl.ProgressToken)
.ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (filesetIds.Length == 0)
@@ -68,7 +68,7 @@ internal static class SearchEntriesHandler
paths = paths.Select(path => Util.AppendDirSeparator(path)).ToArray();
result.FileVersions = await db
.SearchEntries(paths, filter, filesetIds, offset, limit)
.SearchEntries(paths, filter, filesetIds, offset, limit, result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
}
Duplicati/Library/Main/Operation/TestHandler.cs
+66 -65
View File
@@ -39,12 +39,12 @@ namespace Duplicati.Library.Main.Operation
private static readonly string LOGTAG = Logging.Log.LogTagFromType<TestHandler>();
private readonly Options m_options;
private readonly TestResults m_results;
private readonly TestResults m_result;
public TestHandler(Options options, TestResults results)
{
m_options = options;
m_results = results;
m_result = results;
}
public async Task RunAsync(long samples, IBackendManager backendManager)
@@ -52,17 +52,17 @@ namespace Duplicati.Library.Main.Operation
if (!System.IO.File.Exists(m_options.Dbpath))
throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist");
await using var db = await LocalTestDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache).ConfigureAwait(false);
await Utility.UpdateOptionsFromDb(db, m_options)
await using var db = await LocalTestDatabase.CreateAsync(m_options.Dbpath, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await Utility.VerifyOptionsAndUpdateDatabase(db, m_options)
await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await db
.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks)
.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_results.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false);
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
await DoRunAsync(samples, db, backendManager).ConfigureAwait(false);
await db.Transaction
.CommitAsync("TestHandlerComplete")
@@ -72,35 +72,35 @@ namespace Duplicati.Library.Main.Operation
public async Task DoRunAsync(long samples, LocalTestDatabase db, IBackendManager backend)
{
var files = await db
.SelectTestTargets(samples, m_options)
.ToListAsync()
.SelectTestTargets(samples, m_options, m_result.TaskControl.ProgressToken)
.ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
m_results.OperationProgressUpdater.UpdatePhase(OperationPhase.Verify_Running);
m_results.OperationProgressUpdater.UpdateProgress(0);
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Verify_Running);
m_result.OperationProgressUpdater.UpdateProgress(0);
var progress = 0L;
if (m_options.FullRemoteVerification != Options.RemoteTestStrategy.False)
{
var faultyIndexFiles = new List<IRemoteVolume>();
await foreach (var (tf, hash, size, name) in backend.GetFilesOverlappedAsync(files, m_results.TaskControl.ProgressToken).ConfigureAwait(false))
await foreach (var (tf, hash, size, name) in backend.GetFilesOverlappedAsync(files, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
{
var vol = new RemoteVolume(name, hash, size);
try
{
if (!await m_results.TaskControl.ProgressRendevouz().ConfigureAwait(false))
if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
{
await backend.WaitForEmptyAsync(db, m_results.TaskControl.ProgressToken).ConfigureAwait(false);
m_results.EndTime = DateTime.UtcNow;
await backend.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
m_result.EndTime = DateTime.UtcNow;
return;
}
progress++;
m_results.OperationProgressUpdater.UpdateProgress((float)progress / files.Count);
m_result.OperationProgressUpdater.UpdateProgress((float)progress / files.Count);
KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>> res;
using (tf)
res = await TestVolumeInternals(db, vol, tf, m_options, m_options.FullBlockVerification ? 1.0 : 0.2)
res = await TestVolumeInternals(db, vol, tf, m_options, m_options.FullBlockVerification ? 1.0 : 0.2, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
var parsedInfo = VolumeBase.ParseFilename(vol.Name);
@@ -119,14 +119,14 @@ namespace Duplicati.Library.Main.Operation
faultyIndexFiles.Add(vol);
}
m_results.AddResult(res.Key, res.Value);
m_result.AddResult(res.Key, res.Value);
if (!string.IsNullOrWhiteSpace(vol.Hash) && vol.Size > 0)
{
if (res.Value == null || !res.Value.Any())
{
var rv = await db
.GetRemoteVolume(vol.Name)
.GetRemoteVolume(vol.Name, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
if (rv.ID < 0)
@@ -141,7 +141,7 @@ namespace Duplicati.Library.Main.Operation
{
Logging.Log.WriteInformationMessage(LOGTAG, "CaptureHashAndSize", "Successfully captured hash and size for {0}, updating database", vol.Name);
await db
.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Verified, vol.Size, vol.Hash)
.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Verified, vol.Size, vol.Hash, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
}
@@ -150,16 +150,16 @@ namespace Duplicati.Library.Main.Operation
}
await db
.UpdateVerificationCount(vol.Name)
.UpdateVerificationCount(vol.Name, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
catch (Exception ex)
{
m_results.AddResult(vol.Name, [new KeyValuePair<TestEntryStatus, string>(TestEntryStatus.Error, ex.Message)]);
m_result.AddResult(vol.Name, [new KeyValuePair<TestEntryStatus, string>(TestEntryStatus.Error, ex.Message)]);
Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileProcessingFailed", ex, "Failed to process file {0}", vol.Name);
if (ex.IsAbortException())
{
m_results.EndTime = DateTime.UtcNow;
m_result.EndTime = DateTime.UtcNow;
throw;
}
}
@@ -170,7 +170,7 @@ namespace Duplicati.Library.Main.Operation
if (m_options.ReplaceFaultyIndexFiles)
{
Logging.Log.WriteWarningMessage(LOGTAG, "FaultyIndexFiles", null, "Found {0} faulty index files, repairing now", faultyIndexFiles.Count);
await ReplaceFaultyIndexFilesAsync(faultyIndexFiles, backend, db, m_results.TaskControl.ProgressToken).ConfigureAwait(false);
await ReplaceFaultyIndexFilesAsync(faultyIndexFiles, backend, db, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
}
else
Logging.Log.WriteWarningMessage(LOGTAG, "FaultyIndexFiles", null, "Found {0} faulty index files, use the option {1} to repair them", faultyIndexFiles.Count, "--replace-faulty-index-files");
@@ -183,26 +183,26 @@ namespace Duplicati.Library.Main.Operation
{
try
{
if (!await m_results.TaskControl.ProgressRendevouz().ConfigureAwait(false))
if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
{
m_results.EndTime = DateTime.UtcNow;
m_result.EndTime = DateTime.UtcNow;
return;
}
progress++;
m_results.OperationProgressUpdater.UpdateProgress((float)progress / files.Count);
m_result.OperationProgressUpdater.UpdateProgress((float)progress / files.Count);
if (f.Size <= 0 || string.IsNullOrWhiteSpace(f.Hash))
{
Logging.Log.WriteInformationMessage(LOGTAG, "MissingRemoteHash", "No hash or size recorded for {0}, performing full verification", f.Name);
KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>> res;
(var tf, var hash, var size) = await backend.GetWithInfoAsync(f.Name, f.Hash, f.Size, m_results.TaskControl.ProgressToken).ConfigureAwait(false);
(var tf, var hash, var size) = await backend.GetWithInfoAsync(f.Name, f.Hash, f.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
using (tf)
res = await TestVolumeInternals(db, f, tf, m_options, 1)
res = await TestVolumeInternals(db, f, tf, m_options, 1, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
m_results.AddResult(res.Key, res.Value);
m_result.AddResult(res.Key, res.Value);
if (!string.IsNullOrWhiteSpace(hash) && size > 0)
{
@@ -216,7 +216,7 @@ namespace Duplicati.Library.Main.Operation
{
Logging.Log.WriteInformationMessage(LOGTAG, "CapturedHashAndSize", "Successfully captured hash and size for {0}, updating database", f.Name);
await db
.UpdateRemoteVolume(f.Name, RemoteVolumeState.Verified, size, hash)
.UpdateRemoteVolume(f.Name, RemoteVolumeState.Verified, size, hash, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
}
}
@@ -224,39 +224,39 @@ namespace Duplicati.Library.Main.Operation
}
else
{
using (var tf = await backend.GetAsync(f.Name, f.Hash, f.Size, m_results.TaskControl.ProgressToken).ConfigureAwait(false))
using (var tf = await backend.GetAsync(f.Name, f.Hash, f.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
{ }
}
await db
.UpdateVerificationCount(f.Name)
.UpdateVerificationCount(f.Name, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
m_results.AddResult(f.Name, []);
m_result.AddResult(f.Name, []);
}
catch (Exception ex)
{
m_results.AddResult(f.Name, [new KeyValuePair<TestEntryStatus, string>(TestEntryStatus.Error, ex.Message)]);
m_result.AddResult(f.Name, [new KeyValuePair<TestEntryStatus, string>(TestEntryStatus.Error, ex.Message)]);
Logging.Log.WriteErrorMessage(LOGTAG, "FailedToProcessFile", ex, "Failed to process file {0}", f.Name);
if (ex.IsAbortOrCancelException())
{
m_results.EndTime = DateTime.UtcNow;
m_result.EndTime = DateTime.UtcNow;
throw;
}
}
}
}
m_results.EndTime = DateTime.UtcNow;
m_result.EndTime = DateTime.UtcNow;
// generate a backup error status when any test is failing - except for 'extra' status
// because these problems don't block database rebuilding.
var filtered = from n in m_results.Verifications where n.Value.Any(x => x.Key != TestEntryStatus.Extra) select n;
var filtered = from n in m_result.Verifications where n.Value.Any(x => x.Key != TestEntryStatus.Extra) select n;
if (!filtered.Any())
{
Logging.Log.WriteInformationMessage(LOGTAG, "Test results", "Successfully verified {0} remote files", m_results.VerificationsActualLength);
Logging.Log.WriteInformationMessage(LOGTAG, "Test results", "Successfully verified {0} remote files", m_result.VerificationsActualLength);
}
else
{
Logging.Log.WriteErrorMessage(LOGTAG, "Test results", null, "Verified {0} remote files with {1} problem(s)", m_results.VerificationsActualLength, filtered.Count());
Logging.Log.WriteErrorMessage(LOGTAG, "Test results", null, "Verified {0} remote files with {1} problem(s)", m_result.VerificationsActualLength, filtered.Count());
}
}
@@ -268,8 +268,9 @@ namespace Duplicati.Library.Main.Operation
/// <param name="tf">The path to the downloaded copy of the file.</param>
/// <param name="options">The options to use for the test.</param>
/// <param name="sample_percent">A value between 0 and 1 that indicates how many blocks are tested in a dblock file.</param>
/// <param name="cancellationToken">A cancellation token to cancel the operation.</param>
/// <returns>A task that when awaited returns a key-value pair where the key is the volume name and the value is a list of test results.</returns>
public static async Task<KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>> TestVolumeInternals(LocalTestDatabase db, IRemoteVolume vol, string tf, Options options, double sample_percent)
public static async Task<KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>> TestVolumeInternals(LocalTestDatabase db, IRemoteVolume vol, string tf, Options options, double sample_percent, CancellationToken cancellationToken)
{
var hashsize = HashFactory.HashSizeBytes(options.BlockHashAlgorithm);
var parsedInfo = Volumes.VolumeBase.ParseFilename(vol.Name);
@@ -280,15 +281,15 @@ namespace Duplicati.Library.Main.Operation
case RemoteVolumeType.Files:
//Compare with db and see if all files are accounted for
// with correct file hashes and blocklist hashes
await using (var fl = await db.CreateFilelist(vol.Name).ConfigureAwait(false))
await using (var fl = await db.CreateFilelist(vol.Name, cancellationToken).ConfigureAwait(false))
{
using (var rd = new Volumes.FilesetVolumeReader(parsedInfo.CompressionModule, tf, options))
foreach (var f in rd.Files)
await fl
.Add(f.Path, f.Size, f.Hash, f.Metasize, f.Metahash, f.BlocklistHashes, f.Type, f.Time)
.Add(f.Path, f.Size, f.Hash, f.Metasize, f.Metahash, f.BlocklistHashes, f.Type, f.Time, cancellationToken)
.ConfigureAwait(false);
return new KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>(vol.Name, await fl.Compare().ToListAsync().ConfigureAwait(false));
return new KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>(vol.Name, await fl.Compare(cancellationToken).ToListAsync(cancellationToken: cancellationToken).ConfigureAwait(false));
}
case RemoteVolumeType.Index:
@@ -301,17 +302,17 @@ namespace Duplicati.Library.Main.Operation
foreach (var v in rd.Volumes)
{
blocklinks.Add(new Tuple<string, string, long>(v.Filename, v.Hash, v.Length));
await using (var bl = await db.CreateBlocklist(v.Filename).ConfigureAwait(false))
await using (var bl = await db.CreateBlocklist(v.Filename, cancellationToken).ConfigureAwait(false))
{
foreach (var h in v.Blocks)
await bl
.AddBlock(h.Key, h.Value)
.AddBlock(h.Key, h.Value, cancellationToken)
.ConfigureAwait(false);
combined.AddRange(
await bl
.Compare()
.ToListAsync()
.Compare(cancellationToken)
.ToListAsync(cancellationToken: cancellationToken)
.ConfigureAwait(false)
);
}
@@ -320,17 +321,17 @@ namespace Duplicati.Library.Main.Operation
if (options.IndexfilePolicy == Options.IndexFileStrategy.Full)
{
var hashesPerBlock = options.Blocksize / options.BlockhashSize;
await using (var bl = await db.CreateBlocklistHashList(vol.Name).ConfigureAwait(false))
await using (var bl = await db.CreateBlocklistHashList(vol.Name, cancellationToken).ConfigureAwait(false))
{
foreach (var b in rd.BlockLists)
await bl
.AddBlockHash(b.Hash, b.Length)
.AddBlockHash(b.Hash, b.Length, cancellationToken)
.ConfigureAwait(false);
combined.AddRange(
await bl
.Compare(hashesPerBlock, options.BlockhashSize, options.Blocksize)
.ToListAsync()
.Compare(hashesPerBlock, options.BlockhashSize, options.Blocksize, cancellationToken)
.ToListAsync(cancellationToken: cancellationToken)
.ConfigureAwait(false)
);
}
@@ -338,17 +339,17 @@ namespace Duplicati.Library.Main.Operation
}
// Compare with db and see that all blocklists are listed
await using (var il = await db.CreateIndexlist(vol.Name).ConfigureAwait(false))
await using (var il = await db.CreateIndexlist(vol.Name, cancellationToken).ConfigureAwait(false))
{
foreach (var t in blocklinks)
await il
.AddBlockLink(t.Item1, t.Item2, t.Item3)
.AddBlockLink(t.Item1, t.Item2, t.Item3, cancellationToken)
.ConfigureAwait(false);
combined.AddRange(
await il
.Compare()
.ToArrayAsync()
.Compare(cancellationToken)
.ToArrayAsync(cancellationToken: cancellationToken)
.ConfigureAwait(false)
);
}
@@ -356,13 +357,13 @@ namespace Duplicati.Library.Main.Operation
return new KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>(vol.Name, combined);
case RemoteVolumeType.Blocks:
using (var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm))
await using (var bl = await db.CreateBlocklist(vol.Name).ConfigureAwait(false))
await using (var bl = await db.CreateBlocklist(vol.Name, cancellationToken).ConfigureAwait(false))
using (var rd = new Volumes.BlockVolumeReader(parsedInfo.CompressionModule, tf, options))
{
//Verify that all blocks are in the file
foreach (var b in rd.Blocks)
await bl
.AddBlock(b.Key, b.Value)
.AddBlock(b.Key, b.Value, cancellationToken)
.ConfigureAwait(false);
//Select random blocks and verify their hashes match the filename and size
@@ -392,8 +393,8 @@ namespace Duplicati.Library.Main.Operation
vol.Name,
changes.Union(
await bl
.Compare()
.ToListAsync()
.Compare(cancellationToken)
.ToListAsync(cancellationToken: cancellationToken)
.ConfigureAwait(false)
)
);
@@ -407,14 +408,14 @@ namespace Duplicati.Library.Main.Operation
private async Task ReplaceFaultyIndexFilesAsync(List<IRemoteVolume> faultyIndexFiles, IBackendManager backendManager, LocalTestDatabase db, CancellationToken cancellationToken)
{
await using var repairdb =
await LocalRepairDatabase.CreateAsync(db)
await LocalRepairDatabase.CreateAsync(db, null, cancellationToken)
.ConfigureAwait(false);
foreach (var vol in faultyIndexFiles)
{
if (!await m_results.TaskControl.ProgressRendevouz().ConfigureAwait(false))
if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
{
m_results.EndTime = DateTime.UtcNow;
m_result.EndTime = DateTime.UtcNow;
return;
}
@@ -429,8 +430,8 @@ namespace Duplicati.Library.Main.Operation
}
else
{
await backendManager.DeleteAsync(vol.Name, vol.Size, true, m_results.TaskControl.ProgressToken).ConfigureAwait(false);
await backendManager.WaitForEmptyAsync(repairdb, m_results.TaskControl.ProgressToken).ConfigureAwait(false);
await backendManager.DeleteAsync(vol.Name, vol.Size, true, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
await backendManager.WaitForEmptyAsync(repairdb, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
await repairdb.Transaction
.CommitAsync("ReplaceFaultyIndexFileCommit")
.ConfigureAwait(false);
Duplicati/Library/Main/Operation/VacuumHandler.cs
+2 -2
View File
@@ -38,10 +38,10 @@ namespace Duplicati.Library.Main.Operation
public virtual async Task RunAsync()
{
await using var db =
await Database.LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Vacuum", false, m_options.SqlitePageCache)
await Database.LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Vacuum", false, m_options.SqlitePageCache, null, m_result.TaskControl.ProgressToken)
.ConfigureAwait(false);
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Vacuum_Running);
await db.Vacuum().ConfigureAwait(false);
await db.Vacuum(m_result.TaskControl.ProgressToken).ConfigureAwait(false);
m_result.EndTime = DateTime.UtcNow;
}
}
Duplicati/Library/Main/Utility.cs
+13 -9
View File
@@ -28,6 +28,7 @@ using System.Threading.Tasks;
using Newtonsoft.Json;
using Duplicati.Library.Main.Database;
using Duplicati.Library.Utility;
using System.Threading;
namespace Duplicati.Library.Main
{
@@ -103,11 +104,12 @@ namespace Duplicati.Library.Main
/// </summary>
/// <param name="db">The database to read from.</param>
/// <param name="options">The options to update.</param>
/// <param name="cancellationToken">A cancellation token to cancel the operation.</param>
/// <returns>A task that completes when the options have been updated.</returns>
internal static async Task UpdateOptionsFromDb(LocalDatabase db, Options options)
internal static async Task UpdateOptionsFromDb(LocalDatabase db, Options options, CancellationToken cancellationToken)
{
string n = null;
var opts = await db.GetDbOptions().ConfigureAwait(false);
var opts = await db.GetDbOptions(cancellationToken).ConfigureAwait(false);
if (opts.ContainsKey("blocksize") && (!options.RawOptions.TryGetValue("blocksize", out n) || string.IsNullOrEmpty(n)))
options.RawOptions["blocksize"] = opts["blocksize"] + "b";
@@ -121,11 +123,12 @@ namespace Duplicati.Library.Main
/// Checks if the database contains options that need to be verified, such as the blocksize.
/// </summary>
/// <param name="db">The database to check.</param>
/// <param name="cancellationToken">A cancellation token to cancel the operation.</param>
/// <returns>A task that when awaited returns <c>true</c> if the database contains options that need to be verified; <c>false</c> otherwise.</returns>
internal static async Task<bool> ContainsOptionsForVerification(LocalDatabase db)
internal static async Task<bool> ContainsOptionsForVerification(LocalDatabase db, CancellationToken cancellationToken)
{
var opts = await db.GetDbOptions().ConfigureAwait(false);
await db.Transaction.CommitAsync().ConfigureAwait(false);
var opts = await db.GetDbOptions(cancellationToken).ConfigureAwait(false);
await db.Transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
return new[] {
"blocksize",
"blockhash",
@@ -140,8 +143,9 @@ namespace Duplicati.Library.Main
/// </summary>
/// <param name="db">The database to check.</param>
/// <param name="options">The options to verify.</param>
/// <param name="cancellationToken">A cancellation token to cancel the operation.</param>
/// <returns>A task that completes when the options have been verified and the database has been updated if needed.</returns>
internal static async Task VerifyOptionsAndUpdateDatabase(LocalDatabase db, Options options)
internal static async Task VerifyOptionsAndUpdateDatabase(LocalDatabase db, Options options, CancellationToken cancellationToken)
{
var newDict = new Dictionary<string, string>
{
@@ -149,7 +153,7 @@ namespace Duplicati.Library.Main
{ "blockhash", options.BlockHashAlgorithm },
{ "filehash", options.FileHashAlgorithm }
};
var opts = await db.GetDbOptions().ConfigureAwait(false);
var opts = await db.GetDbOptions(cancellationToken).ConfigureAwait(false);
if (options.NoEncryption)
{
@@ -199,7 +203,7 @@ namespace Duplicati.Library.Main
}
//Extra sanity check
if (await db.GetBlocksLargerThan(options.Blocksize).ConfigureAwait(false) > 0)
if (await db.GetBlocksLargerThan(options.Blocksize, cancellationToken).ConfigureAwait(false) > 0)
throw new Duplicati.Library.Interface.UserInformationException("You have attempted to change the block-size on an existing backup, which is not supported. Please configure a new clean backup if you want to change the block-size.", "BlockSizeChangeNotSupported");
if (needsUpdate)
@@ -209,7 +213,7 @@ namespace Duplicati.Library.Main
if (!newDict.ContainsKey(k.Key))
newDict[k.Key] = k.Value;
await db.SetDbOptions(newDict).ConfigureAwait(false);
await db.SetDbOptions(newDict, cancellationToken).ConfigureAwait(false);
}
}
}