Extended tests and fixed a case where a left-over upload could cause errors.
This commit is contained in:
@@ -742,9 +742,9 @@ AND Fileset.ID NOT IN
|
||||
var opts = GetDbOptions();
|
||||
|
||||
if (value)
|
||||
opts["repair-in-progress"] = "true";
|
||||
opts[key] = "true";
|
||||
else
|
||||
opts.Remove("repair-in-progress");
|
||||
opts.Remove(key);
|
||||
|
||||
SetDbOptions(opts);
|
||||
}
|
||||
|
||||
@@ -77,15 +77,13 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "PreviousBackupFilelistUpload", "Uploading filelist from previous interrupted backup");
|
||||
|
||||
var incompleteSet = incompleteFilesets.Last();
|
||||
var badIds = from n in incompleteFilesets select n.Key;
|
||||
var badIds = incompleteFilesets.Select(n => n.Key);
|
||||
|
||||
var prevs = (from n in await database.GetFilesetTimesAsync()
|
||||
where
|
||||
n.Key < incompleteSet.Key
|
||||
&&
|
||||
!badIds.Contains(n.Key)
|
||||
orderby n.Key
|
||||
select n.Key).ToArray();
|
||||
var prevs = (await database.GetFilesetTimesAsync())
|
||||
.Where(n => n.Key < incompleteSet.Key && !badIds.Contains(n.Key))
|
||||
.OrderBy(n => n.Key)
|
||||
.Select(n => n.Key)
|
||||
.ToArray();
|
||||
|
||||
var prevId = prevs.Length == 0 ? -1 : prevs.Last();
|
||||
|
||||
|
||||
@@ -344,7 +344,7 @@ namespace Duplicati.Library.Main.Operation
|
||||
}
|
||||
else
|
||||
{
|
||||
await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, new string[] { lastTempFilelist }, logErrors: false, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean).ConfigureAwait(false);
|
||||
await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist], [], logErrors: false, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (RemoteListVerificationException ex)
|
||||
@@ -364,7 +364,7 @@ namespace Duplicati.Library.Main.Operation
|
||||
result.SetDatabase(database);
|
||||
|
||||
Log.WriteInformationMessage(LOGTAG, "BackendCleanupFinished", "Backend cleanup finished, retrying verification");
|
||||
await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, new string[] { lastTempFilelist }, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false);
|
||||
await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist], [], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
throw;
|
||||
@@ -504,11 +504,11 @@ namespace Duplicati.Library.Main.Operation
|
||||
}
|
||||
}
|
||||
|
||||
private async Task PostBackupVerification(string currentFilelistVolume, IBackendManager backendManager)
|
||||
private async Task PostBackupVerification(string currentFilelistVolume, string previousTemporaryFilelist, IBackendManager backendManager)
|
||||
{
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupVerify);
|
||||
using (new Logging.Timer(LOGTAG, "AfterBackupVerify", "AfterBackupVerify"))
|
||||
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, m_result.BackendWriter, new string[] { currentFilelistVolume }, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false);
|
||||
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, m_result.BackendWriter, [currentFilelistVolume], [previousTemporaryFilelist], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false);
|
||||
await backendManager.WaitForEmptyAsync(m_database, m_transaction, m_taskReader.ProgressToken);
|
||||
|
||||
long remoteVolumeCount = m_database.GetRemoteVolumes().LongCount(x => x.State == RemoteVolumeState.Verified);
|
||||
@@ -752,7 +752,7 @@ namespace Duplicati.Library.Main.Operation
|
||||
if (m_options.NoBackendverification)
|
||||
await UpdateStorageStatsFromDatabase(m_result, m_database, m_options, backendManager, m_taskReader.ProgressToken).ConfigureAwait(false);
|
||||
else
|
||||
await PostBackupVerification(filesetvolume.RemoteFilename, backendManager).ConfigureAwait(false);
|
||||
await PostBackupVerification(filesetvolume.RemoteFilename, lastTempFilelist, backendManager).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -108,7 +108,7 @@ namespace Duplicati.Library.Main.Operation
|
||||
{
|
||||
LocalBackupDatabase backupDatabase = new LocalBackupDatabase(database, options);
|
||||
IEnumerable<string> protectedFiles = backupDatabase.GetTemporaryFilelistVolumeNames(latestVolumesOnly, transaction);
|
||||
await VerifyRemoteList(backend, options, database, backendWriter, protectedFiles, logErrors: true, verifyMode: verifyMode).ConfigureAwait(false);
|
||||
await VerifyRemoteList(backend, options, database, backendWriter, protectedFiles, [], logErrors: true, verifyMode: verifyMode).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -121,11 +121,12 @@ namespace Duplicati.Library.Main.Operation
|
||||
/// <param name="database">The database to compare with</param>
|
||||
/// <param name="log">The log instance to use</param>
|
||||
/// <param name="protectedFiles">Filenames that should be exempted from deletion</param>
|
||||
/// <param name="strictExcemptFiles">Filenames that should be exempted from strict verification</param>
|
||||
/// <param name="verifyMode">The mode to use for verification</param>
|
||||
/// <param name="logErrors">Disable the logging of errors to prevent spamming the log; exceptions will be thrown regardless</param>
|
||||
public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable<string> protectedFiles, bool logErrors, VerifyMode verifyMode)
|
||||
public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable<string> protectedFiles, IEnumerable<string> strictExcemptFiles, bool logErrors, VerifyMode verifyMode)
|
||||
{
|
||||
var tp = await RemoteListAnalysis(backend, options, database, log, protectedFiles, verifyMode).ConfigureAwait(false);
|
||||
var tp = await RemoteListAnalysis(backend, options, database, log, protectedFiles, strictExcemptFiles, verifyMode).ConfigureAwait(false);
|
||||
long extraCount = 0;
|
||||
long missingCount = 0;
|
||||
|
||||
@@ -232,9 +233,10 @@ namespace Duplicati.Library.Main.Operation
|
||||
/// <param name="options">The options used</param>
|
||||
/// <param name="database">The database to compare with</param>
|
||||
/// <param name="protectedFiles">Filenames that should be exempted from deletion</param>
|
||||
/// <param name="strictExcemptFiles">Filenames that should be exempted from strict verification</param>
|
||||
/// <param name="verifyMode">The mode to use for verification</param>
|
||||
/// <returns>The result of the analysis</returns>
|
||||
public static async Task<RemoteAnalysisResult> RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IBackendWriter log, IEnumerable<string> protectedFiles, VerifyMode verifyMode)
|
||||
public static async Task<RemoteAnalysisResult> RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IBackendWriter log, IEnumerable<string> protectedFiles, IEnumerable<string> strictExcemptFiles, VerifyMode verifyMode)
|
||||
{
|
||||
// If the last operation completed, no cleanup should be required
|
||||
if (verifyMode == VerifyMode.VerifyAndClean && !database.UncleanShutdown)
|
||||
@@ -246,7 +248,8 @@ namespace Duplicati.Library.Main.Operation
|
||||
|
||||
var rawlist = await backendManager.ListAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
var lookup = new Dictionary<string, Volumes.IParsedVolume>();
|
||||
protectedFiles = protectedFiles ?? Enumerable.Empty<string>();
|
||||
protectedFiles = protectedFiles ?? [];
|
||||
strictExcemptFiles = strictExcemptFiles ?? [];
|
||||
|
||||
var remotelist = (from n in rawlist
|
||||
let p = Volumes.VolumeBase.ParseFilename(n)
|
||||
@@ -284,7 +287,8 @@ namespace Duplicati.Library.Main.Operation
|
||||
|
||||
var missing = new List<RemoteVolumeEntry>();
|
||||
var missingHash = new List<Tuple<long, RemoteVolumeEntry>>();
|
||||
var cleanupRemovedRemoteVolumes = new HashSet<string>();
|
||||
var temporaryAndDeletedVolumes = new HashSet<string>();
|
||||
var missingUploadingVolumes = new HashSet<string>();
|
||||
|
||||
foreach (var e in database.DuplicateRemoteVolumes())
|
||||
{
|
||||
@@ -297,8 +301,7 @@ namespace Duplicati.Library.Main.Operation
|
||||
var locallist = database.GetRemoteVolumes();
|
||||
foreach (var i in locallist)
|
||||
{
|
||||
Volumes.IParsedVolume r;
|
||||
var remoteFound = lookup.TryGetValue(i.Name, out r);
|
||||
var remoteFound = lookup.TryGetValue(i.Name, out var r);
|
||||
var correctSize = remoteFound && i.Size >= 0 && (i.Size == r.File.Size || r.File.Size < 0);
|
||||
|
||||
lookup.Remove(i.Name);
|
||||
@@ -336,7 +339,7 @@ namespace Duplicati.Library.Main.Operation
|
||||
else
|
||||
{
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "RemoteUnwantedMissingFile", "removing file listed as {0}: {1}", i.State, i.Name);
|
||||
cleanupRemovedRemoteVolumes.Add(i.Name);
|
||||
temporaryAndDeletedVolumes.Add(i.Name);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -354,15 +357,15 @@ namespace Duplicati.Library.Main.Operation
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name);
|
||||
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null);
|
||||
}
|
||||
else if (verifyMode == VerifyMode.VerifyStrict)
|
||||
else if (verifyMode == VerifyMode.VerifyStrict && !strictExcemptFiles.Any(pf => pf == i.Name))
|
||||
{
|
||||
Logging.Log.WriteErrorMessage(LOGTAG, "UnexpectedUploadingFile", null, "Unexpected file in uploading state: {0}", i.Name);
|
||||
throw new RemoteListVerificationException($"The remote volume {i.Name} is in uploading state, failing", "UnexpectedUploadingFile");
|
||||
throw new RemoteListVerificationException($"The missing remote volume {i.Name} is in uploading state and strict mode is on", "UnexpectedUploadingFile");
|
||||
}
|
||||
else
|
||||
{
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "SchedulingMissingFileForDelete", "scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name);
|
||||
cleanupRemovedRemoteVolumes.Add(i.Name);
|
||||
missingUploadingVolumes.Add(i.Name);
|
||||
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null);
|
||||
}
|
||||
}
|
||||
@@ -372,10 +375,10 @@ namespace Duplicati.Library.Main.Operation
|
||||
{
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name);
|
||||
}
|
||||
else if (verifyMode == VerifyMode.VerifyStrict)
|
||||
else if (verifyMode == VerifyMode.VerifyStrict && !strictExcemptFiles.Any(pf => pf == i.Name))
|
||||
{
|
||||
Logging.Log.WriteErrorMessage(LOGTAG, "UnexpectedUploadingFile", null, "Unexpected file in uploading state: {0}", i.Name);
|
||||
throw new RemoteListVerificationException($"The remote volume {i.Name} is in uploading state, failing", "UnexpectedUploadingFile");
|
||||
throw new RemoteListVerificationException($"The remote volume {i.Name} is in uploading state, partially uploaded, and strict mode is on", "UnexpectedUploadingFile");
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -415,11 +418,11 @@ namespace Duplicati.Library.Main.Operation
|
||||
}
|
||||
|
||||
// cleanup deleted volumes in DB en block
|
||||
if (verifyMode == VerifyMode.VerifyStrict && cleanupRemovedRemoteVolumes.Count > 0)
|
||||
throw new RemoteListVerificationException($"The remote volumes {string.Join(", ", cleanupRemovedRemoteVolumes)} are supposed to be deleted, but strict mode is on. Try running the \"repair\" command", "DeleteDuringStrictMode");
|
||||
if (verifyMode == VerifyMode.VerifyStrict && missingUploadingVolumes.Count > 0)
|
||||
throw new RemoteListVerificationException($"The remote volumes {string.Join(", ", missingUploadingVolumes)} are supposed to be deleted, but strict mode is on. Try running the \"repair\" command", "DeleteDuringStrictMode");
|
||||
else if (verifyMode == VerifyMode.VerifyAndClean)
|
||||
{
|
||||
database.RemoveRemoteVolumes(cleanupRemovedRemoteVolumes, null);
|
||||
database.RemoveRemoteVolumes(missingUploadingVolumes.Concat(temporaryAndDeletedVolumes), null);
|
||||
// Clear the flag after we have cleaned up
|
||||
if (!options.Dryrun)
|
||||
database.UncleanShutdown = false;
|
||||
|
||||
@@ -65,7 +65,7 @@ namespace Duplicati.Library.Main.Operation
|
||||
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsInDatabase", "No broken filesets found in database, checking for missing remote files");
|
||||
|
||||
var remotestate = FilelistProcessor.RemoteListAnalysis(backendManager, options, db, result.BackendWriter, null, FilelistProcessor.VerifyMode.VerifyOnly).Await();
|
||||
var remotestate = FilelistProcessor.RemoteListAnalysis(backendManager, options, db, result.BackendWriter, null, null, FilelistProcessor.VerifyMode.VerifyOnly).Await();
|
||||
if (!remotestate.ParsedVolumes.Any())
|
||||
throw new UserInformationException("No remote volumes were found, refusing purge", "CannotPurgeWithNoRemoteVolumes");
|
||||
|
||||
|
||||
@@ -91,7 +91,7 @@ namespace Duplicati.Library.Main.Operation
|
||||
if (m_options.NoBackendverification)
|
||||
FilelistProcessor.VerifyLocalList(backendManager, db, cancellationToken).Await();
|
||||
else
|
||||
FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean).Await();
|
||||
FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, null, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean).Await();
|
||||
}
|
||||
|
||||
var filesets = db.FilesetTimes.OrderByDescending(x => x.Value).ToArray();
|
||||
|
||||
@@ -134,7 +134,7 @@ namespace Duplicati.Library.Main.Operation
|
||||
if (db.RepairInProgress)
|
||||
throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsInRepairState");
|
||||
|
||||
var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, m_result.BackendWriter, null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false);
|
||||
var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, m_result.BackendWriter, null, null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false);
|
||||
var buffer = new byte[m_options.Blocksize];
|
||||
var hashsize = HashFactory.HashSizeBytes(m_options.BlockHashAlgorithm);
|
||||
|
||||
|
||||
@@ -887,102 +887,25 @@ namespace Duplicati.UnitTest
|
||||
[Category("Disruption")]
|
||||
public void TestFailedDblockUploadWithOperationCancellation()
|
||||
{
|
||||
var testopts = TestOptions;
|
||||
testopts["number-of-retries"] = "1";
|
||||
testopts["dblock-size"] = "10mb";
|
||||
|
||||
// Make a base backup
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
||||
{
|
||||
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER });
|
||||
Assert.AreEqual(0, backupResults.Errors.Count());
|
||||
Assert.AreEqual(0, backupResults.Warnings.Count());
|
||||
}
|
||||
|
||||
// Make a new backup that fails uploading a dblock file
|
||||
ModifySourceFiles();
|
||||
|
||||
// Deterministic error backend
|
||||
Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend());
|
||||
var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER;
|
||||
var uploadCount = 0;
|
||||
var failedFile = string.Empty;
|
||||
|
||||
// Fail the dlist upload
|
||||
DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) =>
|
||||
{
|
||||
if (action == DeterministicErrorBackend.BackendAction.PutBefore && remotename.Contains(".dblock."))
|
||||
{
|
||||
// Fail once with a cancellation
|
||||
if (Interlocked.Increment(ref uploadCount) == 1)
|
||||
{
|
||||
failedFile = remotename;
|
||||
throw new OperationCanceledException();
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
};
|
||||
|
||||
using (var c = new Library.Main.Controller(failtarget, testopts, null))
|
||||
{
|
||||
var sink = new LogSink();
|
||||
c.AppendSink(sink);
|
||||
var res = c.Backup(new string[] { DATAFOLDER });
|
||||
if (uploadCount == 0)
|
||||
Assert.Fail("Upload count was not incremented");
|
||||
|
||||
Assert.AreEqual(2, c.List().Filesets.Count());
|
||||
Assert.AreEqual(7 * 2 + 1, sink.CompletedFiles.Count); // 7 completed (dblock + dindex) + 1 dlist
|
||||
Assert.AreEqual(7 * 2 + 2, sink.StartedFiles.Count); // 1 retry
|
||||
Assert.AreEqual(1, res.BackendStatistics.RetryAttempts);
|
||||
}
|
||||
|
||||
ModifySourceFiles();
|
||||
// Create a regular backup
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
||||
{
|
||||
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER });
|
||||
Assert.AreEqual(0, backupResults.Errors.Count());
|
||||
Assert.AreEqual(0, backupResults.Warnings.Count());
|
||||
}
|
||||
|
||||
// Verify that all is in order
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null))
|
||||
{
|
||||
var r = c.Test(long.MaxValue);
|
||||
Assert.AreEqual(0, r.Errors.Count());
|
||||
Assert.AreEqual(0, r.Warnings.Count());
|
||||
Assert.IsFalse(r.Verifications.Any(p => p.Value.Any()));
|
||||
}
|
||||
|
||||
// Test that we can recreate
|
||||
var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite");
|
||||
if (File.Exists(recreatedDatabaseFile))
|
||||
File.Delete(recreatedDatabaseFile);
|
||||
|
||||
testopts["dbpath"] = recreatedDatabaseFile;
|
||||
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
||||
{
|
||||
IRepairResults repairResults = c.Repair();
|
||||
Assert.AreEqual(0, repairResults.Errors.Count());
|
||||
Assert.AreEqual(0, repairResults.Warnings.Count());
|
||||
}
|
||||
|
||||
// Check that we have 3 versions
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
||||
{
|
||||
IListResults listResults = c.List();
|
||||
Assert.AreEqual(0, listResults.Errors.Count());
|
||||
Assert.AreEqual(0, listResults.Warnings.Count());
|
||||
Assert.AreEqual(3, listResults.Filesets.Count());
|
||||
}
|
||||
TestFailedUploadWithOperationCancellation(".dblock.");
|
||||
}
|
||||
|
||||
|
||||
[Test]
|
||||
[Category("Disruption")]
|
||||
public void TestFailedDindexUploadWithOperationCancellation()
|
||||
{
|
||||
TestFailedUploadWithOperationCancellation(".dindex.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
[Category("Disruption")]
|
||||
public void TestFailedDlistUploadWithOperationCancellation()
|
||||
{
|
||||
TestFailedUploadWithOperationCancellation(".dlist.");
|
||||
}
|
||||
|
||||
private void TestFailedUploadWithOperationCancellation(string filefragment)
|
||||
{
|
||||
var testopts = TestOptions;
|
||||
testopts["number-of-retries"] = "1";
|
||||
@@ -1003,19 +926,16 @@ namespace Duplicati.UnitTest
|
||||
Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend());
|
||||
var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER;
|
||||
var uploadCount = 0;
|
||||
var failedFile = string.Empty;
|
||||
var failstep = DeterministicErrorBackend.BackendAction.PutBefore;
|
||||
|
||||
// Fail the dlist upload
|
||||
// Fail the upload before
|
||||
DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) =>
|
||||
{
|
||||
if (action == DeterministicErrorBackend.BackendAction.PutBefore && remotename.Contains(".dindex."))
|
||||
if (action == failstep && remotename.Contains(filefragment))
|
||||
{
|
||||
// Fail once with a cancellation
|
||||
if (Interlocked.Increment(ref uploadCount) == 1)
|
||||
{
|
||||
failedFile = remotename;
|
||||
throw new OperationCanceledException();
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
@@ -1030,6 +950,27 @@ namespace Duplicati.UnitTest
|
||||
Assert.Fail("Upload count was not incremented");
|
||||
|
||||
Assert.AreEqual(2, c.List().Filesets.Count());
|
||||
Assert.AreEqual(2, Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).Count());
|
||||
Assert.AreEqual(7 * 2 + 1, sink.CompletedFiles.Count); // 7 completed (dblock + dindex) + 1 dlist
|
||||
Assert.AreEqual(7 * 2 + 2, sink.StartedFiles.Count); // 1 retry
|
||||
Assert.AreEqual(sink.CompletedFiles.Count(x => x.Contains(".dblock.")), sink.CompletedFiles.Count(x => x.Contains(".dindex.")));
|
||||
Assert.AreEqual(1, res.BackendStatistics.RetryAttempts);
|
||||
}
|
||||
|
||||
uploadCount = 0;
|
||||
failstep = DeterministicErrorBackend.BackendAction.PutAfter;
|
||||
|
||||
ModifySourceFiles();
|
||||
using (var c = new Library.Main.Controller(failtarget, testopts, null))
|
||||
{
|
||||
var sink = new LogSink();
|
||||
c.AppendSink(sink);
|
||||
var res = c.Backup(new string[] { DATAFOLDER });
|
||||
if (uploadCount == 0)
|
||||
Assert.Fail("Upload count was not incremented");
|
||||
|
||||
Assert.AreEqual(3, c.List().Filesets.Count());
|
||||
Assert.AreEqual(3, Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).Count());
|
||||
Assert.AreEqual(7 * 2 + 1, sink.CompletedFiles.Count); // 7 completed (dblock + dindex) + 1 dlist
|
||||
Assert.AreEqual(7 * 2 + 2, sink.StartedFiles.Count); // 1 retry
|
||||
Assert.AreEqual(sink.CompletedFiles.Count(x => x.Contains(".dblock.")), sink.CompletedFiles.Count(x => x.Contains(".dindex.")));
|
||||
@@ -1074,96 +1015,7 @@ namespace Duplicati.UnitTest
|
||||
IListResults listResults = c.List();
|
||||
Assert.AreEqual(0, listResults.Errors.Count());
|
||||
Assert.AreEqual(0, listResults.Warnings.Count());
|
||||
Assert.AreEqual(3, listResults.Filesets.Count());
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
[Category("Disruption")]
|
||||
public void TestFailedDlistUploadWithOperationCancellation()
|
||||
{
|
||||
var testopts = TestOptions;
|
||||
testopts["number-of-retries"] = "1";
|
||||
testopts["dblock-size"] = "10mb";
|
||||
|
||||
// Make a base backup
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
||||
{
|
||||
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER });
|
||||
Assert.AreEqual(0, backupResults.Errors.Count());
|
||||
Assert.AreEqual(0, backupResults.Warnings.Count());
|
||||
}
|
||||
|
||||
// Make a new backup that fails uploading a dblock file
|
||||
ModifySourceFiles();
|
||||
|
||||
// Deterministic error backend
|
||||
Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend());
|
||||
var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER;
|
||||
var uploadCount = 0;
|
||||
|
||||
// Fail the dlist upload
|
||||
DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) =>
|
||||
{
|
||||
if (action == DeterministicErrorBackend.BackendAction.PutBefore && remotename.Contains(".dlist."))
|
||||
{
|
||||
if (Interlocked.Increment(ref uploadCount) == 1)
|
||||
throw new OperationCanceledException();
|
||||
}
|
||||
|
||||
return false;
|
||||
};
|
||||
|
||||
using (var c = new Library.Main.Controller(failtarget, testopts, null))
|
||||
{
|
||||
var res = c.Backup(new string[] { DATAFOLDER });
|
||||
var dlistCount = Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).Count();
|
||||
if (uploadCount == 0)
|
||||
Assert.Fail("Upload count was not incremented");
|
||||
Assert.AreEqual(2, c.List().Filesets.Count());
|
||||
Assert.AreEqual(2, dlistCount);
|
||||
Assert.AreEqual(1, res.BackendStatistics.RetryAttempts);
|
||||
}
|
||||
|
||||
ModifySourceFiles();
|
||||
// Create a regular backup
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
||||
{
|
||||
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER });
|
||||
Assert.AreEqual(0, backupResults.Errors.Count());
|
||||
Assert.AreEqual(0, backupResults.Warnings.Count());
|
||||
}
|
||||
|
||||
// Verify that all is in order
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null))
|
||||
{
|
||||
var r = c.Test(long.MaxValue);
|
||||
Assert.AreEqual(0, r.Errors.Count());
|
||||
Assert.AreEqual(0, r.Warnings.Count());
|
||||
Assert.IsFalse(r.Verifications.Any(p => p.Value.Any()));
|
||||
}
|
||||
|
||||
// Test that we can recreate
|
||||
var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite");
|
||||
if (File.Exists(recreatedDatabaseFile))
|
||||
File.Delete(recreatedDatabaseFile);
|
||||
|
||||
testopts["dbpath"] = recreatedDatabaseFile;
|
||||
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
||||
{
|
||||
IRepairResults repairResults = c.Repair();
|
||||
Assert.AreEqual(0, repairResults.Errors.Count());
|
||||
Assert.AreEqual(0, repairResults.Warnings.Count());
|
||||
}
|
||||
|
||||
// Check that we have 3 versions
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
||||
{
|
||||
IListResults listResults = c.List();
|
||||
Assert.AreEqual(0, listResults.Errors.Count());
|
||||
Assert.AreEqual(0, listResults.Warnings.Count());
|
||||
Assert.AreEqual(3, listResults.Filesets.Count());
|
||||
Assert.AreEqual(4, listResults.Filesets.Count());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user