From 655f5ef63575925b5d1dcf1a9230310b2c556c69 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 20 Feb 2025 11:00:28 +0100 Subject: [PATCH] Added interuption test This adds a test with a failed upload. The failed upload is done in parallel, so (at least) one of the uploads succeeeds and one upload fails. The backup then errors, but a subsequent backup continues without errors and creates a synthetic filelist. Also refactored the DeterministicErrorBackend to not use magic strings. And made it actually deterministic. --- Duplicati/UnitTest/CompactDisruptionTests.cs | 24 ++--- .../UnitTest/DeterministicErrorBackend.cs | 44 +++++--- Duplicati/UnitTest/DisruptionTests.cs | 100 ++++++++++++++++++ Duplicati/UnitTest/IssueTests.cs | 8 +- 4 files changed, 143 insertions(+), 33 deletions(-) diff --git a/Duplicati/UnitTest/CompactDisruptionTests.cs b/Duplicati/UnitTest/CompactDisruptionTests.cs index 38da492cf..8a8f9328c 100644 --- a/Duplicati/UnitTest/CompactDisruptionTests.cs +++ b/Duplicati/UnitTest/CompactDisruptionTests.cs @@ -135,13 +135,13 @@ namespace Duplicati.UnitTest target = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; // Fail the compact after the first dblock put is completed bool firstPutCompleted = false; - DeterministicErrorBackend.ErrorGenerator = (string action, string remotename) => + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => { - if (firstPutCompleted && (action == "get_0" || action == "get_1")) + if (firstPutCompleted && action.IsGetOperation) { return true; } - if (action == "put_1" || action == "put_async") + if (action == DeterministicErrorBackend.BackendAction.PutAfter) { firstPutCompleted = true; } @@ -250,13 +250,13 @@ namespace Duplicati.UnitTest target = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; // Fail the compact after the first dblock put is completed bool firstPutCompleted = false; - DeterministicErrorBackend.ErrorGenerator = (string action, string remotename) => + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => { - if (firstPutCompleted && (action == "get_0" || action == "get_1")) + if (firstPutCompleted && action.IsGetOperation) { return true; } - if (action == "put_1" || action == "put_async") + if (action == DeterministicErrorBackend.BackendAction.PutAfter) { firstPutCompleted = true; } @@ -382,13 +382,13 @@ namespace Duplicati.UnitTest target = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; // Fail the compact after the first dblock put is completed bool firstPutCompleted = false; - DeterministicErrorBackend.ErrorGenerator = (string action, string remotename) => + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => { - if (firstPutCompleted && (action == "get_0" || action == "get_1")) + if (firstPutCompleted && action.IsGetOperation) { return true; } - if (action == "put_1" || action == "put_async") + if (action == DeterministicErrorBackend.BackendAction.PutAfter) { firstPutCompleted = true; } @@ -523,13 +523,13 @@ namespace Duplicati.UnitTest target = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; // Fail the compact after the first dblock put is completed bool firstPutCompleted = false; - DeterministicErrorBackend.ErrorGenerator = (string action, string remotename) => + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => { - if (firstPutCompleted && (action == "get_0" || action == "get_1")) + if (firstPutCompleted && action.IsGetOperation) { return true; } - if (action == "put_1" || action == "put_async") + if (action == DeterministicErrorBackend.BackendAction.PutAfter) { firstPutCompleted = true; } diff --git a/Duplicati/UnitTest/DeterministicErrorBackend.cs b/Duplicati/UnitTest/DeterministicErrorBackend.cs index 34a62ad7e..8551fc904 100644 --- a/Duplicati/UnitTest/DeterministicErrorBackend.cs +++ b/Duplicati/UnitTest/DeterministicErrorBackend.cs @@ -31,14 +31,26 @@ namespace Duplicati.UnitTest { public class DeterministicErrorBackend : IBackend, IStreamingBackend { + public sealed record BackendAction(string Code) + { + public static readonly BackendAction GetBefore = new BackendAction("get_0"); + public static readonly BackendAction GetAfter = new BackendAction("get_1"); + public static readonly BackendAction PutBefore = new BackendAction("put_0"); + public static readonly BackendAction PutAfter = new BackendAction("put_1"); + public static readonly BackendAction DeleteBefore = new BackendAction("delete_0"); + public static readonly BackendAction DeleteAfter = new BackendAction("delete_1"); + + public bool IsGetOperation => this == GetBefore || this == GetAfter; + public bool IsPutOperation => this == PutBefore || this == PutAfter; + public bool IsDeleteOperation => this == DeleteBefore || this == DeleteAfter; + } + public class DeterministicErrorBackendException(string message) : Exception(message) { }; static DeterministicErrorBackend() { WrappedBackend = "file"; } - private static readonly Random random = new Random(42); - // return true to throw exception, parameters: (action, remotename) - public static Func ErrorGenerator = null; + public static Func ErrorGenerator = null; public static string WrappedBackend { get; set; } @@ -54,7 +66,7 @@ namespace Duplicati.UnitTest m_backend = (IStreamingBackend)Library.DynamicLoader.BackendLoader.GetBackend(u, options); } - private void ThrowError(string action, string remotename) + private void ThrowError(BackendAction action, string remotename) { if (ErrorGenerator != null && ErrorGenerator(action, remotename)) { @@ -64,17 +76,15 @@ namespace Duplicati.UnitTest #region IStreamingBackend implementation public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { - var uploadError = random.NextDouble() > 0.9; - - using (var f = new Library.Utility.ProgressReportingStream(stream, x => { if (uploadError && stream.Position > stream.Length / 2) throw new DeterministicErrorBackendException("Random upload failure"); })) - await m_backend.PutAsync(remotename, f, cancelToken).ConfigureAwait(false); - ThrowError("put_async", remotename); + ThrowError(BackendAction.PutBefore, remotename); + await m_backend.PutAsync(remotename, stream, cancelToken).ConfigureAwait(false); + ThrowError(BackendAction.PutAfter, remotename); } public async Task GetAsync(string remotename, Stream stream, CancellationToken cancellationToken) { - ThrowError("get_0", remotename); + ThrowError(BackendAction.GetBefore, remotename); await m_backend.GetAsync(remotename, stream, cancellationToken).ConfigureAwait(false); - ThrowError("get_1", remotename); + ThrowError(BackendAction.GetAfter, remotename); } #endregion @@ -85,21 +95,21 @@ namespace Duplicati.UnitTest } public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) { - ThrowError("put_0", remotename); + ThrowError(BackendAction.PutBefore, remotename); await m_backend.PutAsync(remotename, filename, cancelToken).ConfigureAwait(false); - ThrowError("put_1", remotename); + ThrowError(BackendAction.PutAfter, remotename); } public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken) { - ThrowError("get_0", remotename); + ThrowError(BackendAction.GetBefore, remotename); await m_backend.GetAsync(remotename, filename, cancelToken).ConfigureAwait(false); - ThrowError("get_1", remotename); + ThrowError(BackendAction.GetAfter, remotename); } public async Task DeleteAsync(string remotename, CancellationToken cancelToken) { - ThrowError("delete_0", remotename); + ThrowError(BackendAction.DeleteBefore, remotename); await m_backend.DeleteAsync(remotename, cancelToken).ConfigureAwait(false); - ThrowError("delete_1", remotename); + ThrowError(BackendAction.DeleteAfter, remotename); } public Task TestAsync(CancellationToken cancelToken) { diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs index c23a15124..16fac2c5d 100644 --- a/Duplicati/UnitTest/DisruptionTests.cs +++ b/Duplicati/UnitTest/DisruptionTests.cs @@ -752,5 +752,105 @@ namespace Duplicati.UnitTest } } } + + [Test] + [Category("Disruption")] + public void TestFailedUploadWithNoRetries() + { + var testopts = TestOptions; + testopts["number-of-retries"] = "0"; + testopts["dblock-size"] = "10mb"; + + // Make a base backup + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + } + + // Make a new backup that fails uploading a dblock file + ModifySourceFiles(); + + // Deterministic error backend + Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend()); + var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; + + var hasFailed = false; + var secondUploadStarted = false; + var secondUploadCompleted = false; + + // Fail the compact after the first dblock put is completed + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => + { + if (action.IsGetOperation) + { + return true; + } + + if (!hasFailed && action == DeterministicErrorBackend.BackendAction.PutBefore) + { + // We only fail one upload, but there are no retries + hasFailed = true; + + // Make sure we can start a second upload + Thread.Sleep(1000); + return true; + } + + if (action == DeterministicErrorBackend.BackendAction.PutBefore) + secondUploadStarted = true; + if (action == DeterministicErrorBackend.BackendAction.PutAfter) + secondUploadCompleted = true; + + return false; + }; + + using (var c = new Library.Main.Controller(failtarget, testopts, null)) + Assert.Throws(() => c.Backup(new string[] { DATAFOLDER })); + + Assert.That(secondUploadStarted, Is.True, "Second upload was not started"); + Assert.That(secondUploadCompleted, Is.True, "Second upload was not started"); + + // Create a regular backup + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER }); + Assert.AreEqual(0, backupResults.Errors.Count()); + Assert.AreEqual(0, backupResults.Warnings.Count()); + } + + // Verify that all is in order + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null)) + { + var r = c.Test(long.MaxValue); + Assert.AreEqual(0, r.Errors.Count()); + Assert.AreEqual(0, r.Warnings.Count()); + Assert.IsFalse(r.Verifications.Any(p => p.Value.Any())); + } + + // Test that we can recreate + var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite"); + if (File.Exists(recreatedDatabaseFile)) + File.Delete(recreatedDatabaseFile); + + testopts["dbpath"] = recreatedDatabaseFile; + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IRepairResults repairResults = c.Repair(); + Assert.AreEqual(0, repairResults.Errors.Count()); + Assert.AreEqual(0, repairResults.Warnings.Count()); + } + + // Check that we have 3 versions + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + IListResults listResults = c.List(); + Assert.AreEqual(0, listResults.Errors.Count()); + Assert.AreEqual(0, listResults.Warnings.Count()); + Assert.AreEqual(3, listResults.Filesets.Count()); + } + } } } \ No newline at end of file diff --git a/Duplicati/UnitTest/IssueTests.cs b/Duplicati/UnitTest/IssueTests.cs index 62ddc787b..a0302b149 100644 --- a/Duplicati/UnitTest/IssueTests.cs +++ b/Duplicati/UnitTest/IssueTests.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2025, The Duplicati Team +// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -68,10 +68,10 @@ namespace Duplicati.UnitTest // Fail after index file put target = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; string interruptedName = ""; - DeterministicErrorBackend.ErrorGenerator = (string action, string remotename) => + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => { // Fail dindex upload - if (action.StartsWith("put") && remotename.Contains("dindex")) + if (action == DeterministicErrorBackend.BackendAction.PutAfter && remotename.Contains("dindex")) { interruptedName = remotename; return true; @@ -460,7 +460,7 @@ namespace Duplicati.UnitTest bool failed = false; long accessCounter = 0; long errorIdx = 0; - DeterministicErrorBackend.ErrorGenerator = (string action, string remotename) => + DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) => { ++accessCounter; if (accessCounter >= errorIdx)