haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #5986 from duplicati/feature/test-failed-upload

Browse Source 
Added interuption test
This commit is contained in:
Kenneth Skovhede
2025-02-24 13:06:37 +01:00
committed by GitHub
parent 5aa06f3cd4 655f5ef635
commit 4cd137130e
4 changed files with 143 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/UnitTest/CompactDisruptionTests.cs
+12 -12
View File
@@ -135,13 +135,13 @@ namespace Duplicati.UnitTest
target = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER;
// Fail the compact after the first dblock put is completed
bool firstPutCompleted = false;
DeterministicErrorBackend.ErrorGenerator = (string action, string remotename) =>
DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) =>
{
if (firstPutCompleted && (action == "get_0" || action == "get_1"))
if (firstPutCompleted && action.IsGetOperation)
{
return true;
}
if (action == "put_1" || action == "put_async")
if (action == DeterministicErrorBackend.BackendAction.PutAfter)
{
firstPutCompleted = true;
}
@@ -250,13 +250,13 @@ namespace Duplicati.UnitTest
target = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER;
// Fail the compact after the first dblock put is completed
bool firstPutCompleted = false;
DeterministicErrorBackend.ErrorGenerator = (string action, string remotename) =>
DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) =>
{
if (firstPutCompleted && (action == "get_0" || action == "get_1"))
if (firstPutCompleted && action.IsGetOperation)
{
return true;
}
if (action == "put_1" || action == "put_async")
if (action == DeterministicErrorBackend.BackendAction.PutAfter)
{
firstPutCompleted = true;
}
@@ -382,13 +382,13 @@ namespace Duplicati.UnitTest
target = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER;
// Fail the compact after the first dblock put is completed
bool firstPutCompleted = false;
DeterministicErrorBackend.ErrorGenerator = (string action, string remotename) =>
DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) =>
{
if (firstPutCompleted && (action == "get_0" || action == "get_1"))
if (firstPutCompleted && action.IsGetOperation)
{
return true;
}
if (action == "put_1" || action == "put_async")
if (action == DeterministicErrorBackend.BackendAction.PutAfter)
{
firstPutCompleted = true;
}
@@ -523,13 +523,13 @@ namespace Duplicati.UnitTest
target = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER;
// Fail the compact after the first dblock put is completed
bool firstPutCompleted = false;
DeterministicErrorBackend.ErrorGenerator = (string action, string remotename) =>
DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) =>
{
if (firstPutCompleted && (action == "get_0" || action == "get_1"))
if (firstPutCompleted && action.IsGetOperation)
{
return true;
}
if (action == "put_1" || action == "put_async")
if (action == DeterministicErrorBackend.BackendAction.PutAfter)
{
firstPutCompleted = true;
}
Duplicati/UnitTest/DeterministicErrorBackend.cs
+27 -17
View File
@@ -31,14 +31,26 @@ namespace Duplicati.UnitTest
{
public class DeterministicErrorBackend : IBackend, IStreamingBackend
{
public sealed record BackendAction(string Code)
{
public static readonly BackendAction GetBefore = new BackendAction("get_0");
public static readonly BackendAction GetAfter = new BackendAction("get_1");
public static readonly BackendAction PutBefore = new BackendAction("put_0");
public static readonly BackendAction PutAfter = new BackendAction("put_1");
public static readonly BackendAction DeleteBefore = new BackendAction("delete_0");
public static readonly BackendAction DeleteAfter = new BackendAction("delete_1");
public bool IsGetOperation => this == GetBefore || this == GetAfter;
public bool IsPutOperation => this == PutBefore || this == PutAfter;
public bool IsDeleteOperation => this == DeleteBefore || this == DeleteAfter;
}
public class DeterministicErrorBackendException(string message) : Exception(message) { };
static DeterministicErrorBackend() { WrappedBackend = "file"; }
private static readonly Random random = new Random(42);
// return true to throw exception, parameters: (action, remotename)
public static Func<string, string, bool> ErrorGenerator = null;
public static Func<BackendAction, string, bool> ErrorGenerator = null;
public static string WrappedBackend { get; set; }
@@ -54,7 +66,7 @@ namespace Duplicati.UnitTest
m_backend = (IStreamingBackend)Library.DynamicLoader.BackendLoader.GetBackend(u, options);
}
private void ThrowError(string action, string remotename)
private void ThrowError(BackendAction action, string remotename)
{
if (ErrorGenerator != null && ErrorGenerator(action, remotename))
{
@@ -64,17 +76,15 @@ namespace Duplicati.UnitTest
#region IStreamingBackend implementation
public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
var uploadError = random.NextDouble() > 0.9;
using (var f = new Library.Utility.ProgressReportingStream(stream, x => { if (uploadError && stream.Position > stream.Length / 2) throw new DeterministicErrorBackendException("Random upload failure"); }))
await m_backend.PutAsync(remotename, f, cancelToken).ConfigureAwait(false);
ThrowError("put_async", remotename);
ThrowError(BackendAction.PutBefore, remotename);
await m_backend.PutAsync(remotename, stream, cancelToken).ConfigureAwait(false);
ThrowError(BackendAction.PutAfter, remotename);
}
public async Task GetAsync(string remotename, Stream stream, CancellationToken cancellationToken)
{
ThrowError("get_0", remotename);
ThrowError(BackendAction.GetBefore, remotename);
await m_backend.GetAsync(remotename, stream, cancellationToken).ConfigureAwait(false);
ThrowError("get_1", remotename);
ThrowError(BackendAction.GetAfter, remotename);
}
#endregion
@@ -85,21 +95,21 @@ namespace Duplicati.UnitTest
}
public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken)
{
ThrowError("put_0", remotename);
ThrowError(BackendAction.PutBefore, remotename);
await m_backend.PutAsync(remotename, filename, cancelToken).ConfigureAwait(false);
ThrowError("put_1", remotename);
ThrowError(BackendAction.PutAfter, remotename);
}
public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken)
{
ThrowError("get_0", remotename);
ThrowError(BackendAction.GetBefore, remotename);
await m_backend.GetAsync(remotename, filename, cancelToken).ConfigureAwait(false);
ThrowError("get_1", remotename);
ThrowError(BackendAction.GetAfter, remotename);
}
public async Task DeleteAsync(string remotename, CancellationToken cancelToken)
{
ThrowError("delete_0", remotename);
ThrowError(BackendAction.DeleteBefore, remotename);
await m_backend.DeleteAsync(remotename, cancelToken).ConfigureAwait(false);
ThrowError("delete_1", remotename);
ThrowError(BackendAction.DeleteAfter, remotename);
}
public Task TestAsync(CancellationToken cancelToken)
{
Duplicati/UnitTest/DisruptionTests.cs
+100
View File
@@ -752,5 +752,105 @@ namespace Duplicati.UnitTest
}
}
}
[Test]
[Category("Disruption")]
public void TestFailedUploadWithNoRetries()
{
var testopts = TestOptions;
testopts["number-of-retries"] = "0";
testopts["dblock-size"] = "10mb";
// Make a base backup
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Make a new backup that fails uploading a dblock file
ModifySourceFiles();
// Deterministic error backend
Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend());
var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER;
var hasFailed = false;
var secondUploadStarted = false;
var secondUploadCompleted = false;
// Fail the compact after the first dblock put is completed
DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) =>
{
if (action.IsGetOperation)
{
return true;
}
if (!hasFailed && action == DeterministicErrorBackend.BackendAction.PutBefore)
{
// We only fail one upload, but there are no retries
hasFailed = true;
// Make sure we can start a second upload
Thread.Sleep(1000);
return true;
}
if (action == DeterministicErrorBackend.BackendAction.PutBefore)
secondUploadStarted = true;
if (action == DeterministicErrorBackend.BackendAction.PutAfter)
secondUploadCompleted = true;
return false;
};
using (var c = new Library.Main.Controller(failtarget, testopts, null))
Assert.Throws<DeterministicErrorBackend.DeterministicErrorBackendException>(() => c.Backup(new string[] { DATAFOLDER }));
Assert.That(secondUploadStarted, Is.True, "Second upload was not started");
Assert.That(secondUploadCompleted, Is.True, "Second upload was not started");
// Create a regular backup
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IBackupResults backupResults = c.Backup(new string[] { DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Verify that all is in order
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null))
{
var r = c.Test(long.MaxValue);
Assert.AreEqual(0, r.Errors.Count());
Assert.AreEqual(0, r.Warnings.Count());
Assert.IsFalse(r.Verifications.Any(p => p.Value.Any()));
}
// Test that we can recreate
var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite");
if (File.Exists(recreatedDatabaseFile))
File.Delete(recreatedDatabaseFile);
testopts["dbpath"] = recreatedDatabaseFile;
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IRepairResults repairResults = c.Repair();
Assert.AreEqual(0, repairResults.Errors.Count());
Assert.AreEqual(0, repairResults.Warnings.Count());
}
// Check that we have 3 versions
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
IListResults listResults = c.List();
Assert.AreEqual(0, listResults.Errors.Count());
Assert.AreEqual(0, listResults.Warnings.Count());
Assert.AreEqual(3, listResults.Filesets.Count());
}
}
}
}
Duplicati/UnitTest/IssueTests.cs
+4 -4
View File
@@ -1,4 +1,4 @@
// Copyright (C) 2025, The Duplicati Team
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
@@ -68,10 +68,10 @@ namespace Duplicati.UnitTest
// Fail after index file put
target = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER;
string interruptedName = "";
DeterministicErrorBackend.ErrorGenerator = (string action, string remotename) =>
DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) =>
{
// Fail dindex upload
if (action.StartsWith("put") && remotename.Contains("dindex"))
if (action == DeterministicErrorBackend.BackendAction.PutAfter && remotename.Contains("dindex"))
{
interruptedName = remotename;
return true;
@@ -460,7 +460,7 @@ namespace Duplicati.UnitTest
bool failed = false;
long accessCounter = 0;
long errorIdx = 0;
DeterministicErrorBackend.ErrorGenerator = (string action, string remotename) =>
DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) =>
{
++accessCounter;
if (accessCounter >= errorIdx)