haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add categories and reproduction of issue #5038.

Browse Source
This commit is contained in:
Jojo-1000
2023-10-08 22:25:44 +02:00
parent ea6c418b7b
commit b4d0ed2cd9
3 changed files with 198 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/UnitTest/CompactDisruptionTests.cs
+1 -1
View File
@@ -29,7 +29,7 @@ namespace Duplicati.UnitTest
}
[Test]
[Category("Disruption")]
[Category("Disruption"), Category("Bug"), Explicit("Known bug")]
public void InterruptedCompact()
{
// Reproduction steps from issue #4129 with smaller sizes
Duplicati/UnitTest/GeneralBlackBoxTesting.cs
+24 -12
View File
@@ -22,6 +22,8 @@ using System.Collections.Generic;
using System.Net;
using System.Reflection;
using Duplicati.Library.Utility;
using Duplicati.Library.Interface;
using System.Threading;
namespace Duplicati.UnitTest
{
@@ -36,10 +38,10 @@ namespace Duplicati.UnitTest
get
{
var rx = new System.Text.RegularExpressions.Regex("r(?<number>\\d+)");
return
return
from n in Directory.EnumerateDirectories(SOURCE_FOLDERS)
let m = rx.Match(n)
where m.Success
where m.Success
orderby int.Parse(m.Groups["number"].Value)
select n;
}
@@ -53,7 +55,7 @@ namespace Duplicati.UnitTest
}
}
protected string TestTarget
protected string TestTarget
{
get
{
@@ -74,40 +76,50 @@ namespace Duplicati.UnitTest
{
Console.WriteLine("downloading test file to: {0}, length: {1}", destinationFilePath, wr.ContentLength);
var maxAttempts = 5;
while (maxAttempts-- > 0) {
try {
while (maxAttempts-- > 0)
{
try
{
DateTime beginTime = DateTime.Now;
client.DownloadFile(url, destinationFilePath);
long length = new System.IO.FileInfo(destinationFilePath).Length;
Console.WriteLine("downloaded test file: {0}: length {1}, duration {2}", destinationFilePath, length, (DateTime.Now - beginTime).TotalSeconds);
if (length == wr.ContentLength) {
if (length == wr.ContentLength)
{
maxAttempts = -1;
} else {
}
else
{
Console.WriteLine("invalid downloaded length {0}, should be {1}...", length, wr.ContentLength);
System.Threading.Thread.Sleep(120000);
}
}
catch (WebException ex)
{
if (ex.Response == null){
if (ex.Response == null)
{
Console.WriteLine("ex.Response is null !");
} else {
}
else
{
Console.WriteLine("exception {0}", ex);
throw;
}
}
}
if (maxAttempts == 0) {
if (maxAttempts == 0)
{
throw new Exception(string.Format("Unable to download test file from {0}", url));
}
}
System.IO.Compression.ZipFile.ExtractToDirectory(this.zipFilepath, BASEFOLDER);
}
public override void OneTimeTearDown()
{
base.OneTimeTearDown();
if (Directory.Exists(SOURCE_FOLDERS))
{
Directory.Delete(SOURCE_FOLDERS, true);
@@ -124,7 +136,7 @@ namespace Duplicati.UnitTest
{
SVNCheckoutTest.RunTest(TestFolders.Take(5).ToArray(), TestOptions, TestTarget);
}
[Test]
[Category("SVNDataLong")]
public void TestWithSVNLong()
Duplicati/UnitTest/IssueTests.cs
+173 -12
View File
@@ -1,10 +1,13 @@
using Duplicati.Library.Interface;
using Duplicati.Library.Main;
using NUnit.Framework;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
@@ -14,23 +17,22 @@ namespace Duplicati.UnitTest
{
[Test]
[Category("Targeted")]
[TestCase(true), TestCase(false)]
public void Issue5023ReferencedFileMissing(bool compactBeforeRecreate)
[Category("Targeted"), Category("Bug"), Explicit("Known bug")]
public void Issue5023ReferencedFileMissing([Values] bool compactBeforeRecreate)
{
// Reproduction for part of issue #5023
// Error during repair: "Remote file referenced as x by y, but not found in list, registering a missing remote file"
// Can be caused by interrupted index upload followed by compact before the repair
var testopts = TestOptions;
testopts["backup-test-samples"] = "0";
testopts["number-of-retries"] = "0";
testopts["dblock-size"] = "20KB";
testopts["threshold"] = "1";
testopts["keep-versions"] = "1";
testopts["no-auto-compact"] = "true";
testopts["no-encryption"] = "true";
testopts.Remove("passphrase");
var testopts = new Dictionary<string, string>(TestOptions)
{
["number-of-retries"] = "0",
["dblock-size"] = "20KB",
["threshold"] = "1",
["keep-versions"] = "1",
["no-auto-compact"] = "true",
["no-encryption"] = "true"
};
const long filesize = 1024;
@@ -109,5 +111,164 @@ namespace Duplicati.UnitTest
TestUtils.AssertResults(repairResults);
}
}
[Test]
[Category("Disruption"), Category("Bug"), Explicit("Known bug")]
public void TestSystematicErrors()
{
// Attempt to recreate other bugs from #5023, but not successful
var testopts = new Dictionary<string, string>(TestOptions)
{
["number-of-retries"] = "0",
["dblock-size"] = "20KB",
["threshold"] = "1",
["keep-versions"] = "5",
["no-encryption"] = "true",
["disable-synthetic-filelist"] = "true"
};
//testopts["rebuild-missing-dblock-files"] = "true";
string target = "file://" + TARGETFOLDER;
string targetError = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER;
int maxFiles = 10;
List<string> files = new List<string>();
bool failed = false;
long accessCounter = 0;
long errorIdx = 0;
DeterministicErrorBackend.ErrorGenerator = (string action, string remotename) =>
{
++accessCounter;
if (accessCounter >= errorIdx)
{
return true;
}
return false;
};
for (int i = 0; i < maxFiles; ++i)
{
string f = Path.Combine(DATAFOLDER, "f" + i);
TestUtils.WriteTestFile(f, 1024 * 20);
files.Add(f);
}
// Initial backup
using (var c = new Library.Main.Controller(target, testopts, null))
{
IBackupResults backupResults = c.Backup(new[] { DATAFOLDER });
TestUtils.AssertResults(backupResults);
}
while (errorIdx < (maxFiles + 2))
{
if (errorIdx % 10 == 0)
{
TestContext.WriteLine("Error index {0}", errorIdx);
}
accessCounter = 0;
try
{
using (var c = new Library.Main.Controller(targetError, testopts, null))
{
IBackupResults backupResults = c.Backup(new[] { DATAFOLDER });
TestUtils.AssertResults(backupResults);
}
}
catch (AssertionException) { throw; }
catch { }
Thread.Sleep(1000);
try
{
using (var c = new Library.Main.Controller(target, testopts, null))
{
IBackupResults backupResults = c.Backup(new[] { DATAFOLDER });
TestUtils.AssertResults(backupResults);
}
}
catch (UserInformationException e)
{
TestContext.WriteLine("Error at index {0}: {1}", errorIdx, e.Message);
if (e.HelpID == "MissingRemoteFiles" || e.HelpID == "ExtraRemoteFiles")
{
using (var c = new Library.Main.Controller(target, testopts, null))
{
IRepairResults repairResults = c.Repair();
TestUtils.AssertResults(repairResults);
}
}
failed = true;
}
Thread.Sleep(1000);
foreach (string f in files)
{
TestUtils.WriteTestFile(f, 1024 * 20);
}
++errorIdx;
}
TestContext.WriteLine("Ran {0} iterations", errorIdx);
Assert.IsFalse(failed);
}
[Test, Sequential]
[Category("Targeted"), Category("Bug"), Category("Non-critical"), Explicit("Known bug")]
[TestCase(false, true), TestCase(true, true), TestCase(true, false)]
public void Issue5038MissingListBlocklist(bool sameVersion, bool blockFirst)
{
// Backup containing the blocklist of a file BEFORE the file causes a dindex with missing blocklist entry
// This is not critical, because it only requires extra block volume downloads
var testopts = new Dictionary<string, string>(TestOptions)
{
["no-encryption"] = "true"
};
string filename = Path.Combine(DATAFOLDER, "file");
// Start with z to process blockfile after file (at least on some systems)
string blockfile = Path.Combine(DATAFOLDER, blockFirst ? "block" : "zblock");
string target = "file://" + TARGETFOLDER;
byte[] block1 = new byte[10 * 1024];
for (int i = 0; i < block1.Length; ++i)
{
block1[i] = 1;
}
byte[] block2 = new byte[10 * 1024];
for (int i = 0; i < block1.Length; ++i)
{
block1[i] = 2;
}
HashAlgorithm blockhasher = Library.Utility.HashAlgorithmHelper.Create(new Options(testopts).BlockHashAlgorithm);
var hash1 = blockhasher.ComputeHash(block1, 0, block1.Length);
var hash2 = blockhasher.ComputeHash(block2, 0, block2.Length);
byte[] blockfileContent = hash1.Concat(hash2).ToArray();
TestUtils.WriteFile(blockfile, blockfileContent);
if (!sameVersion)
{
// Backup blockfile first
using (var c = new Library.Main.Controller(target, testopts, null))
{
IBackupResults backupResults = c.Backup(new[] { DATAFOLDER });
TestUtils.AssertResults(backupResults);
}
}
byte[] combined = block1.Concat(block2).ToArray();
TestUtils.WriteFile(filename, combined);
// Backup file that would produce blockfile
using (var c = new Library.Main.Controller(target, testopts, null))
{
IBackupResults backupResults = c.Backup(new[] { DATAFOLDER });
TestUtils.AssertResults(backupResults);
}
// Recreate database downloads block volume
File.Delete(DBFILE);
using (var c = new Library.Main.Controller(target, testopts, null))
{
IRepairResults repairResults = c.Repair();
TestUtils.AssertResults(repairResults);
Assert.IsNull(repairResults.Messages.FirstOrDefault(v => v.Contains("ProcessingRequiredBlocklistVolumes")),
"Blocklist download pass was required");
}
}
}
}