diff --git a/Duplicati/UnitTest/CompactDisruptionTests.cs b/Duplicati/UnitTest/CompactDisruptionTests.cs index 660b86956..4642cba6c 100644 --- a/Duplicati/UnitTest/CompactDisruptionTests.cs +++ b/Duplicati/UnitTest/CompactDisruptionTests.cs @@ -29,7 +29,7 @@ namespace Duplicati.UnitTest } [Test] - [Category("Disruption")] + [Category("Disruption"), Category("Bug"), Explicit("Known bug")] public void InterruptedCompact() { // Reproduction steps from issue #4129 with smaller sizes diff --git a/Duplicati/UnitTest/GeneralBlackBoxTesting.cs b/Duplicati/UnitTest/GeneralBlackBoxTesting.cs index 7b0065dae..298f34547 100644 --- a/Duplicati/UnitTest/GeneralBlackBoxTesting.cs +++ b/Duplicati/UnitTest/GeneralBlackBoxTesting.cs @@ -22,6 +22,8 @@ using System.Collections.Generic; using System.Net; using System.Reflection; using Duplicati.Library.Utility; +using Duplicati.Library.Interface; +using System.Threading; namespace Duplicati.UnitTest { @@ -36,10 +38,10 @@ namespace Duplicati.UnitTest get { var rx = new System.Text.RegularExpressions.Regex("r(?\\d+)"); - return + return from n in Directory.EnumerateDirectories(SOURCE_FOLDERS) let m = rx.Match(n) - where m.Success + where m.Success orderby int.Parse(m.Groups["number"].Value) select n; } @@ -53,7 +55,7 @@ namespace Duplicati.UnitTest } } - protected string TestTarget + protected string TestTarget { get { @@ -74,40 +76,50 @@ namespace Duplicati.UnitTest { Console.WriteLine("downloading test file to: {0}, length: {1}", destinationFilePath, wr.ContentLength); var maxAttempts = 5; - while (maxAttempts-- > 0) { - try { + while (maxAttempts-- > 0) + { + try + { DateTime beginTime = DateTime.Now; client.DownloadFile(url, destinationFilePath); long length = new System.IO.FileInfo(destinationFilePath).Length; Console.WriteLine("downloaded test file: {0}: length {1}, duration {2}", destinationFilePath, length, (DateTime.Now - beginTime).TotalSeconds); - if (length == wr.ContentLength) { + if (length == wr.ContentLength) + { maxAttempts = -1; - } else { + } + else + { Console.WriteLine("invalid downloaded length {0}, should be {1}...", length, wr.ContentLength); System.Threading.Thread.Sleep(120000); } } catch (WebException ex) { - if (ex.Response == null){ + if (ex.Response == null) + { Console.WriteLine("ex.Response is null !"); - } else { + } + else + { Console.WriteLine("exception {0}", ex); throw; } } } - if (maxAttempts == 0) { + if (maxAttempts == 0) + { throw new Exception(string.Format("Unable to download test file from {0}", url)); } } - + System.IO.Compression.ZipFile.ExtractToDirectory(this.zipFilepath, BASEFOLDER); } public override void OneTimeTearDown() { + base.OneTimeTearDown(); if (Directory.Exists(SOURCE_FOLDERS)) { Directory.Delete(SOURCE_FOLDERS, true); @@ -124,7 +136,7 @@ namespace Duplicati.UnitTest { SVNCheckoutTest.RunTest(TestFolders.Take(5).ToArray(), TestOptions, TestTarget); } - + [Test] [Category("SVNDataLong")] public void TestWithSVNLong() diff --git a/Duplicati/UnitTest/IssueTests.cs b/Duplicati/UnitTest/IssueTests.cs index 52cea3c1f..b59f8de5a 100644 --- a/Duplicati/UnitTest/IssueTests.cs +++ b/Duplicati/UnitTest/IssueTests.cs @@ -1,10 +1,13 @@ using Duplicati.Library.Interface; +using Duplicati.Library.Main; using NUnit.Framework; using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Security.Cryptography; using System.Text; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; @@ -14,23 +17,22 @@ namespace Duplicati.UnitTest { [Test] - [Category("Targeted")] - [TestCase(true), TestCase(false)] - public void Issue5023ReferencedFileMissing(bool compactBeforeRecreate) + [Category("Targeted"), Category("Bug"), Explicit("Known bug")] + public void Issue5023ReferencedFileMissing([Values] bool compactBeforeRecreate) { // Reproduction for part of issue #5023 // Error during repair: "Remote file referenced as x by y, but not found in list, registering a missing remote file" // Can be caused by interrupted index upload followed by compact before the repair - var testopts = TestOptions; - testopts["backup-test-samples"] = "0"; - testopts["number-of-retries"] = "0"; - testopts["dblock-size"] = "20KB"; - testopts["threshold"] = "1"; - testopts["keep-versions"] = "1"; - testopts["no-auto-compact"] = "true"; - testopts["no-encryption"] = "true"; - testopts.Remove("passphrase"); + var testopts = new Dictionary(TestOptions) + { + ["number-of-retries"] = "0", + ["dblock-size"] = "20KB", + ["threshold"] = "1", + ["keep-versions"] = "1", + ["no-auto-compact"] = "true", + ["no-encryption"] = "true" + }; const long filesize = 1024; @@ -109,5 +111,164 @@ namespace Duplicati.UnitTest TestUtils.AssertResults(repairResults); } } + + + [Test] + [Category("Disruption"), Category("Bug"), Explicit("Known bug")] + public void TestSystematicErrors() + { + // Attempt to recreate other bugs from #5023, but not successful + var testopts = new Dictionary(TestOptions) + { + ["number-of-retries"] = "0", + ["dblock-size"] = "20KB", + ["threshold"] = "1", + ["keep-versions"] = "5", + ["no-encryption"] = "true", + ["disable-synthetic-filelist"] = "true" + }; + //testopts["rebuild-missing-dblock-files"] = "true"; + string target = "file://" + TARGETFOLDER; + string targetError = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER; + int maxFiles = 10; + List files = new List(); + bool failed = false; + long accessCounter = 0; + long errorIdx = 0; + DeterministicErrorBackend.ErrorGenerator = (string action, string remotename) => + { + ++accessCounter; + if (accessCounter >= errorIdx) + { + return true; + } + return false; + }; + for (int i = 0; i < maxFiles; ++i) + { + string f = Path.Combine(DATAFOLDER, "f" + i); + TestUtils.WriteTestFile(f, 1024 * 20); + files.Add(f); + } + // Initial backup + using (var c = new Library.Main.Controller(target, testopts, null)) + { + IBackupResults backupResults = c.Backup(new[] { DATAFOLDER }); + TestUtils.AssertResults(backupResults); + } + while (errorIdx < (maxFiles + 2)) + { + if (errorIdx % 10 == 0) + { + TestContext.WriteLine("Error index {0}", errorIdx); + } + accessCounter = 0; + try + { + using (var c = new Library.Main.Controller(targetError, testopts, null)) + { + IBackupResults backupResults = c.Backup(new[] { DATAFOLDER }); + TestUtils.AssertResults(backupResults); + } + } + catch (AssertionException) { throw; } + catch { } + Thread.Sleep(1000); + try + { + using (var c = new Library.Main.Controller(target, testopts, null)) + { + IBackupResults backupResults = c.Backup(new[] { DATAFOLDER }); + TestUtils.AssertResults(backupResults); + } + } + catch (UserInformationException e) + { + TestContext.WriteLine("Error at index {0}: {1}", errorIdx, e.Message); + if (e.HelpID == "MissingRemoteFiles" || e.HelpID == "ExtraRemoteFiles") + { + using (var c = new Library.Main.Controller(target, testopts, null)) + { + IRepairResults repairResults = c.Repair(); + TestUtils.AssertResults(repairResults); + } + } + failed = true; + } + Thread.Sleep(1000); + foreach (string f in files) + { + TestUtils.WriteTestFile(f, 1024 * 20); + } + ++errorIdx; + } + TestContext.WriteLine("Ran {0} iterations", errorIdx); + Assert.IsFalse(failed); + } + + [Test, Sequential] + [Category("Targeted"), Category("Bug"), Category("Non-critical"), Explicit("Known bug")] + [TestCase(false, true), TestCase(true, true), TestCase(true, false)] + public void Issue5038MissingListBlocklist(bool sameVersion, bool blockFirst) + { + // Backup containing the blocklist of a file BEFORE the file causes a dindex with missing blocklist entry + // This is not critical, because it only requires extra block volume downloads + var testopts = new Dictionary(TestOptions) + { + ["no-encryption"] = "true" + }; + + string filename = Path.Combine(DATAFOLDER, "file"); + // Start with z to process blockfile after file (at least on some systems) + string blockfile = Path.Combine(DATAFOLDER, blockFirst ? "block" : "zblock"); + string target = "file://" + TARGETFOLDER; + + byte[] block1 = new byte[10 * 1024]; + for (int i = 0; i < block1.Length; ++i) + { + block1[i] = 1; + } + byte[] block2 = new byte[10 * 1024]; + for (int i = 0; i < block1.Length; ++i) + { + block1[i] = 2; + } + + HashAlgorithm blockhasher = Library.Utility.HashAlgorithmHelper.Create(new Options(testopts).BlockHashAlgorithm); + + var hash1 = blockhasher.ComputeHash(block1, 0, block1.Length); + var hash2 = blockhasher.ComputeHash(block2, 0, block2.Length); + + byte[] blockfileContent = hash1.Concat(hash2).ToArray(); + TestUtils.WriteFile(blockfile, blockfileContent); + if (!sameVersion) + { + // Backup blockfile first + using (var c = new Library.Main.Controller(target, testopts, null)) + { + IBackupResults backupResults = c.Backup(new[] { DATAFOLDER }); + TestUtils.AssertResults(backupResults); + } + } + + byte[] combined = block1.Concat(block2).ToArray(); + TestUtils.WriteFile(filename, combined); + // Backup file that would produce blockfile + using (var c = new Library.Main.Controller(target, testopts, null)) + { + IBackupResults backupResults = c.Backup(new[] { DATAFOLDER }); + TestUtils.AssertResults(backupResults); + } + + // Recreate database downloads block volume + File.Delete(DBFILE); + using (var c = new Library.Main.Controller(target, testopts, null)) + { + IRepairResults repairResults = c.Repair(); + TestUtils.AssertResults(repairResults); + Assert.IsNull(repairResults.Messages.FirstOrDefault(v => v.Contains("ProcessingRequiredBlocklistVolumes")), + "Blocklist download pass was required"); + } + } } }