From f0d8677ef50232d7ffee881dde26e83074450556 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 23 Sep 2025 11:12:36 +0200 Subject: [PATCH] Include duplicated blocks in recreated index files This fixes #6504 --- .../Library/Main/Database/LocalDatabase.cs | 2 + Duplicati/UnitTest/Issue6504.cs | 138 ++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 Duplicati/UnitTest/Issue6504.cs diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 0864266bf..5adde9fc7 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -2051,6 +2051,8 @@ namespace Duplicati.Library.Main.Database SELECT ""Hash"", ""Size"" FROM ""Block"" WHERE ""VolumeID"" = @VolumeId UNION ALL SELECT ""Hash"", ""Size"" FROM ""DeletedBlock"" WHERE ""VolumeID"" = @VolumeId + UNION ALL + SELECT ""Hash"", ""Size"" FROM ""DuplicateBlock"" INNER JOIN ""Block"" ON ""DuplicateBlock"".""BlockID"" = ""Block"".""ID"" WHERE ""DuplicateBlock"".""VolumeID"" = @VolumeId ) SELECT DISTINCT ""Hash"", ""Size"" FROM AllBlocks ", token) diff --git a/Duplicati/UnitTest/Issue6504.cs b/Duplicati/UnitTest/Issue6504.cs new file mode 100644 index 000000000..488818326 --- /dev/null +++ b/Duplicati/UnitTest/Issue6504.cs @@ -0,0 +1,138 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System.IO; +using System.Linq; +using NUnit.Framework; +using Duplicati.Library.Main.Volumes; +using Duplicati.Library.SQLiteHelper; +using Duplicati.Library.Main.Database; + +namespace Duplicati.UnitTest; + +public class Issue6504 : BasicSetupHelper +{ + [Test] + [Category("Targeted")] + public void RecreateIndexFilesShouldHandleDuplicatedBlocks() + { + var testopts = TestOptions.Expand(new { no_encryption = true, keep_versions = 1 }); + + var patha = Path.Combine(DATAFOLDER, "A.txt"); + var pathb = Path.Combine(DATAFOLDER, "B.txt"); + + File.WriteAllText(patha, "A"); + File.WriteAllText(pathb, "B"); + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup(new[] { patha, pathb })); + + var original_dblock_file = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).First(); + var original_dindex_file = Directory.GetFiles(TARGETFOLDER, "*.dindex.*", SearchOption.TopDirectoryOnly).First(); + + var opts = new Library.Main.Options(testopts); + + // Create a duplicated dblock file + var newdblockpath = Path.Combine(TARGETFOLDER, VolumeBase.GenerateFilename(Library.Main.RemoteVolumeType.Blocks, opts, VolumeWriterBase.GenerateGuid(), System.DateTime.UtcNow)); + File.Copy(original_dblock_file, newdblockpath); + + var oldname = Path.GetFileName(original_dblock_file); + var newname = Path.GetFileName(newdblockpath); + + // Duplicate the index file as well, but pointing to the new duplicated block + var newdindexpath_broken = Path.Combine(TARGETFOLDER, VolumeBase.GenerateFilename(Library.Main.RemoteVolumeType.Index, new Library.Main.Options(testopts), VolumeWriterBase.GenerateGuid(), System.DateTime.UtcNow)); + var newdindexpath_correct = Path.Combine(TARGETFOLDER, VolumeBase.GenerateFilename(Library.Main.RemoteVolumeType.Index, new Library.Main.Options(testopts), VolumeWriterBase.GenerateGuid(), System.DateTime.UtcNow)); + using (var wr = new IndexVolumeWriter(opts)) + { + using (var rd = new IndexVolumeReader(opts.CompressionModule, original_dindex_file, opts, opts.BlockhashSize)) + { + foreach (var n in rd.Volumes) + { + wr.StartVolume(newname); + foreach (var x in n.Blocks.Take(2)) // Only take two to simulate a corrupted index file + wr.AddBlock(x.Key, x.Value); + wr.FinishVolume(n.Hash, n.Length); + } + } + wr.Close(); + + File.Copy(wr.LocalFilename, newdindexpath_broken); + } + + using (var wr = new IndexVolumeWriter(opts)) + { + using (var rd = new IndexVolumeReader(opts.CompressionModule, original_dindex_file, opts, opts.BlockhashSize)) + wr.CopyFrom(rd, (_) => newname); + wr.Close(); + + File.Copy(wr.LocalFilename, newdindexpath_correct); + } + + + // Prepare for accepting the new duplicated block by recreating the database + File.Delete(DBFILE); + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Repair()); + + // Force the error by making sure the duplicate blocks are from both dblock files + using (var db = SQLiteLoader.LoadConnection(DBFILE)) + using (var cmd = db.CreateCommand()) + { + cmd.CommandText = @"SELECT BlockId,VolumeID FROM DuplicateBlock"; + var duplicatedBlocks = cmd.ExecuteReaderEnumerable().Select(r => new { BlockId = r.GetInt64(0), VolumeID = r.GetInt64(1) }).ToList(); + + cmd.CommandText = @"SELECT ID,VolumeID,Hash FROM Block"; + var blocks = cmd.ExecuteReaderEnumerable().Select(r => new { ID = r.GetInt64(0), VolumeID = r.GetInt64(1), Hash = r.GetString(2) }).ToList(); + + if (duplicatedBlocks.Select(x => x.VolumeID).Distinct().Count() == 1) + { + // All duplicated blocks are from the same volume, change one to be from the other volume + var blockToChange = duplicatedBlocks.First(); + var duplicateVolumeId = blockToChange.VolumeID; + var otherVolumeId = blocks.First(x => x.VolumeID != duplicateVolumeId && x.ID != blockToChange.BlockId).VolumeID; + + cmd.CommandText = "UPDATE DuplicateBlock SET VolumeID=@VolumeID WHERE BlockID=@BlockID"; + cmd.Parameters.AddWithValue("@VolumeID", otherVolumeId); + cmd.Parameters.AddWithValue("@BlockID", blockToChange.BlockId); + cmd.ExecuteNonQuery(); + cmd.Parameters.Clear(); + + cmd.CommandText = "UPDATE Block SET VolumeID=@VolumeID WHERE ID=@BlockID"; + cmd.Parameters.AddWithValue("@VolumeID", duplicateVolumeId); + cmd.Parameters.AddWithValue("@BlockID", blockToChange.BlockId); + cmd.ExecuteNonQuery(); + } + } + + + // Make sure the test rewrites the faulty index file + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + var res = c.Test(100); + Assert.That(res.Warnings.Count, Is.EqualTo(1), "Expected one warning about faulty index files"); + Assert.That(res.Warnings.Any(c => c.Contains("FaultyIndexFiles")), Is.True, "Expected a warning about faulty index files"); + } + + // Second run should not have faulty index files + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Test(100)); + } +}