diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index 6423746e7..e6579f853 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -725,6 +725,36 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" } } + /// + /// Move blocks that are not referenced by any files to DeletedBlock table. + /// + /// Needs to be called after the last FindMissingBlocklistHashes, otherwise the tables are not up to date. + public void CleanupDeletedBlocks(System.Data.IDbTransaction transaction) + { + // Find out which blocks are deleted and move them into DeletedBlock, so that compact notices these blocks are empty + // Deleted blocks do not appear in the BlocksetEntry and not in the BlocklistHash table + + var tmptablename = "DeletedBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + + using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) + using (var cmd = m_connection.CreateCommand(tr.Parent)) + { + // 1. Select blocks not used by any file and not as a blocklist into temporary table + cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" + AS SELECT ""Block"".""ID"", ""Block"".""Hash"", ""Block"".""Size"", ""Block"".""VolumeID"" FROM ""Block"" + WHERE ""Block"".""ID"" NOT IN (SELECT ""BlocksetEntry"".""BlockID"" FROM ""BlocksetEntry"") + AND ""Block"".""Hash"" NOT IN (SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"")", tmptablename)); + // 2. Insert blocks into DeletedBlock table + cmd.ExecuteNonQuery(string.Format(@"INSERT INTO ""DeletedBlock"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""{0}""", tmptablename)); + // 3. Remove blocks from Block table + cmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""Block"" WHERE ""ID"" IN (SELECT ""ID"" FROM ""{0}"")", tmptablename)); + cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}""", tmptablename)); + tr.Commit(); + } + + + } + public override void Dispose() { using (var cmd = m_connection.CreateCommand()) diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index 1e1e72370..d7d9f5a19 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -1,4 +1,4 @@ -// Copyright (C) 2024, The Duplicati Team +// Copyright (C) 2024, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a @@ -582,6 +582,14 @@ namespace Duplicati.Library.Main.Operation backend.WaitForComplete(restoredb, null); + if (!m_options.RepairOnlyPaths) + { + // All blocks are collected and added into the Block table + // Find out which blocks are deleted and move them into DeletedBlock, + // so that compact can calculate the unused space + restoredb.CleanupDeletedBlocks(null); + } + // In some cases we have a stale reference from an index file to a deleted block file if (!m_options.UnittestMode) restoredb.CleanupMissingVolumes(); diff --git a/Duplicati/UnitTest/BorderTests.cs b/Duplicati/UnitTest/BorderTests.cs index 6e9015b37..3e05176f9 100644 --- a/Duplicati/UnitTest/BorderTests.cs +++ b/Duplicati/UnitTest/BorderTests.cs @@ -309,6 +309,14 @@ namespace Duplicati.UnitTest Assert.AreEqual((filenames.Count * 3) + 1, r.Files.Count()); } + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null)) + { + var r = c.Test(long.MaxValue); + Assert.AreEqual(0, r.Errors.Count()); + Assert.AreEqual(0, r.Warnings.Count()); + Assert.IsFalse(r.Verifications.Any(p => p.Value.Any())); + } + testopts["dbpath"] = this.recreatedDatabaseFile; using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) @@ -358,6 +366,14 @@ namespace Duplicati.UnitTest Assert.AreEqual((filenames.Count * 3) + 1, r.Files.Count()); } + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null)) + { + var r = c.Test(long.MaxValue); + Assert.AreEqual(0, r.Errors.Count()); + Assert.AreEqual(0, r.Warnings.Count()); + Assert.IsFalse(r.Verifications.Any(p => p.Value.Any())); + } + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { restore_path = RESTOREFOLDER }), null)) { var r = c.Restore(null); diff --git a/Duplicati/UnitTest/CompactDisruptionTests.cs b/Duplicati/UnitTest/CompactDisruptionTests.cs index 07526ab9a..fb88aa1f3 100644 --- a/Duplicati/UnitTest/CompactDisruptionTests.cs +++ b/Duplicati/UnitTest/CompactDisruptionTests.cs @@ -566,9 +566,7 @@ namespace Duplicati.UnitTest using (var c = new Library.Main.Controller(target, testopts, null)) { ICompactResults compactResults = c.Compact(); - // This currently fails, because the DeletedBlocks table is not populated after a repair - // Once PR #4982 is merged, this should work - // Assert.Greater(compactResults.DownloadedFileCount, 0, "No compact operation was performed"); + Assert.Greater(compactResults.DownloadedFileCount, 0, "No compact operation was performed"); } // Make sure there are no errors after success compacting