From 2c5c01160ca4ff7c155b419d39e6f34799cfefd2 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 16 Jun 2025 16:35:07 +0200 Subject: [PATCH] Added a check and repair for empty metadata This PR adds a check for empty metadatasets (length = 0) and replaces such sets with a default metadataset. This fixes #6350 --- .../Library/Main/Database/LocalDatabase.cs | 34 +++++++ .../Database/LocalListBrokenFilesDatabase.cs | 34 ------- .../Main/Database/LocalRepairDatabase.cs | 93 +++++++++++++++++++ .../Library/Main/Operation/RepairHandler.cs | 1 + Duplicati/UnitTest/RepairHandlerTests.cs | 36 +++++++ 5 files changed, 164 insertions(+), 34 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 682ae312d..1f22b6976 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -1733,6 +1733,40 @@ ORDER BY ""A"".""BlocksetID"", ""B"".""Index"" } } + /// + /// Returns the ID of an empty metadata blockset. If no empty blockset is found, it returns the ID of the smallest blockset that is not in the given block volume IDs. + /// If no such blockset is found, it returns -1. + /// + /// The volume ids to ignore when searching for a suitable metadata block + /// The hash of the empty blockset + /// The size of the empty blockset + /// The transaction to use for the query + /// The ID of the empty metadata blockset, or -1 if no suitable blockset is found + public long GetEmptyMetadataBlocksetId(IEnumerable blockVolumeIds, string emptyHash, long emptyHashSize, IDbTransaction? transaction) + { + using var cmd = Connection.CreateCommand(transaction, @"SELECT ""ID"" FROM ""Blockset"" WHERE ""FullHash"" = @EmptyHash AND ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))") + .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) + .SetParameterValue("@EmptyHash", emptyHash) + .SetParameterValue("@EmptyHashSize", emptyHashSize); + + var res = cmd.ExecuteScalarInt64(-1); + + // No empty block found, try to find a zero-length block instead + if (res < 0 && emptyHashSize != 0) + res = cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))") + .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) + .SetParameterValue("@EmptyHashSize", 0) + .ExecuteScalarInt64(-1); + + // No empty block found, pick the smallest one + if (res < 0) + res = cmd.SetCommandAndParameters(@"SELECT ""Blockset"".""ID"" FROM ""BlocksetEntry"", ""Blockset"", ""Metadataset"", ""Block"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) AND ""Blockset"".""Length"" > 0 ORDER BY ""Blockset"".""Length"" ASC LIMIT 1") + .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) + .ExecuteScalarInt64(-1); + + return res; + } + public virtual void Dispose() { if (IsDisposed) diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index 5ce5a498c..9db3f367e 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -133,40 +133,6 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN cmd.ExecuteNonQuery(); } - /// - /// Returns the ID of an empty metadata blockset. If no empty blockset is found, it returns the ID of the smallest blockset that is not in the given block volume IDs. - /// If no such blockset is found, it returns -1. - /// - /// The volume ids to ignore when searching for a suitable metadata block - /// The hash of the empty blockset - /// The size of the empty blockset - /// The transaction to use for the query - /// The ID of the empty metadata blockset, or -1 if no suitable blockset is found - public long GetEmptyMetadataBlocksetId(IEnumerable blockVolumeIds, string emptyHash, long emptyHashSize, IDbTransaction? transaction) - { - using var cmd = Connection.CreateCommand(transaction, @"SELECT ""ID"" FROM ""Blockset"" WHERE ""FullHash"" = @EmptyHash AND ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))") - .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) - .SetParameterValue("@EmptyHash", emptyHash) - .SetParameterValue("@EmptyHashSize", emptyHashSize); - - var res = cmd.ExecuteScalarInt64(-1); - - // No empty block found, try to find a zero-length block instead - if (res < 0 && emptyHashSize != 0) - res = cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))") - .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) - .SetParameterValue("@EmptyHashSize", 0) - .ExecuteScalarInt64(-1); - - // No empty block found, pick the smallest one - if (res < 0) - res = cmd.SetCommandAndParameters(@"SELECT ""Blockset"".""ID"" FROM ""BlocksetEntry"", ""Blockset"", ""Metadataset"", ""Block"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) ORDER BY ""Blockset"".""Length"" ASC LIMIT 1") - .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) - .ExecuteScalarInt64(-1); - - return res; - } - /// /// Replaces the metadata blockset ID in the Metadataset table with the empty blockset ID for all fileset entries that are not in any block volume. /// This is used to clean up the metadata blocksets that are now missing. diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index 818609ed7..c184ecef5 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -990,6 +990,99 @@ ORDER BY foreach (var rd in cmd.ExecuteReaderEnumerable()) yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); } + + public void FixEmptyMetadatasets(Options options) + { + using var tr = m_connection.BeginTransactionSafe(); + using var cmd = m_connection.CreateCommand(tr, + @"SELECT COUNT(*) + FROM Metadataset + JOIN Blockset ON Metadataset.BlocksetID = Blockset.ID + WHERE Blockset.Length = 0"); + var emptyMetaCount = cmd.ExecuteScalarInt64(0); + if (emptyMetaCount <= 0) + return; + + Logging.Log.WriteInformationMessage(LOGTAG, "ZeroLengthMetadata", "Found {0} zero-length metadata entries", emptyMetaCount); + + // Create replacement metadata + var emptyMeta = Utility.WrapMetadata(new Dictionary(), options); + var emptyBlocksetId = GetEmptyMetadataBlocksetId(Array.Empty(), emptyMeta.FileHash, emptyMeta.Blob.Length, tr); + if (emptyBlocksetId < 0) + throw new Interface.UserInformationException( + "Failed to locate an empty metadata blockset to replace missing metadata. Set the option --disable-replace-missing-metadata=true to ignore this and drop files with missing metadata.", + "FailedToLocateEmptyMetadataBlockset"); + + // Step 1: Create temp table with Metadataset IDs referencing empty blocksets (excluding the one to keep) + var tablename = "FixMetadatasets-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + + try + { + cmd.SetCommandAndParameters( + @$"CREATE TEMP TABLE ""{tablename}"" AS + SELECT m.ID AS MetadataID, m.BlocksetID + FROM Metadataset m + JOIN Blockset b ON m.BlocksetID = b.ID + WHERE b.Length = 0 + AND m.BlocksetID != @KeepBlockset") + .SetParameterValue("@KeepBlockset", emptyBlocksetId) + .ExecuteNonQuery(); + + // Step 2: Update FileLookup to use a valid metadata ID + cmd.SetCommandAndParameters( + @$"UPDATE FileLookup + SET MetadataID = ( + SELECT ID FROM Metadataset + WHERE BlocksetID = @KeepBlockset + LIMIT 1 + ) + WHERE MetadataID IN (SELECT MetadataID FROM ""{tablename}"")") + .SetParameterValue("@KeepBlockset", emptyBlocksetId) + .ExecuteNonQuery(); + + // Step 3: Delete obsolete Metadataset entries + cmd.SetCommandAndParameters( + @$"DELETE FROM Metadataset + WHERE ID IN (SELECT MetadataID FROM ""{tablename}"")") + .ExecuteNonQuery(); + + // Step 4: Delete orphaned blocksets (affected only) + cmd.SetCommandAndParameters( + @$"DELETE FROM Blockset + WHERE ID IN (SELECT BlocksetID FROM ""{tablename}"") + AND NOT EXISTS (SELECT 1 FROM Metadataset WHERE BlocksetID = Blockset.ID) + AND NOT EXISTS (SELECT 1 FROM File WHERE BlocksetID = Blockset.ID)") + .ExecuteNonQuery(); + + // Step 5: Confirm all broken metadata entries are resolved + cmd.SetCommandAndParameters( + @"SELECT COUNT(*) + FROM Metadataset + JOIN Blockset ON Metadataset.BlocksetID = Blockset.ID + WHERE Blockset.Length = 0"); + + var remaining = cmd.ExecuteScalarInt64(0); + if (remaining > 0) + throw new Interface.UserInformationException( + "Some zero-length metadata entries could not be repaired.", + "MetadataRepairFailed"); + + Logging.Log.WriteInformationMessage(LOGTAG, "ZeroLengthMetadataRepaired", "Zero length metadata entries repaired successfully"); + tr.Commit(); + } + finally + { + try + { + cmd.SetCommandAndParameters(FormatInvariant($@"DROP TABLE IF EXISTS ""{tablename}"" ")) + .ExecuteNonQuery(); + } + catch (Exception ex) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "ErrorDroppingTempTable", ex, "Failed to drop temporary table {0}: {1}", tablename, ex.Message); + } + } + } } } diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index e609bbfc6..1e36d7e94 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -1157,6 +1157,7 @@ namespace Duplicati.Library.Main.Operation db.FixDuplicateFileentries(); db.FixDuplicateBlocklistHashes(m_options.Blocksize, m_options.BlockhashSize); db.FixMissingBlocklistHashes(m_options.BlockHashAlgorithm, m_options.Blocksize); + db.FixEmptyMetadatasets(m_options); } } } diff --git a/Duplicati/UnitTest/RepairHandlerTests.cs b/Duplicati/UnitTest/RepairHandlerTests.cs index 907d5ccae..f99b8b910 100644 --- a/Duplicati/UnitTest/RepairHandlerTests.cs +++ b/Duplicati/UnitTest/RepairHandlerTests.cs @@ -635,5 +635,41 @@ namespace Duplicati.UnitTest Assert.AreEqual(fileLookupEntries, cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM FileLookup")); } } + + [Test] + [Category("RepairHandler")] + public void RepairReplacesZeroLengthMetadata() + { + var options = new Dictionary(this.TestOptions); + File.WriteAllText(Path.Combine(this.DATAFOLDER, "a.txt"), "a"); + File.WriteAllText(Path.Combine(this.DATAFOLDER, "b.txt"), "b"); + using (var c = new Controller("file://" + this.TARGETFOLDER, options, null)) + TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); + + using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0)) + using (var cmd = con.CreateCommand()) + { + var lookupId = cmd.ExecuteScalarInt64("SELECT ID FROM FileLookup LIMIT 1"); + var metaId = cmd.SetCommandAndParameters("SELECT MetadataID FROM FileLookup WHERE ID = @Id") + .SetParameterValue("@Id", lookupId).ExecuteScalarInt64(-1); + var blocksetId = cmd.SetCommandAndParameters("SELECT BlocksetID FROM Metadataset WHERE ID = @Id") + .SetParameterValue("@Id", metaId).ExecuteScalarInt64(-1); + cmd.SetCommandAndParameters("UPDATE Blockset SET Length = 0 WHERE ID = @Id") + .SetParameterValue("@Id", blocksetId).ExecuteNonQuery(); + } + + using (var c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + var res = c.Repair(); + Assert.AreEqual(0, res.Errors.Count()); + Assert.AreEqual(0, res.Warnings.Count()); + } + using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0)) + using (var cmd = con.CreateCommand()) + { + var remaining = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM Metadataset JOIN Blockset ON Metadataset.BlocksetID = Blockset.ID WHERE Blockset.Length = 0"); + Assert.AreEqual(0, remaining, "Zero-length metadata should have been replaced"); + } + } } } \ No newline at end of file