haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added a check and repair for empty metadata

Browse Source 
This PR adds a check for empty metadatasets (length = 0) and replaces such sets with a default metadataset.

This fixes #6350
This commit is contained in:
Kenneth Skovhede
2025-06-16 16:35:07 +02:00
parent 10c1c0443e
commit 2c5c01160c
5 changed files with 164 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Main/Database/LocalDatabase.cs
+34
View File
@@ -1733,6 +1733,40 @@ ORDER BY ""A"".""BlocksetID"", ""B"".""Index""
}
}
/// <summary>
/// Returns the ID of an empty metadata blockset. If no empty blockset is found, it returns the ID of the smallest blockset that is not in the given block volume IDs.
/// If no such blockset is found, it returns -1.
/// </summary>
/// <param name="blockVolumeIds">The volume ids to ignore when searching for a suitable metadata block</param>
/// <param name="emptyHash">The hash of the empty blockset</param>
/// <param name="emptyHashSize">The size of the empty blockset</param>
/// <param name="transaction">The transaction to use for the query</param>
/// <returns>The ID of the empty metadata blockset, or -1 if no suitable blockset is found</returns>
public long GetEmptyMetadataBlocksetId(IEnumerable<long> blockVolumeIds, string emptyHash, long emptyHashSize, IDbTransaction? transaction)
{
using var cmd = Connection.CreateCommand(transaction, @"SELECT ""ID"" FROM ""Blockset"" WHERE ""FullHash"" = @EmptyHash AND ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))")
.ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds)
.SetParameterValue("@EmptyHash", emptyHash)
.SetParameterValue("@EmptyHashSize", emptyHashSize);
var res = cmd.ExecuteScalarInt64(-1);
// No empty block found, try to find a zero-length block instead
if (res < 0 && emptyHashSize != 0)
res = cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))")
.ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds)
.SetParameterValue("@EmptyHashSize", 0)
.ExecuteScalarInt64(-1);
// No empty block found, pick the smallest one
if (res < 0)
res = cmd.SetCommandAndParameters(@"SELECT ""Blockset"".""ID"" FROM ""BlocksetEntry"", ""Blockset"", ""Metadataset"", ""Block"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) AND ""Blockset"".""Length"" > 0 ORDER BY ""Blockset"".""Length"" ASC LIMIT 1")
.ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds)
.ExecuteScalarInt64(-1);
return res;
}
public virtual void Dispose()
{
if (IsDisposed)
Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs
-34
View File
@@ -133,40 +133,6 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN
cmd.ExecuteNonQuery();
}
/// <summary>
/// Returns the ID of an empty metadata blockset. If no empty blockset is found, it returns the ID of the smallest blockset that is not in the given block volume IDs.
/// If no such blockset is found, it returns -1.
/// </summary>
/// <param name="blockVolumeIds">The volume ids to ignore when searching for a suitable metadata block</param>
/// <param name="emptyHash">The hash of the empty blockset</param>
/// <param name="emptyHashSize">The size of the empty blockset</param>
/// <param name="transaction">The transaction to use for the query</param>
/// <returns>The ID of the empty metadata blockset, or -1 if no suitable blockset is found</returns>
public long GetEmptyMetadataBlocksetId(IEnumerable<long> blockVolumeIds, string emptyHash, long emptyHashSize, IDbTransaction? transaction)
{
using var cmd = Connection.CreateCommand(transaction, @"SELECT ""ID"" FROM ""Blockset"" WHERE ""FullHash"" = @EmptyHash AND ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))")
.ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds)
.SetParameterValue("@EmptyHash", emptyHash)
.SetParameterValue("@EmptyHashSize", emptyHashSize);
var res = cmd.ExecuteScalarInt64(-1);
// No empty block found, try to find a zero-length block instead
if (res < 0 && emptyHashSize != 0)
res = cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))")
.ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds)
.SetParameterValue("@EmptyHashSize", 0)
.ExecuteScalarInt64(-1);
// No empty block found, pick the smallest one
if (res < 0)
res = cmd.SetCommandAndParameters(@"SELECT ""Blockset"".""ID"" FROM ""BlocksetEntry"", ""Blockset"", ""Metadataset"", ""Block"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) ORDER BY ""Blockset"".""Length"" ASC LIMIT 1")
.ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds)
.ExecuteScalarInt64(-1);
return res;
}
/// <summary>
/// Replaces the metadata blockset ID in the Metadataset table with the empty blockset ID for all fileset entries that are not in any block volume.
/// This is used to clean up the metadata blocksets that are now missing.
Duplicati/Library/Main/Database/LocalRepairDatabase.cs
+93
View File
@@ -990,6 +990,99 @@ ORDER BY
foreach (var rd in cmd.ExecuteReaderEnumerable())
yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2));
}
public void FixEmptyMetadatasets(Options options)
{
using var tr = m_connection.BeginTransactionSafe();
using var cmd = m_connection.CreateCommand(tr,
@"SELECT COUNT(*)
FROM Metadataset
JOIN Blockset ON Metadataset.BlocksetID = Blockset.ID
WHERE Blockset.Length = 0");
var emptyMetaCount = cmd.ExecuteScalarInt64(0);
if (emptyMetaCount <= 0)
return;
Logging.Log.WriteInformationMessage(LOGTAG, "ZeroLengthMetadata", "Found {0} zero-length metadata entries", emptyMetaCount);
// Create replacement metadata
var emptyMeta = Utility.WrapMetadata(new Dictionary<string, string>(), options);
var emptyBlocksetId = GetEmptyMetadataBlocksetId(Array.Empty<long>(), emptyMeta.FileHash, emptyMeta.Blob.Length, tr);
if (emptyBlocksetId < 0)
throw new Interface.UserInformationException(
"Failed to locate an empty metadata blockset to replace missing metadata. Set the option --disable-replace-missing-metadata=true to ignore this and drop files with missing metadata.",
"FailedToLocateEmptyMetadataBlockset");
// Step 1: Create temp table with Metadataset IDs referencing empty blocksets (excluding the one to keep)
var tablename = "FixMetadatasets-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
try
{
cmd.SetCommandAndParameters(
@$"CREATE TEMP TABLE ""{tablename}"" AS
SELECT m.ID AS MetadataID, m.BlocksetID
FROM Metadataset m
JOIN Blockset b ON m.BlocksetID = b.ID
WHERE b.Length = 0
AND m.BlocksetID != @KeepBlockset")
.SetParameterValue("@KeepBlockset", emptyBlocksetId)
.ExecuteNonQuery();
// Step 2: Update FileLookup to use a valid metadata ID
cmd.SetCommandAndParameters(
@$"UPDATE FileLookup
SET MetadataID = (
SELECT ID FROM Metadataset
WHERE BlocksetID = @KeepBlockset
LIMIT 1
)
WHERE MetadataID IN (SELECT MetadataID FROM ""{tablename}"")")
.SetParameterValue("@KeepBlockset", emptyBlocksetId)
.ExecuteNonQuery();
// Step 3: Delete obsolete Metadataset entries
cmd.SetCommandAndParameters(
@$"DELETE FROM Metadataset
WHERE ID IN (SELECT MetadataID FROM ""{tablename}"")")
.ExecuteNonQuery();
// Step 4: Delete orphaned blocksets (affected only)
cmd.SetCommandAndParameters(
@$"DELETE FROM Blockset
WHERE ID IN (SELECT BlocksetID FROM ""{tablename}"")
AND NOT EXISTS (SELECT 1 FROM Metadataset WHERE BlocksetID = Blockset.ID)
AND NOT EXISTS (SELECT 1 FROM File WHERE BlocksetID = Blockset.ID)")
.ExecuteNonQuery();
// Step 5: Confirm all broken metadata entries are resolved
cmd.SetCommandAndParameters(
@"SELECT COUNT(*)
FROM Metadataset
JOIN Blockset ON Metadataset.BlocksetID = Blockset.ID
WHERE Blockset.Length = 0");
var remaining = cmd.ExecuteScalarInt64(0);
if (remaining > 0)
throw new Interface.UserInformationException(
"Some zero-length metadata entries could not be repaired.",
"MetadataRepairFailed");
Logging.Log.WriteInformationMessage(LOGTAG, "ZeroLengthMetadataRepaired", "Zero length metadata entries repaired successfully");
tr.Commit();
}
finally
{
try
{
cmd.SetCommandAndParameters(FormatInvariant($@"DROP TABLE IF EXISTS ""{tablename}"" "))
.ExecuteNonQuery();
}
catch (Exception ex)
{
Logging.Log.WriteVerboseMessage(LOGTAG, "ErrorDroppingTempTable", ex, "Failed to drop temporary table {0}: {1}", tablename, ex.Message);
}
}
}
}
}
Duplicati/Library/Main/Operation/RepairHandler.cs
+1
View File
@@ -1157,6 +1157,7 @@ namespace Duplicati.Library.Main.Operation
db.FixDuplicateFileentries();
db.FixDuplicateBlocklistHashes(m_options.Blocksize, m_options.BlockhashSize);
db.FixMissingBlocklistHashes(m_options.BlockHashAlgorithm, m_options.Blocksize);
db.FixEmptyMetadatasets(m_options);
}
}
}
Duplicati/UnitTest/RepairHandlerTests.cs
+36
View File
@@ -635,5 +635,41 @@ namespace Duplicati.UnitTest
Assert.AreEqual(fileLookupEntries, cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM FileLookup"));
}
}
[Test]
[Category("RepairHandler")]
public void RepairReplacesZeroLengthMetadata()
{
var options = new Dictionary<string, string>(this.TestOptions);
File.WriteAllText(Path.Combine(this.DATAFOLDER, "a.txt"), "a");
File.WriteAllText(Path.Combine(this.DATAFOLDER, "b.txt"), "b");
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER }));
using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0))
using (var cmd = con.CreateCommand())
{
var lookupId = cmd.ExecuteScalarInt64("SELECT ID FROM FileLookup LIMIT 1");
var metaId = cmd.SetCommandAndParameters("SELECT MetadataID FROM FileLookup WHERE ID = @Id")
.SetParameterValue("@Id", lookupId).ExecuteScalarInt64(-1);
var blocksetId = cmd.SetCommandAndParameters("SELECT BlocksetID FROM Metadataset WHERE ID = @Id")
.SetParameterValue("@Id", metaId).ExecuteScalarInt64(-1);
cmd.SetCommandAndParameters("UPDATE Blockset SET Length = 0 WHERE ID = @Id")
.SetParameterValue("@Id", blocksetId).ExecuteNonQuery();
}
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
var res = c.Repair();
Assert.AreEqual(0, res.Errors.Count());
Assert.AreEqual(0, res.Warnings.Count());
}
using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0))
using (var cmd = con.CreateCommand())
{
var remaining = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM Metadataset JOIN Blockset ON Metadataset.BlocksetID = Blockset.ID WHERE Blockset.Length = 0");
Assert.AreEqual(0, remaining, "Zero-length metadata should have been replaced");
}
}
}
}