haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix database inconsistent after metadata delete

Browse Source 
In the case of an interrupted upload, Duplicati will remove files that did not make it to the destination and then cleanup the database.

In the special case where multiple files were sharing exact metadata, the cleanup step could sometimes remove metadata that was needed. This would be detected and the repair rolled back, but the backup would fail to run.

With the fix the criteria for removing metadata has been narrowed to the correct subset.
This commit is contained in:
Kenneth Skovhede
2026-07-13 13:45:36 +02:00
parent 2238bb89e3
commit 2f65cdee22
2 changed files with 319 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Main/Database/Local/LocalDatabase.cs
+11 -8
View File
@@ -1106,7 +1106,14 @@ namespace Duplicati.Library.Main.Database.Local
FROM ""{blocksetidstable}""
";
// Create a temp table to associate metadata that is being deleted to a fileset
// Create a temp table to associate metadata that is being deleted to a fileset.
// Only metadata whose own blockset is on a deleted volume should be captured here.
// Files whose data blockset is being deleted are already handled by the
// blockset-based FilesetEntry/FileLookup deletion below — they do not need
// their metadata (which may be shared with other files) to be deleted too.
// Previously, an OR clause also captured metadata referenced by files whose
// data blockset was being deleted, which caused shared metadata to be
// incorrectly deleted, orphaning FilesetEntry records in other filesets.
var metadataFilesetQuery = $@"
SELECT
""Metadataset"".""ID"",
@@ -1117,18 +1124,14 @@ namespace Duplicati.Library.Main.Database.Local
INNER JOIN ""FilesetEntry""
ON ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID""
WHERE ""Metadataset"".""BlocksetID"" IN ({bsIdsSubQuery})
OR ""Metadataset"".""ID"" IN (
SELECT ""MetadataID""
FROM ""FileLookup""
WHERE ""BlocksetID"" IN ({bsIdsSubQuery})
)
";
var metadataFilesetTable = $"DelMetadataFilesetIds-{temptransguid}";
await deletecmd.ExecuteNonQueryAsync($@"
CREATE TEMP TABLE ""{metadataFilesetTable}"" (
""MetadataID"" INTEGER PRIMARY KEY,
""FilesetID"" INTEGER
""MetadataID"" INTEGER,
""FilesetID"" INTEGER,
CONSTRAINT ""PK_{metadataFilesetTable}"" PRIMARY KEY (""MetadataID"", ""FilesetID"")
)
", token)
.ConfigureAwait(false);
Duplicati/UnitTest/SharedMetadataRemoveRemoteVolumesTests.cs
+308
View File
@@ -0,0 +1,308 @@
// Copyright (C) 2026, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Duplicati.Library.DynamicLoader;
using Duplicati.Library.Main;
using Duplicati.Library.Main.Database.Local;
using NUnit.Framework;
using Assert = NUnit.Framework.Legacy.ClassicAssert;
namespace Duplicati.UnitTest
{
/// <summary>
/// Integration tests that reproduce the "Detected N file(s) in FilesetEntry
/// without corresponding FileLookup entry" error through real backup operations.
///
/// The bug occurs when <see cref="LocalDatabase.RemoveRemoteVolumesAsync"/> removes
/// a dblock volume whose data blocks belong to files that share metadata (Metadataset)
/// with files in other filesets. The metadata-based deletion logic over-deletes
/// shared metadata, orphaning FilesetEntry records.
/// </summary>
[TestFixture]
public class SharedMetadataRemoveRemoteVolumesTests : BasicSetupHelper
{
/// <summary>
/// Reproduces the issue by:
/// 1. Running a successful backup (Fileset 0) with files that share metadata
/// 2. Adding a new file with identical metadata (same size, timestamps) but different
/// data content, and running a backup that is interrupted during dblock upload,
/// leaving a dblock volume in Uploading state (registered in DB, not on remote)
/// 3. Running a third backup — its PreBackupVerify calls VerifyRemoteList which
/// detects the Uploading dblock (not on remote) and calls RemoveRemoteVolumesAsync
/// to clean it up
///
/// Before the fix: Step 3 throws ConstraintException because removing the Uploading
/// dblock incorrectly captures shared metadata (via the OR clause in
/// metadataFilesetQuery), causing FilesetEntry records from Fileset 0 to be orphaned.
///
/// After the fix: Step 3 succeeds — only the data blocks and their directly
/// associated FileLookup/FilesetEntry records are removed; shared metadata and
/// files in other filesets are preserved.
/// </summary>
[Test]
[Category("Disruption")]
public async Task InterruptedBackupWithSharedMetadataDoesNotCorruptDatabaseAsync()
{
var testopts = TestOptions;
testopts["blocksize"] = "1kb";
testopts["dblock-size"] = "50kb";
testopts["number-of-retries"] = "0";
testopts["no-encryption"] = "true";
// Create files with DIFFERENT data content but IDENTICAL metadata (same size,
// same timestamps, same permissions). This ensures:
// - Each file has unique data blocks (no data deduplication between files)
// - All files share the same Metadataset (metadata is deduplicated by hash)
//
// The shared metadata is the precondition for the bug: when the Uploading dblock
// containing file D's data blocks is removed, the OR clause in metadataFilesetQuery
// captures the shared Metadataset because file D references it, even though the
// metadata blockset itself is on a different (valid) volume.
var fixedTime = new DateTime(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc);
for (var i = 0; i < 3; i++)
{
var path = Path.Combine(DATAFOLDER, $"file-{i}.txt");
var data = new byte[2048];
new Random(i + 1).NextBytes(data);
File.WriteAllBytes(path, data);
File.SetLastWriteTime(path, fixedTime);
File.SetCreationTime(path, fixedTime);
}
// Step 1: Run a successful backup — this creates Fileset 0
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
TestUtils.AssertResults(await c.BackupAsync(new[] { DATAFOLDER }));
using (var c = new Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null))
TestUtils.AssertResults(await c.TestAsync(long.MaxValue));
// Step 2: Add a new file with IDENTICAL metadata (same size, same timestamps)
// but DIFFERENT data content. This file will share the Metadataset with the
// existing files, but its data blocks will be unique and go on a new dblock.
Thread.Sleep(1000);
var newFilePath = Path.Combine(DATAFOLDER, "file-new.txt");
var newData = new byte[2048];
new Random(999).NextBytes(newData);
File.WriteAllBytes(newFilePath, newData);
File.SetLastWriteTime(newFilePath, fixedTime);
File.SetCreationTime(newFilePath, fixedTime);
// Interrupt the backup by failing the dblock upload
BackendLoader.AddBackend(new DeterministicErrorBackend());
var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER;
var dblockUploadFailed = false;
DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) =>
{
if (action == DeterministicErrorBackend.BackendAction.PutBefore && remotename.Contains(".dblock."))
{
dblockUploadFailed = true;
return true;
}
return false;
};
using (var c = new Controller(failtarget, testopts, null))
{
var ex = Assert.CatchAsync(async () => await c.BackupAsync(new[] { DATAFOLDER }));
UnwrapAggregateException(ref ex);
Assert.That(ex, Is.TypeOf<DeterministicErrorBackend.DeterministicErrorBackendException>(),
$"Expected DeterministicErrorBackendException, got: {ex?.GetType().Name}: {ex?.Message}");
}
Assert.That(dblockUploadFailed, Is.True, "The dblock upload should have failed");
DeterministicErrorBackend.ErrorGenerator = null;
// Verify preconditions: metadata is shared across filesets, and the Uploading
// dblock contains data blocks for a file that shares that metadata
Console.WriteLine("=== Database state after interruption ===");
using (var db = await LocalDatabase.CreateLocalDatabaseAsync(testopts["dbpath"], "test", true, null, CancellationToken.None))
{
Console.WriteLine(TestUtils.DumpTable(db.Connection.CreateCommand(), "Remotevolume", null));
Console.WriteLine("--- Shared metadata across filesets ---");
using (var cmd = db.Connection.CreateCommand())
{
cmd.CommandText = @"
SELECT ""Metadataset"".""ID"", COUNT(DISTINCT ""FilesetEntry"".""FilesetID"") AS fileset_count, COUNT(*) AS file_count
FROM ""Metadataset""
INNER JOIN ""FileLookup"" ON ""FileLookup"".""MetadataID"" = ""Metadataset"".""ID""
INNER JOIN ""FilesetEntry"" ON ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID""
GROUP BY ""Metadataset"".""ID""
HAVING fileset_count > 1
";
using var reader = await cmd.ExecuteReaderAsync();
while (await reader.ReadAsync())
Console.WriteLine($" MetadataID={reader.GetInt64(0)}, filesets={reader.GetInt64(1)}, files={reader.GetInt64(2)}");
}
Console.WriteLine("--- Files using blocks on Uploading/Temporary volumes ---");
using (var cmd2 = db.Connection.CreateCommand())
{
cmd2.CommandText = @"
SELECT DISTINCT ""FileLookup"".""ID"", ""FileLookup"".""Path"", ""FileLookup"".""MetadataID""
FROM ""FileLookup""
INNER JOIN ""BlocksetEntry"" ON ""BlocksetEntry"".""BlocksetID"" = ""FileLookup"".""BlocksetID""
INNER JOIN ""Block"" ON ""Block"".""ID"" = ""BlocksetEntry"".""BlockID""
INNER JOIN ""Remotevolume"" ON ""Remotevolume"".""ID"" = ""Block"".""VolumeID""
WHERE ""Remotevolume"".""State"" IN ('Temporary', 'Uploading')
";
using var reader2 = await cmd2.ExecuteReaderAsync();
while (await reader2.ReadAsync())
Console.WriteLine($" File ID={reader2.GetInt64(0)}, Path={reader2.GetString(1)}, MetadataID={reader2.GetInt64(2)}");
}
}
// Step 3: Run a new backup. The PreBackupVerify step will:
// a) List remote files
// b) Find the Uploading dblock volume in the DB that is NOT on the remote
// c) Call RemoveRemoteVolumesAsync to clean up the orphaned volume
//
// Before the fix: This throws ConstraintException (or causes TestAsync failures)
// because the Uploading dblock contains data blocks for file-new.txt which
// shares a Metadataset with files in Fileset 0, and the metadata-based
// deletion over-deletes shared metadata.
// After the fix: This succeeds and the backup completes normally.
Thread.Sleep(1000);
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
TestUtils.AssertResults(await c.BackupAsync(new[] { DATAFOLDER }));
// Step 4: Verify the database is consistent
using (var c = new Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null))
TestUtils.AssertResults(await c.TestAsync(long.MaxValue));
// Step 5: Verify we have the expected number of filesets
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
{
var listResults = await c.ListAsync();
TestUtils.AssertResults(listResults);
Assert.GreaterOrEqual(listResults.Filesets.Count(), 2,
$"Expected at least 2 filesets, got {listResults.Filesets.Count()}");
}
}
/// <summary>
/// Reproduces the issue with multiple filesets sharing metadata and a larger
/// interrupted backup. Two successful backups create filesets with shared metadata,
/// then a third backup is interrupted, leaving dblock volumes in Uploading state.
/// The cleanup must not corrupt the database.
/// </summary>
[Test]
[Category("Disruption")]
public async Task InterruptedBackupWithMultipleFilesetsAndSharedMetadataDoesNotCorruptDatabaseAsync()
{
var testopts = TestOptions;
testopts["blocksize"] = "1kb";
testopts["dblock-size"] = "50kb";
testopts["number-of-retries"] = "0";
testopts["no-encryption"] = "true";
var fixedTime = new DateTime(2026, 1, 1, 12, 0, 0, DateTimeKind.Utc);
// Create initial files with different content but identical metadata
for (var i = 0; i < 3; i++)
{
var path = Path.Combine(DATAFOLDER, $"file-{i}.txt");
var data = new byte[2048];
new Random(i + 1).NextBytes(data);
File.WriteAllBytes(path, data);
File.SetLastWriteTime(path, fixedTime);
File.SetCreationTime(path, fixedTime);
}
// First successful backup
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
TestUtils.AssertResults(await c.BackupAsync(new[] { DATAFOLDER }));
// Second successful backup — modify a file so we get 2 filesets with shared metadata
Thread.Sleep(1000);
{
var path = Path.Combine(DATAFOLDER, "file-0.txt");
var data = new byte[2048];
new Random(100).NextBytes(data);
File.WriteAllBytes(path, data);
File.SetLastWriteTime(path, fixedTime);
File.SetCreationTime(path, fixedTime);
}
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
TestUtils.AssertResults(await c.BackupAsync(new[] { DATAFOLDER }));
// Now interrupt a third backup with new files that share metadata
Thread.Sleep(1000);
for (var i = 3; i < 6; i++)
{
var path = Path.Combine(DATAFOLDER, $"file-{i}.txt");
var data = new byte[2048];
new Random(i + 1).NextBytes(data);
File.WriteAllBytes(path, data);
File.SetLastWriteTime(path, fixedTime);
File.SetCreationTime(path, fixedTime);
}
BackendLoader.AddBackend(new DeterministicErrorBackend());
var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER;
DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) =>
{
if (action == DeterministicErrorBackend.BackendAction.PutBefore && remotename.Contains(".dblock."))
return true;
return false;
};
using (var c = new Controller(failtarget, testopts, null))
{
var ex = Assert.CatchAsync(async () => await c.BackupAsync(new[] { DATAFOLDER }));
UnwrapAggregateException(ref ex);
Assert.That(ex, Is.TypeOf<DeterministicErrorBackend.DeterministicErrorBackendException>());
}
DeterministicErrorBackend.ErrorGenerator = null;
// The cleanup backup must succeed without ConstraintException
Thread.Sleep(1000);
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
TestUtils.AssertResults(await c.BackupAsync(new[] { DATAFOLDER }));
using (var c = new Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null))
TestUtils.AssertResults(await c.TestAsync(long.MaxValue));
}
[TearDown]
public void TearDown()
{
DeterministicErrorBackend.ErrorGenerator = null;
}
private static void UnwrapAggregateException(ref Exception ex)
{
while (ex is AggregateException ae && ae.InnerException != null)
ex = ae.InnerException;
}
}
}