Kenneth Skovhede
179d1ceb47
This PR has a large blast radius because it takes the final step and bumps up the Controller to be fully async. We have historically done a piece-by-piece update, so all operations were already async but the controller interface was kept synchronous. With this update, the controller is now fully async and all tests are updated. Most places where the new C# compiler warns about function names not ending in `Async` were also adressed, giving a massive refactor change. Functionally, no changes are done.
263 lines
12 KiB
C#
263 lines
12 KiB
C#
// Copyright (C) 2026, The Duplicati Team
|
|
// https://duplicati.com, hello@duplicati.com
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a
|
|
// copy of this software and associated documentation files (the "Software"),
|
|
// to deal in the Software without restriction, including without limitation
|
|
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
// and/or sell copies of the Software, and to permit persons to whom the
|
|
// Software is furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in
|
|
// all copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
|
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
// DEALINGS IN THE SOFTWARE.
|
|
|
|
using System;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using NUnit.Framework;
|
|
using Duplicati.Library.Main;
|
|
using Microsoft.Data.Sqlite;
|
|
using Assert = NUnit.Framework.Legacy.ClassicAssert;
|
|
using System.Threading.Tasks;
|
|
|
|
namespace Duplicati.UnitTest
|
|
{
|
|
/// <summary>
|
|
/// Test for Issue 6688: Recreate makes bad DB referencing Temporary Remotevolume ID if dindex lacks dblock
|
|
///
|
|
/// When recreating a database from remote files, if a dindex file references a dblock that is missing,
|
|
/// the recreate operation may complete with apparent success but the resulting database contains
|
|
/// invalid references - DeletedBlock entries pointing to Temporary RemoteVolume IDs.
|
|
/// </summary>
|
|
public class Issue6688 : BasicSetupHelper
|
|
{
|
|
[Test]
|
|
[Category("Issue6688")]
|
|
[Category("Recreate")]
|
|
public async Task RecreateDbShouldNotReferenceTemporaryVolumesInDeletedBlockAsync()
|
|
{
|
|
// Arrange
|
|
var testopts = TestOptions;
|
|
testopts["backup-test-samples"] = "0";
|
|
testopts["no-encryption"] = "true";
|
|
testopts["keep-versions"] = "0";
|
|
testopts["blocksize"] = "1024"; // Small block size for testing
|
|
|
|
var sourceFolder = Path.Combine(DATAFOLDER, "source");
|
|
Directory.CreateDirectory(sourceFolder);
|
|
|
|
// Step 1: Create a file with enough content to create blocks and perform initial backup
|
|
var testContent = string.Join("\n", Enumerable.Range(0, 100).Select(i => $"Line {i}: Test content for issue 6688 - this is test data to ensure we have enough content to create blocks."));
|
|
File.WriteAllText(Path.Combine(sourceFolder, "test.txt"), testContent);
|
|
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
{
|
|
var res = await c.BackupAsync(new[] { sourceFolder });
|
|
Assert.AreEqual(0, res.Errors.Count(), "Initial backup should succeed");
|
|
}
|
|
|
|
// Get the dblock files (there may be multiple with small blocksize)
|
|
var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*");
|
|
Assert.IsTrue(dblockFiles.Length > 0, "Should have at least one dblock file");
|
|
var dblockPath = dblockFiles[0];
|
|
|
|
// Get the dindex files
|
|
var dindexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex.*");
|
|
Assert.IsTrue(dindexFiles.Length > 0, "Should have at least one dindex file");
|
|
|
|
// Step 2: Delete the local database to force recreate
|
|
File.Delete(testopts["dbpath"]);
|
|
|
|
// Step 3: Delete the dblock file (simulate corruption/missing file)
|
|
// Keep the dindex file which now references a missing dblock
|
|
File.Delete(dblockPath);
|
|
|
|
await TestContext.Progress.WriteLineAsync($"Deleted dblock: {Path.GetFileName(dblockPath)}");
|
|
await TestContext.Progress.WriteLineAsync($"Remaining dindex files: {string.Join(", ", dindexFiles.Select(Path.GetFileName))}");
|
|
|
|
// Step 4: Recreate the database
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
{
|
|
try
|
|
{
|
|
var res = await c.RepairAsync();
|
|
await TestContext.Progress.WriteLineAsync($"Repair result: {res.ParsedResult}");
|
|
|
|
// The operation may succeed with warnings, but we need to verify DB state
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// It's acceptable for repair to fail with a clear error
|
|
await TestContext.Progress.WriteLineAsync($"Repair failed: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
// Step 5: Verify the database does not contain invalid references
|
|
VerifyNoInvalidDeletedBlockReferences(testopts["dbpath"]);
|
|
}
|
|
|
|
[Test]
|
|
[Category("Issue6688")]
|
|
[Category("Recreate")]
|
|
public async Task RecreateDbShouldNotReferenceTemporaryVolumes_WhenDindexReferencesMissingDblockAsync()
|
|
{
|
|
// This test more closely reproduces the issue scenario:
|
|
// 1. Create two backups (versions)
|
|
// 2. Delete the first version's dblock file
|
|
// 3. Keep the first version's dindex file (which references the deleted dblock)
|
|
// 4. Delete the database
|
|
// 5. Recreate - this should handle the missing dblock gracefully
|
|
|
|
var testopts = TestOptions;
|
|
testopts["backup-test-samples"] = "0";
|
|
testopts["no-encryption"] = "true";
|
|
testopts["keep-versions"] = "0";
|
|
|
|
var sourceFolder = Path.Combine(DATAFOLDER, "source");
|
|
Directory.CreateDirectory(sourceFolder);
|
|
|
|
// Step 1: Create file A and backup (Version 1)
|
|
File.WriteAllText(Path.Combine(sourceFolder, "A.txt"), "Content A - version 1");
|
|
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
{
|
|
var res = await c.BackupAsync(new[] { sourceFolder });
|
|
Assert.AreEqual(0, res.Errors.Count(), "First backup should succeed");
|
|
}
|
|
|
|
// Save the dindex file from first backup
|
|
var dindexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex.*");
|
|
Assert.IsTrue(dindexFiles.Length > 0, "Should have dindex after first backup");
|
|
var dindex1Path = dindexFiles[0];
|
|
var savedDindexPath = Path.Combine(DATAFOLDER, "dindex1_backup.zip");
|
|
File.Copy(dindex1Path, savedDindexPath, true);
|
|
|
|
// Get the first backup's dblock
|
|
var dblockFiles1 = Directory.GetFiles(TARGETFOLDER, "*.dblock.*");
|
|
Assert.IsTrue(dblockFiles1.Length > 0, "Should have dblock after first backup");
|
|
var firstDblock = dblockFiles1[0];
|
|
|
|
// Step 2: Modify file and backup again (Version 2)
|
|
File.WriteAllText(Path.Combine(sourceFolder, "A.txt"), "Content A - version 2 - modified");
|
|
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
{
|
|
var res = await c.BackupAsync(new[] { sourceFolder });
|
|
Assert.AreEqual(0, res.Errors.Count(), "Second backup should succeed");
|
|
}
|
|
|
|
// Step 3: Delete the database and prepare for recreate
|
|
File.Delete(testopts["dbpath"]);
|
|
|
|
// Delete the first dblock file (simulating missing dblock)
|
|
File.Delete(firstDblock);
|
|
|
|
// Copy the old dindex back (it references the now-deleted dblock)
|
|
// Rename it to ensure it's processed early
|
|
var earlyDindexPath = Path.Combine(TARGETFOLDER, "duplicati-00000000000000000000000000000000.dindex.zip");
|
|
File.Copy(savedDindexPath, earlyDindexPath, true);
|
|
|
|
await TestContext.Progress.WriteLineAsync($"Deleted first dblock: {Path.GetFileName(firstDblock)}");
|
|
await TestContext.Progress.WriteLineAsync($"Restored old dindex: {Path.GetFileName(savedDindexPath)}");
|
|
|
|
// Step 4: Recreate the database
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
{
|
|
try
|
|
{
|
|
var res = await c.RepairAsync();
|
|
await TestContext.Progress.WriteLineAsync($"Repair result: {res.ParsedResult}");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
await TestContext.Progress.WriteLineAsync($"Repair failed: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
// Step 5: Verify the database does not contain invalid references
|
|
VerifyNoInvalidDeletedBlockReferences(testopts["dbpath"]);
|
|
}
|
|
|
|
private void VerifyNoInvalidDeletedBlockReferences(string dbPath)
|
|
{
|
|
using var connection = new SqliteConnection($"Data Source={dbPath};Pooling=false");
|
|
connection.Open();
|
|
var command = connection.CreateCommand();
|
|
|
|
// Check for DeletedBlock entries pointing to Temporary volumes
|
|
command.CommandText = @"
|
|
SELECT COUNT(*)
|
|
FROM DeletedBlock
|
|
WHERE VolumeID IN (SELECT ID FROM RemoteVolume WHERE State = 'Temporary')
|
|
";
|
|
var tempCount = (long)command.ExecuteScalar();
|
|
|
|
// Check for DeletedBlock entries with orphaned VolumeID references
|
|
command.CommandText = @"
|
|
SELECT COUNT(*)
|
|
FROM DeletedBlock
|
|
WHERE VolumeID NOT IN (SELECT ID FROM RemoteVolume)
|
|
";
|
|
var orphanedCount = (long)command.ExecuteScalar();
|
|
|
|
// Log diagnostic information
|
|
TestContext.Progress.WriteLine($"DeletedBlock entries referencing Temporary volumes: {tempCount}");
|
|
TestContext.Progress.WriteLine($"DeletedBlock entries with orphaned VolumeID: {orphanedCount}");
|
|
|
|
// List all RemoteVolumes and their states
|
|
command.CommandText = @"
|
|
SELECT ID, Name, State, Type FROM RemoteVolume ORDER BY Type, Name
|
|
";
|
|
using (var reader = command.ExecuteReader())
|
|
{
|
|
TestContext.Progress.WriteLine("\nRemoteVolume entries:");
|
|
while (reader.Read())
|
|
{
|
|
var id = reader.GetInt64(0);
|
|
var name = reader.GetString(1);
|
|
var state = reader.GetString(2);
|
|
var type = reader.GetString(3);
|
|
TestContext.Progress.WriteLine($" ID={id}, Name={name}, State={state}, Type={type}");
|
|
}
|
|
}
|
|
|
|
// List all DeletedBlock entries
|
|
command.CommandText = @"
|
|
SELECT db.ID, db.Hash, db.Size, db.VolumeID, rv.State
|
|
FROM DeletedBlock db
|
|
LEFT JOIN RemoteVolume rv ON db.VolumeID = rv.ID
|
|
";
|
|
using (var reader = command.ExecuteReader())
|
|
{
|
|
TestContext.Progress.WriteLine("\nDeletedBlock entries:");
|
|
while (reader.Read())
|
|
{
|
|
var id = reader.GetInt64(0);
|
|
var hash = reader.GetString(1);
|
|
var size = reader.GetInt64(2);
|
|
var volId = reader.GetInt64(3);
|
|
var state = reader.IsDBNull(4) ? "NULL (orphaned)" : reader.GetString(4);
|
|
TestContext.Progress.WriteLine($" ID={id}, Hash={hash[..Math.Min(16, hash.Length)]}..., Size={size}, VolumeID={volId}, VolumeState={state}");
|
|
}
|
|
}
|
|
|
|
// These assertions will fail if the bug exists
|
|
Assert.AreEqual(0, tempCount,
|
|
"DeletedBlock table should not contain entries referencing Temporary volumes. " +
|
|
$"Found {tempCount} invalid entries referencing Temporary volumes.");
|
|
|
|
Assert.AreEqual(0, orphanedCount,
|
|
"DeletedBlock table should not contain entries with orphaned VolumeID references. " +
|
|
$"Found {orphanedCount} orphaned entries.");
|
|
}
|
|
}
|
|
}
|