duplicati/Duplicati/UnitTest/EmptyMetadataTests.cs

// Copyright (C) 2026, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.

using System;
using System.Collections.Generic;
using System.IO;
using Duplicati.Library.Main;
using Duplicati.Library.Main.Database;
using Duplicati.Library.Interface;
using Duplicati.Library.SQLiteHelper;
using NUnit.Framework;
using System.Threading;
using System.Threading.Tasks;
using Duplicati.Library.Utility;

namespace Duplicati.UnitTest;

public class EmptyMetadataTests : BasicSetupHelper
{
    [Test]
    [Category("Targeted")]
    public async Task ReplaceMissingMetadataRestoresConsistencyAsync()
    {
        var testopts = TestOptions.Expand(new { no_encryption = true });

        // Create a simple folder structure
        Directory.CreateDirectory(Path.Combine(DATAFOLDER, "folder"));
        File.WriteAllText(Path.Combine(DATAFOLDER, "folder", "file.txt"), "data");

        using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
            TestUtils.AssertResults(await c.BackupAsync(new[] { DATAFOLDER }));

        long metaBlocksetId;
        long filesetId;

        using (var db = await SQLiteLoader.LoadConnectionAsync(DBFILE))
        using (var cmd = db.CreateCommand())
        {
            // Find metadata blockset ID and fileset ID for the backed up folder
            cmd.SetCommandAndParameters($@"SELECT M.""BlocksetID"", FE.""FilesetID""
                                         FROM ""File"" F
                                         JOIN ""FilesetEntry"" FE ON FE.""FileID"" = F.""ID""
                                         JOIN ""Metadataset"" M ON F.""MetadataID"" = M.""ID""
                                         WHERE F.""Path"" LIKE @Path AND F.""BlocksetID"" = {LocalDatabase.FOLDER_BLOCKSET_ID}
                                         ORDER BY FE.""FilesetID"" DESC LIMIT 1");
            cmd.SetParameterValue("@Path", "%folder%");
            using (var rd = await cmd.ExecuteReaderAsync())
            {
                Assert.That(await rd.ReadAsync(), Is.True, "Folder entry not found");
                metaBlocksetId = rd.ConvertValueToInt64(0);
                filesetId = rd.ConvertValueToInt64(1);
            }

            // Remove metadata blockset entries to simulate missing metadata
            cmd.SetCommandAndParameters("DELETE FROM \"BlocksetEntry\" WHERE \"BlocksetID\" = @Id");
            cmd.SetParameterValue("@Id", metaBlocksetId);
            await cmd.ExecuteNonQueryAsync();

            // Remove the now orphaned blockset record
            cmd.SetCommandAndParameters("DELETE FROM \"Blockset\" WHERE \"ID\" = @Id");
            cmd.SetParameterValue("@Id", metaBlocksetId);
            await cmd.ExecuteNonQueryAsync();
        }

        var opts = new Options(testopts);
        var emptyMeta = Duplicati.Library.Main.Utility.WrapMetadata(new Dictionary<string, string>(), opts);

        Assert.ThrowsAsync<DatabaseInconsistencyException>(async () =>
        {
            using var db = await LocalDatabase.CreateLocalDatabaseAsync(DBFILE, "verify", true, null, CancellationToken.None);
            await db.VerifyConsistencyAsync(opts.Blocksize, opts.BlockhashSize, true, CancellationToken.None);
        });

        await using (var db = await LocalListBrokenFilesDatabase.CreateAsync(DBFILE, null, CancellationToken.None).ConfigureAwait(false))
        {
            var blockVolumeIds = Array.Empty<long>();

            var emptyId = await db.GetEmptyMetadataBlocksetIdAsync(blockVolumeIds, emptyMeta.FileHash, emptyMeta.Blob.Length, CancellationToken.None);
            Assert.That(emptyId, Is.GreaterThanOrEqualTo(0), "Empty metadata blockset not found");

            var replaced = await db.ReplaceMetadataAsync(filesetId, emptyId, CancellationToken.None);
            Assert.That(replaced, Is.GreaterThan(0), "No metadata rows replaced");

            await db.Transaction.CommitAsync(CancellationToken.None).ConfigureAwait(false);
        }

        await using (var db = await LocalDatabase.CreateLocalDatabaseAsync(DBFILE, "verify", true, null, CancellationToken.None))
            await db.VerifyConsistencyAsync(opts.Blocksize, opts.BlockhashSize, true, CancellationToken.None);
    }

    /// <summary>
    /// Tests that GetEmptyMetadataBlocksetId works correctly with a large number of block volume IDs
    /// that triggers the temporary table code path (when count > CHUNK_SIZE = 128).
    /// </summary>
    [Test]
    [Category("Database")]
    public async Task GetEmptyMetadataBlocksetId_WithLargeInput_UsesTemporaryTable_Async()
    {
        using var dbfile = new TempFile();
        using var db = await SQLiteLoader.LoadConnectionAsync(dbfile);

        // Use DatabaseUpgrader to create the schema from embedded resources
        DatabaseUpgrader.UpgradeDatabase(db, dbfile, typeof(DatabaseSchemaMarker));

        using var cmd = db.CreateCommand();

        // Insert an operation record (required for LocalDatabase initialization)
        cmd.CommandText = @"INSERT INTO ""Operation"" (""Description"", ""Timestamp"") VALUES ('Test', 0)";
        await cmd.ExecuteNonQueryAsync();

        // Create 150 block volume IDs (exceeds CHUNK_SIZE of 128) to trigger temporary table path
        var blockVolumeIds = new List<long>();
        for (int i = 0; i < 150; i++)
        {
            blockVolumeIds.Add(i + 1);

            // Insert a remote volume for each block volume ID
            cmd.CommandText = $@"
                INSERT INTO ""Remotevolume"" (""ID"", ""OperationID"", ""Name"", ""Type"", ""State"", ""VerificationCount"", ""DeleteGraceTime"", ""ArchiveTime"", ""LockExpirationTime"")
                VALUES ({i + 1}, 1, 'block-volume-{i}.zip', 'Blocks', 'Verified', 0, 0, 0, 0)";
            await cmd.ExecuteNonQueryAsync();
        }

        // Insert a blockset entry that represents empty metadata (hash of empty data)
        // This will be found when GetEmptyMetadataBlocksetId is called
        cmd.CommandText = @"
            INSERT INTO ""Blockset"" (""ID"", ""FullHash"", ""Length"")
            VALUES (1, 'da39a3ee5e6b4b0d3255bfef95601890afd80709', 0)";
        await cmd.ExecuteNonQueryAsync();

        // Close the connection so LocalDatabase can open it
        await db.CloseAsync();

        // Create LocalListBrokenFilesDatabase instance (which inherits GetEmptyMetadataBlocksetId from LocalDatabase)
        await using var localDb = await LocalListBrokenFilesDatabase.CreateAsync(dbfile, null, CancellationToken.None).ConfigureAwait(false);

        // Act: Call GetEmptyMetadataBlocksetId with 150 block volume IDs
        // This should trigger the temporary table code path
        var emptyId = await localDb.GetEmptyMetadataBlocksetIdAsync(blockVolumeIds, "da39a3ee5e6b4b0d3255bfef95601890afd80709", 0, CancellationToken.None);

        // Assert: Should find the empty metadata blockset (ID = 1)
        Assert.That(emptyId, Is.EqualTo(1));
    }
}