// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Linq;
using System.Threading.Tasks;
using System.Collections.Generic;
using Microsoft.Data.Sqlite;
using Duplicati.Library.Utility;
namespace Duplicati.Library.Main.Database
{
///
/// Represents a local database used for testing and verification.
/// Provides methods for creating test databases, updating verification counts, and comparing file, index, and block lists
/// to support integrity checks and remote volume verification during backup testing.
///
internal class LocalTestDatabase : LocalDatabase
{
///
/// Initializes a new instance of the class.
///
/// The path to the database file.
/// The size of the page cache in bytes.
/// An optional existing instance to use. Used to mimic constructor chaining.
/// A task that represents the asynchronous operation. The task result contains the created instance.
public static async Task CreateAsync(string path, long pagecachesize, LocalTestDatabase? dbnew = null)
{
dbnew ??= new LocalTestDatabase();
dbnew = (LocalTestDatabase)
await CreateLocalDatabaseAsync(path, "Test", true, pagecachesize, dbnew)
.ConfigureAwait(false);
dbnew.ShouldCloseConnection = true;
return dbnew;
}
///
/// Creates a new instance of the class using an existing parent database.
///
/// The parent database to use for creating the new test database.
/// An optional existing instance to use. Used to mimic constructor chaining.
/// A task that represents the asynchronous operation. The task result contains the created instance.
public static async Task CreateAsync(LocalDatabase dbparent, LocalTestDatabase? dbnew = null)
{
dbnew ??= new LocalTestDatabase();
return (LocalTestDatabase)
await CreateLocalDatabaseAsync(dbparent, dbnew)
.ConfigureAwait(false);
}
///
/// Updates the verification count for a remote volume with the specified name.
/// Increments the count by 1, or sets it to the maximum of 1 if it was previously 0 or negative.
///
/// The name of the remote volume to update.
/// A task that represents the asynchronous operation.
public async Task UpdateVerificationCount(string name)
{
await using var cmd = m_connection.CreateCommand(m_rtr);
await cmd.SetCommandAndParameters(@"
UPDATE ""RemoteVolume""
SET ""VerificationCount"" = MAX(1,
CASE
WHEN ""VerificationCount"" <= 0
THEN (
SELECT MAX(""VerificationCount"")
FROM ""RemoteVolume""
)
ELSE ""VerificationCount"" + 1
END
)
WHERE ""Name"" = @Name
")
.SetParameterValue("@Name", name)
.ExecuteNonQueryAsync()
.ConfigureAwait(false);
}
///
/// A record representing a remote volume, which implements the interface.
///
private record RemoteVolume : IRemoteVolume
{
///
/// Gets the ID of the remote volume.
///
public long ID { get; init; }
public string Name { get; init; }
public long Size { get; init; }
public string Hash { get; init; }
///
/// Gets the verification count of the remote volume, indicating how many times it has been verified.
///
public long VerificationCount { get; init; }
///
/// Initializes a new instance of the record from a SqliteDataReader.
///
/// The SqliteDataReader containing the data for the remote volume.
public RemoteVolume(SqliteDataReader rd)
{
ID = rd.ConvertValueToInt64(0);
Name = rd.ConvertValueToString(1) ?? "";
Size = rd.ConvertValueToInt64(2);
Hash = rd.ConvertValueToString(3) ?? throw new ArgumentNullException("Hash cannot be null");
VerificationCount = rd.ConvertValueToInt64(4);
}
}
///
/// Filters a list of remote volumes based on their verification count.
/// The method selects volumes that have not been verified, those with a low verification count, and finally those with a high verification count, ensuring a balanced selection.
///
/// The collection of remote volumes to filter.
/// The number of samples to select.
/// The maximum verification count to consider for filtering.
/// A list of remote volumes filtered by verification count.
private static List FilterByVerificationCount(IEnumerable volumes, long samples, long maxverification)
{
var rnd = new Random();
// First round is the new items
var res = (from n in volumes where n.VerificationCount == 0 select n).ToList();
while (res.Count > samples)
res.RemoveAt(rnd.Next(0, res.Count));
// Quick exit if we are done
if (res.Count == samples)
return res;
// Next is the volumes that are not
// verified as much, with preference for low verification count
var starved = (from n in volumes where n.VerificationCount != 0 && n.VerificationCount < maxverification orderby n.VerificationCount select n);
if (starved.Any())
{
var max = starved.Select(x => x.VerificationCount).Max();
var min = starved.Select(x => x.VerificationCount).Min();
for (var i = min; i <= max; i++)
{
var p = starved.Where(x => x.VerificationCount == i).ToList();
while (res.Count < samples && p.Count > 0)
{
var n = rnd.Next(0, p.Count);
res.Add(p[n]);
p.RemoveAt(n);
}
}
// Quick exit if we are done
if (res.Count == samples)
return res;
}
if (maxverification > 0)
{
// Last is the items that are verified mostly
var remainder = (from n in volumes where n.VerificationCount >= maxverification select n).ToList();
while (res.Count < samples && remainder.Count > 0)
{
var n = rnd.Next(0, remainder.Count);
res.Add(remainder[n]);
remainder.RemoveAt(n);
}
}
return res;
}
///
/// Asynchronously selects a set of remote volumes to be tested for integrity, based on the specified sample count and selection options.
/// This method retrieves candidate remote volumes from the database, prioritizes them according to verification count and state, and yields the selected targets for verification.
///
/// The number of remote volumes to select for testing.
/// The options that define selection criteria, such as time, version, and verification strategy.
/// An asynchronous enumerable of representing the selected test targets.
public async IAsyncEnumerable SelectTestTargets(long samples, Options options)
{
var tp = await GetFilelistWhereClause(options.Time, options.Version)
.ConfigureAwait(false);
samples = Math.Max(1, samples);
await using var cmd = m_connection.CreateCommand(m_rtr);
var files = new List();
var max = cmd.ExecuteScalarInt64(@"
SELECT MAX(""VerificationCount"")
FROM ""RemoteVolume""
", 0);
if (options.FullRemoteVerification != Options.RemoteTestStrategy.IndexesOnly)
{
// Select any broken items
cmd.SetCommandAndParameters(@"
SELECT
""ID"",
""Name"",
""Size"",
""Hash"",
""VerificationCount""
FROM
""Remotevolume""
WHERE
(""State"" IN (@States))
AND (
""Hash"" = ''
OR ""Hash"" IS NULL
OR ""Size"" <= 0
)
AND (""ArchiveTime"" = 0)
")
.ExpandInClauseParameterMssqlite("@States", [
RemoteVolumeState.Verified.ToString(),
RemoteVolumeState.Uploaded.ToString()
]);
await using (var rd = cmd.ExecuteReader())
while (rd.Read())
yield return new RemoteVolume(rd);
//First we select some filesets
var whereClause = string.IsNullOrEmpty(tp.Item1) ? " WHERE " : (" " + tp.Item1 + " AND ");
await using (var rd = cmd.SetCommandAndParameters(@$"
SELECT
""A"".""VolumeID"",
""A"".""Name"",
""A"".""Size"",
""A"".""Hash"",
""A"".""VerificationCount""
FROM
(
SELECT
""ID"" AS ""VolumeID"",
""Name"",
""Size"",
""Hash"",
""VerificationCount""
FROM ""Remotevolume""
WHERE
""ArchiveTime"" = 0
AND ""State"" IN (
@State1,
@State2
)
) ""A"",
""Fileset""
{whereClause}
""A"".""VolumeID"" = ""Fileset"".""VolumeID""
ORDER BY ""Fileset"".""Timestamp""
")
.SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString())
.SetParameterValue("@State2", RemoteVolumeState.Verified.ToString())
.SetParameterValues(tp.Item2)
.ExecuteReader())
while (rd.Read())
files.Add(new RemoteVolume(rd));
if (files.Count == 0)
yield break;
if (string.IsNullOrEmpty(tp.Item1))
files = FilterByVerificationCount(files, samples, max).ToList();
foreach (var f in files)
yield return f;
//Then we select some index files
files.Clear();
}
cmd.SetCommandAndParameters(@"
SELECT
""ID"",
""Name"",
""Size"",
""Hash"",
""VerificationCount""
FROM ""Remotevolume""
WHERE
""Type"" = @Type
AND ""State"" IN (@States)
AND ""ArchiveTime"" = 0
")
.SetParameterValue("@Type", RemoteVolumeType.Index.ToString())
.ExpandInClauseParameterMssqlite("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]);
await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false))
while (await rd.ReadAsync().ConfigureAwait(false))
files.Add(new RemoteVolume(rd));
foreach (var f in FilterByVerificationCount(files, samples, max))
yield return f;
if (options.FullRemoteVerification == Options.RemoteTestStrategy.ListAndIndexes || options.FullRemoteVerification == Options.RemoteTestStrategy.IndexesOnly)
yield break;
//And finally some block files
files.Clear();
cmd.SetCommandAndParameters(@"
SELECT
""ID"",
""Name"",
""Size"",
""Hash"",
""VerificationCount""
FROM ""Remotevolume""
WHERE
""Type"" = @Type
AND ""State"" IN (@States)
AND ""ArchiveTime"" = 0
")
.SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString())
.ExpandInClauseParameterMssqlite("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]);
await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false))
while (await rd.ReadAsync().ConfigureAwait(false))
files.Add(new RemoteVolume(rd));
foreach (var f in FilterByVerificationCount(files, samples, max))
yield return f;
}
///
/// Base class for basic lists used in the local test database.
/// Provides methods for creating temporary tables, inserting data, and disposing of resources.
///
private abstract class Basiclist : IDisposable, IAsyncDisposable
{
///
/// The database connection used for executing commands.
///
protected LocalDatabase m_db = null!;
///
/// The name of the volume associated with this list.
///
protected string m_volumename = null!;
///
/// The name of the temporary table used for this list.
///
protected string m_tablename = null!;
///
/// Command used for inserting data into the temporary table.
///
protected SqliteCommand m_insertCommand = null!;
///
/// Calling this constructor will throw an exception. Use the CreateAsync method instead.
///
[Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")]
protected Basiclist(SqliteConnection connection, ReusableTransaction rtr, string volumename, string tablePrefix, string tableFormat, string insertCommand)
{
throw new NotSupportedException("Use CreateAsync method instead.");
}
///
/// Protected constructor to allow derived classes to initialize without parameters.
///
protected Basiclist() { }
///
/// Creates a new instance of the class asynchronously.
///
/// The instance of the to initialize.
/// The local database to use for the list.
/// The name of the volume associated with this list.
/// The prefix for the temporary table name.
/// The SQL format for creating the temporary table.
/// The SQL command for inserting data into the temporary table.
/// A task that represents the asynchronous operation. The task result contains the initialized instance.
protected static async Task CreateAsync(Basiclist bl, LocalDatabase db, string volumename, string tablePrefix, string tableFormat, string insertCommand)
{
bl.m_db = db;
bl.m_volumename = volumename;
var tablename = tablePrefix + "-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
await using (var cmd = bl.m_db.Connection.CreateCommand(bl.m_db.Transaction))
{
await cmd.ExecuteNonQueryAsync($@"
CREATE TEMPORARY TABLE ""{tablename}""
{tableFormat}
")
.ConfigureAwait(false);
bl.m_tablename = tablename;
}
bl.m_insertCommand = await bl.m_db.Connection.CreateCommandAsync($@"
INSERT INTO ""{bl.m_tablename}""
{insertCommand}
")
.ConfigureAwait(false);
return bl;
}
public void Dispose()
{
DisposeAsync().AsTask().Await();
}
public virtual async ValueTask DisposeAsync()
{
if (m_tablename != null)
try
{
await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction.Transaction);
await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tablename}""")
.ConfigureAwait(false);
}
catch { }
finally { m_tablename = null!; }
await m_insertCommand.DisposeAsync().ConfigureAwait(false);
}
}
///
/// Interface for a file list used in the local test database.
/// Provides methods for adding entries, comparing the list with remote volumes, and disposing of resources.
///
public interface IFilelist : IDisposable, IAsyncDisposable
{
///
/// Asynchronously adds a file entry to the file list.
///
/// The path of the file.
/// The size of the file in bytes.
/// The hash of the file, or null if not applicable.
/// The size of the metadata associated with the file.
/// The hash of the metadata associated with the file.
/// A collection of blocklist hashes associated with the file.
/// The type of the file entry.
/// The timestamp of the file entry.
/// A task that completes when the file entry has been added.
Task Add(string path, long size, string hash, long metasize, string metahash, IEnumerable blocklistHashes, FilelistEntryType type, DateTime time);
///
/// Asynchronously compares the file list with remote volumes and yields differences.
///
/// An asynchronous enumerable of key-value pairs representing the comparison results, where the key is the test entry status and the value is the file path.
IAsyncEnumerable> Compare();
}
///
/// Implementation of the interface that manages a list of files in a local test database.
///
private class Filelist : Basiclist, IFilelist
{
///
/// The prefix for the temporary table name used for the file list.
///
private const string TABLE_PREFIX = "Filelist";
///
/// The SQL format for creating the temporary table used for the file list.
///
private const string TABLE_FORMAT = @"
(
""Path"" TEXT NOT NULL,
""Size"" INTEGER NOT NULL,
""Hash"" TEXT NULL,
""Metasize"" INTEGER NOT NULL,
""Metahash"" TEXT NOT NULL
)
";
///
/// The SQL command for inserting data into the temporary table used for the file list.
///
private const string INSERT_COMMAND = @"
(
""Path"",
""Size"",
""Hash"",
""Metasize"",
""Metahash""
)
VALUES (
@Path,
@Size,
@Hash,
@Metasize,
@Metahash
)
";
///
/// Calling this constructor will throw an exception. Use the CreateAsync method instead.
///
[Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")]
public Filelist(SqliteConnection connection, string volumename, ReusableTransaction rtr)
: base(connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND)
{
throw new NotSupportedException("Use CreateAsync method instead.");
}
///
/// Private constructor to allow derived classes to initialize without parameters and to prevent instantiation from outside, which should only be done through the CreateAsync method.
///
private Filelist() { }
///
/// Asynchronously creates a new instance of the class.
///
/// The local database to use for the file list.
/// The name of the volume associated with this file list.
/// A task that when awaited returns a new instance of the class.
public static async Task CreateAsync(LocalDatabase db, string volumename)
{
var bl = new Filelist();
return (Filelist)
await CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND)
.ConfigureAwait(false);
}
public async Task Add(string path, long size, string hash, long metasize, string metahash, IEnumerable blocklistHashes, FilelistEntryType type, DateTime time)
{
await m_insertCommand
.SetTransaction(m_db.Transaction)
.SetParameterValue("@Path", path)
.SetParameterValue("@Size", hash == null ? -1 : size)
.SetParameterValue("@Hash", hash)
.SetParameterValue("@Metasize", metasize)
.SetParameterValue("@Metahash", metahash)
.ExecuteNonQueryAsync()
.ConfigureAwait(false);
}
public async IAsyncEnumerable> Compare()
{
var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
var create = $@"
CREATE TEMPORARY TABLE ""{cmpName}"" AS
SELECT
""A"".""Path"" AS ""Path"",
CASE
WHEN ""B"".""Fullhash"" IS NULL
THEN -1
ELSE ""B"".""Length"" END AS ""Size"",
""B"".""Fullhash"" AS ""Hash"",
""C"".""Length"" AS ""Metasize"",
""C"".""Fullhash"" AS ""Metahash""
FROM (
SELECT
""File"".""Path"",
""File"".""BlocksetID"" AS ""FileBlocksetID"",
""Metadataset"".""BlocksetID"" AS ""MetadataBlocksetID""
FROM
""Remotevolume"",
""Fileset"",
""FilesetEntry"",
""File"",
""Metadataset""
WHERE
""Remotevolume"".""Name"" = @Name
AND ""Fileset"".""VolumeID"" = ""Remotevolume"".""ID""
AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID""
AND ""File"".""ID"" = ""FilesetEntry"".""FileID""
AND ""File"".""MetadataID"" = ""Metadataset"".""ID""
) ""A""
LEFT OUTER JOIN ""Blockset"" ""B""
ON ""B"".""ID"" = ""A"".""FileBlocksetID""
LEFT OUTER JOIN ""Blockset"" ""C""
ON ""C"".""ID""=""A"".""MetadataBlocksetID""
";
var extra = $@"
SELECT
@TypeExtra AS ""Type"",
""{m_tablename}"".""Path"" AS ""Path""
FROM ""{m_tablename}""
WHERE ""{m_tablename}"".""Path"" NOT IN (
SELECT ""Path""
FROM ""{cmpName}""
)";
var missing = $@"
SELECT
@TypeMissing AS ""Type"",
""Path"" AS ""Path""
FROM ""{cmpName}""
WHERE ""Path"" NOT IN (
SELECT ""Path""
FROM ""{m_tablename}""
)
";
var modified = $@"
SELECT
@TypeModified AS ""Type"",
""E"".""Path"" AS ""Path""
FROM
""{m_tablename}"" ""E"",
""{cmpName}"" ""D""
WHERE
""D"".""Path"" = ""E"".""Path""
AND (
""D"".""Size"" != ""E"".""Size""
OR ""D"".""Hash"" != ""E"".""Hash""
OR ""D"".""Metasize"" != ""E"".""Metasize""
OR ""D"".""Metahash"" != ""E"".""Metahash""
)
";
var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"" ";
await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction);
try
{
await cmd
.SetCommandAndParameters(create)
.SetParameterValue("@Name", m_volumename)
.ExecuteNonQueryAsync()
.ConfigureAwait(false);
cmd
.SetCommandAndParameters($"{extra} UNION {missing} UNION {modified}")
.SetParameterValue("@TypeExtra", (int)Interface.TestEntryStatus.Extra)
.SetParameterValue("@TypeMissing", (int)Interface.TestEntryStatus.Missing)
.SetParameterValue("@TypeModified", (int)Interface.TestEntryStatus.Modified);
await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false);
while (await rd.ReadAsync().ConfigureAwait(false))
yield return new KeyValuePair(
(Interface.TestEntryStatus)rd.ConvertValueToInt64(0),
rd.ConvertValueToString(1) ?? ""
);
}
finally
{
try
{
await cmd
.ExecuteNonQueryAsync(drop)
.ConfigureAwait(false);
}
catch { }
}
}
}
///
/// Interface for an index list used in the local test database.
/// Provides methods for adding block links, comparing the index list with remote volumes, and disposing of resources.
///
public interface IIndexlist : IDisposable, IAsyncDisposable
{
///
/// Asynchronously adds a block link to the index list.
///
/// The name of the file associated with the block link.
/// The hash of the block link.
/// The length of the block link in bytes.
/// A task that completes when the block link has been added.
Task AddBlockLink(string filename, string hash, long length);
///
/// Asynchronously compares the index list with remote volumes and yields differences.
///
/// An asynchronous enumerable of key-value pairs representing the comparison results, where the key is the test entry status and the value is the file path.
IAsyncEnumerable> Compare();
}
///
/// Implementation of the interface that manages a list of index entries in a local test database.
///
private class Indexlist : Basiclist, IIndexlist
{
///
/// The prefix for the temporary table name used for the index list.
///
private const string TABLE_PREFIX = "Indexlist";
///
/// The SQL format for creating the temporary table used for the index list.
///
private const string TABLE_FORMAT = @"
(
""Name"" TEXT NOT NULL,
""Hash"" TEXT NOT NULL,
""Size"" INTEGER NOT NULL
)
";
///
/// The SQL command for inserting data into the temporary table used for the index list.
///
private const string INSERT_COMMAND = @"
(
""Name"",
""Hash"",
""Size""
)
VALUES (
@Name,
@Hash,
@Size
)
";
///
/// Calling this constructor will throw an exception. Use the CreateAsync method instead.
///
[Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")]
public Indexlist(SqliteConnection connection, string volumename, ReusableTransaction rtr)
: base(connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND)
{
throw new NotSupportedException("Use CreateAsync method instead.");
}
///
/// Private constructor to allow derived classes to initialize without parameters and to prevent instantiation from outside, which should only be done through the CreateAsync method.
///
private Indexlist() { }
///
/// Asynchronously creates a new instance of the class.
///
/// The local database to use for the index list.
/// The name of the volume associated with this index list.
/// A task that when awaited returns a new instance of the class.
public static async Task CreateAsync(LocalDatabase db, string volumename)
{
var bl = new Indexlist();
return (Indexlist)
await CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND)
.ConfigureAwait(false);
}
public async Task AddBlockLink(string filename, string hash, long length)
{
await m_insertCommand
.SetTransaction(m_db.Transaction)
.SetParameterValue("@Name", filename)
.SetParameterValue("@Hash", hash)
.SetParameterValue("@Size", length)
.ExecuteNonQueryAsync()
.ConfigureAwait(false);
}
public async IAsyncEnumerable> Compare()
{
var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
var create = $@"
CREATE TEMPORARY TABLE ""{cmpName}"" AS
SELECT
""A"".""Name"",
""A"".""Hash"",
""A"".""Size""
FROM
""Remotevolume"" ""A"",
""Remotevolume"" ""B"",
""IndexBlockLink""
WHERE
""B"".""Name"" = @Name
AND ""A"".""ID"" = ""IndexBlockLink"".""BlockVolumeID""
AND ""B"".""ID"" = ""IndexBlockLink"".""IndexVolumeID""
";
var extra = $@"
SELECT
@TypeExtra AS ""Type"",
""{m_tablename}"".""Name"" AS ""Name""
FROM ""{m_tablename}""
WHERE ""{m_tablename}"".""Name"" NOT IN (
SELECT ""Name""
FROM ""{cmpName}""
)
";
var missing = $@"
SELECT
@TypeMissing AS ""Type"",
""Name"" AS ""Name""
FROM ""{cmpName}""
WHERE ""Name"" NOT IN (
SELECT ""Name""
FROM ""{m_tablename}""
)
";
var modified = $@"
SELECT
@TypeModified AS ""Type"",
""E"".""Name"" AS ""Name""
FROM
""{m_tablename}"" ""E"",
""{cmpName}"" ""D""
WHERE
""D"".""Name"" = ""E"".""Name""
AND (
""D"".""Hash"" != ""E"".""Hash""
OR ""D"".""Size"" != ""E"".""Size""
)
";
var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"" ";
await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction);
try
{
await cmd
.SetCommandAndParameters(create)
.SetParameterValue("@Name", m_volumename)
.ExecuteNonQueryAsync()
.ConfigureAwait(false);
cmd
.SetCommandAndParameters($"{extra} UNION {missing} UNION {modified}")
.SetParameterValue("@TypeExtra", (int)Interface.TestEntryStatus.Extra)
.SetParameterValue("@TypeMissing", (int)Interface.TestEntryStatus.Missing)
.SetParameterValue("@TypeModified", (int)Interface.TestEntryStatus.Modified);
await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false);
while (await rd.ReadAsync().ConfigureAwait(false))
yield return new KeyValuePair((Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? "");
}
finally
{
try
{
await cmd
.ExecuteNonQueryAsync(drop)
.ConfigureAwait(false);
}
catch { }
}
}
}
///
/// Interface for a blocklist used in the local test database.
/// Provides methods for adding blocks, comparing the blocklist with remote volumes, and disposing of resources.
///
public interface IBlocklist : IDisposable, IAsyncDisposable
{
///
/// Asynchronously adds a block to the blocklist.
///
/// The key (hash) of the block.
/// The size of the block in bytes.
/// A task that completes when the block has been added.
Task AddBlock(string key, long value);
///
/// Asynchronously compares the blocklist with remote volumes and yields differences.
///
/// An asynchronous enumerable of key-value pairs representing the comparison results, where the key is the test entry status and the value is the block hash.
IAsyncEnumerable> Compare();
}
public interface IBlocklistHashList : IDisposable, IAsyncDisposable
{
///
/// Asynchronously adds a block hash to the blocklist hash list.
///
/// The hash of the block.
/// The size of the block in bytes.
/// A task that completes when the block hash has been added.
Task AddBlockHash(string hash, long size);
///
/// Asynchronously compares the blocklist hash list with remote volumes and yields differences.
///
/// The number of hashes per block.
/// The size of each hash in bytes.
/// The size of each block in bytes.
/// An asynchronous enumerable of key-value pairs representing the comparison results, where the key is the test entry status and the value is the block hash.
IAsyncEnumerable> Compare(int hashesPerBlock, int hashSize, int blockSize);
}
///
/// Implementation of the interface that manages a list of blocks in a local test database.
/// Provides methods for adding blocks, comparing the blocklist with remote volumes, and disposing of resources.
///
private class Blocklist : Basiclist, IBlocklist
{
///
/// The prefix for the temporary table name used for the blocklist.
///
private const string TABLE_PREFIX = "Blocklist";
///
/// The SQL format for creating the temporary table used for the blocklist.
///
private const string TABLE_FORMAT = @"(
""Hash"" TEXT NOT NULL,
""Size"" INTEGER NOT NULL
)";
///
/// The SQL command for inserting data into the temporary table used for the blocklist.
///
private const string INSERT_COMMAND = @"(
""Hash"",
""Size""
)
VALUES (
@Hash,
@Size
)";
///
/// Calling this constructor will throw an exception. Use the CreateAsync method instead.
///
[Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")]
public Blocklist(SqliteConnection connection, string volumename, ReusableTransaction rtr)
{
throw new NotSupportedException("Use CreateAsync method instead.");
}
///
/// Private constructor to allow derived classes to initialize without parameters and to prevent instantiation from outside, which should only be done through the CreateAsync method.
///
private Blocklist() { }
///
/// Asynchronously creates a new instance of the class.
///
/// The local database to use for the blocklist.
/// The name of the volume associated with this blocklist.
public static async Task CreateAsync(LocalDatabase db, string volumename)
{
var bl = new Blocklist();
return (Blocklist)
await Basiclist.CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND)
.ConfigureAwait(false);
}
public async Task AddBlock(string hash, long size)
{
await m_insertCommand
.SetTransaction(m_db.Transaction)
.SetParameterValue("@Hash", hash)
.SetParameterValue("@Size", size)
.ExecuteNonQueryAsync()
.ConfigureAwait(false);
}
public async IAsyncEnumerable> Compare()
{
var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
var curBlocks = @"
SELECT
""Block"".""Hash"" AS ""Hash"",
""Block"".""Size"" AS ""Size""
FROM
""Remotevolume"",
""Block""
WHERE
""Remotevolume"".""Name"" = @Name
AND ""Remotevolume"".""ID"" = ""Block"".""VolumeID""
";
var duplBlocks = @"
SELECT
""Block"".""Hash"" AS ""Hash"",
""Block"".""Size"" AS ""Size""
FROM
""DuplicateBlock"",
""Block""
WHERE
""DuplicateBlock"".""VolumeID"" = (
SELECT ""ID""
FROM ""RemoteVolume""
WHERE ""Name"" = @Name
)
AND ""Block"".""ID"" = ""DuplicateBlock"".""BlockID""
";
var delBlocks = @"
SELECT
""DeletedBlock"".""Hash"" AS ""Hash"",
""DeletedBlock"".""Size"" AS ""Size""
FROM
""DeletedBlock"",
""RemoteVolume""
WHERE
""RemoteVolume"".""Name"" = @Name
AND ""RemoteVolume"".""ID"" = ""DeletedBlock"".""VolumeID""
";
var create = $@"
CREATE TEMPORARY TABLE ""{cmpName}"" AS
SELECT DISTINCT
""Hash"" AS ""Hash"",
""Size"" AS ""Size""
FROM (
{curBlocks}
UNION {delBlocks}
UNION {duplBlocks}
)
";
var extra = $@"
SELECT
@TypeExtra AS ""Type"",
""{m_tablename}"".""Hash"" AS ""Hash""
FROM ""{m_tablename}""
WHERE ""{m_tablename}"".""Hash"" NOT IN (
SELECT ""Hash""
FROM ""{cmpName}""
)
";
var missing = $@"
SELECT
@TypeMissing AS ""Type"",
""Hash"" AS ""Hash""
FROM ""{cmpName}""
WHERE ""Hash"" NOT IN (
SELECT ""Hash""
FROM ""{m_tablename}""
)
";
var modified = $@"
SELECT
@TypeModified AS ""Type"",
""E"".""Hash"" AS ""Hash""
FROM
""{m_tablename}"" E,
""{cmpName}"" D
WHERE
""D"".""Hash"" = ""E"".""Hash""
AND ""D"".""Size"" != ""E"".""Size""
";
var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"" ";
await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction);
try
{
await cmd
.SetCommandAndParameters(create)
.SetParameterValue("@Name", m_volumename)
.ExecuteNonQueryAsync()
.ConfigureAwait(false);
cmd
.SetCommandAndParameters($@"
{extra}
UNION {missing}
UNION {modified}
")
.SetParameterValue("@TypeExtra", (int)Library.Interface.TestEntryStatus.Extra)
.SetParameterValue("@TypeMissing", (int)Library.Interface.TestEntryStatus.Missing)
.SetParameterValue("@TypeModified", (int)Library.Interface.TestEntryStatus.Modified);
await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false);
while (await rd.ReadAsync().ConfigureAwait(false))
yield return new KeyValuePair((Duplicati.Library.Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? "");
}
finally
{
try
{
await cmd
.ExecuteNonQueryAsync(drop)
.ConfigureAwait(false);
}
catch { }
}
}
}
///
/// Implementation of the interface that manages a list of block hashes in a local test database.
/// Provides methods for adding block hashes, comparing the blocklist hash list with remote volumes, and disposing of resources.
///
private class BlocklistHashList : Basiclist, IBlocklistHashList
{
///
/// The prefix for the temporary table name used for the blocklist hash list.
///
private const string TABLE_PREFIX = "BlocklistHashList";
///
/// The SQL format for creating the temporary table used for the blocklist hash list.
///
private const string TABLE_FORMAT = @"(
""Hash"" TEXT NOT NULL,
""Size"" INTEGER NOT NULL
)";
///
/// The SQL command for inserting data into the temporary table used for the blocklist hash list.
///
private const string INSERT_COMMAND = @"(
""Hash"",
""Size""
)
VALUES (
@Hash,
@Size
)";
///
/// Calling this constructor will throw an exception. Use the CreateAsync method instead.
///
[Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")]
public BlocklistHashList(SqliteConnection connection, string volumename, ReusableTransaction rtr)
{
throw new NotSupportedException("Use CreateAsync method instead.");
}
///
/// Private constructor to allow derived classes to initialize without parameters and to prevent instantiation from outside, which should only be done through the CreateAsync method.
///
private BlocklistHashList() { }
///
/// Asynchronously creates a new instance of the class.
///
/// The local database to use for the blocklist hash list.
/// The name of the volume associated with this blocklist hash list.
/// A task that when awaited returns a new instance of the class.
public static async Task CreateAsync(LocalDatabase db, string volumename)
{
var bl = new BlocklistHashList();
return (BlocklistHashList)
await Basiclist.CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND)
.ConfigureAwait(false);
}
public async Task AddBlockHash(string hash, long size)
{
await m_insertCommand
.SetTransaction(m_db.Transaction)
.SetParameterValue("@Hash", hash)
.SetParameterValue("@Size", size)
.ExecuteNonQueryAsync()
.ConfigureAwait(false);
}
public async IAsyncEnumerable> Compare(int hashesPerBlock, int hashSize, int blockSize)
{
var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
var create = $@"
CREATE TEMPORARY TABLE ""{cmpName}"" (
""Hash"" TEXT NOT NULL,
""Size"" INTEGER NOT NULL
);
INSERT INTO ""{cmpName}"" (
""Hash"",
""Size""
)
SELECT
""b"".""Hash"",
""b"".""Size""
FROM ""Block"" ""b""
JOIN (
SELECT
""blh"".""Hash"",
CASE
WHEN ""blh"".""Index"" = (((""bs"".""Length"" + {blockSize} - 1) / {blockSize} - 1) / {hashesPerBlock})
AND ((""bs"".""Length"" + {blockSize} - 1) / {blockSize}) % {hashesPerBlock} != 0
THEN {hashSize} * ((""bs"".""Length"" + {blockSize} - 1) / {blockSize} % {hashesPerBlock})
ELSE {hashSize} * {hashesPerBlock}
END AS ""Size""
FROM ""BlocklistHash"" ""blh""
JOIN ""Blockset"" ""bs""
ON ""bs"".""ID"" = ""blh"".""BlocksetID""
) ""Expected""
ON
""b"".""Hash"" = ""Expected"".""Hash""
AND ""b"".""Size"" = ""Expected"".""Size""
WHERE ""b"".""VolumeID"" IN (
SELECT ""ibl"".""BlockVolumeID""
FROM ""Remotevolume"" ""idx""
JOIN IndexBlockLink ""ibl""
ON ""ibl"".""IndexVolumeID"" = ""idx"".""ID""
WHERE ""idx"".""Name"" = @Name
);
";
var compare = $@"
WITH
""Expected"" AS (
SELECT
""Hash"",
""Size""
FROM ""{cmpName}""
),
""Actual"" AS (
SELECT
""Hash"",
""Size""
FROM ""{m_tablename}""
),
""Extra"" AS (
SELECT
@TypeExtra AS ""Type"",
""a"".""Hash""
FROM ""Actual"" ""a""
LEFT JOIN ""Expected"" ""e""
ON
""a"".""Hash"" = ""e"".""Hash""
AND ""a"".""Size"" = ""e"".""Size""
WHERE ""e"".""Hash"" IS NULL
),
""Missing"" AS (
SELECT
@TypeMissing AS ""Type"",
""e"".""Hash""
FROM ""Expected"" ""e""
LEFT JOIN ""Actual"" ""a""
ON
""a"".""Hash"" = ""e"".""Hash""
AND ""a"".""Size"" = ""e"".""Size""
WHERE ""a"".""Hash"" IS NULL
),
""Modified"" AS (
SELECT
@TypeModified AS ""Type"",
""a"".""Hash""
FROM ""Actual"" ""a""
JOIN ""Expected"" ""e""
ON ""a"".""Hash"" = ""e"".""Hash""
WHERE
""a"".""Size"" != ""e"".""Size""
AND NOT EXISTS (
SELECT 1
FROM ""Extra"" ""x""
WHERE ""x"".""Hash"" = ""a"".""Hash""
)
)
SELECT *
FROM ""Extra""
UNION
SELECT *
FROM ""Missing""
UNION
SELECT *
FROM ""Modified"";
";
var drop = $@"DROP TABLE IF EXISTS ""{cmpName}""";
await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction);
try
{
// Create expected hash+size table filtered by volume
await cmd
.SetCommandAndParameters(create)
.SetParameterValue("@Name", m_volumename)
.ExecuteNonQueryAsync()
.ConfigureAwait(false);
// Compare against actual values inserted into temp table
cmd
.SetCommandAndParameters(compare)
.SetParameterValue("@TypeExtra", (int)Library.Interface.TestEntryStatus.Extra)
.SetParameterValue("@TypeMissing", (int)Library.Interface.TestEntryStatus.Missing)
.SetParameterValue("@TypeModified", (int)Library.Interface.TestEntryStatus.Modified);
await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false);
while (await rd.ReadAsync().ConfigureAwait(false))
yield return new KeyValuePair(
(Library.Interface.TestEntryStatus)rd.ConvertValueToInt64(0),
rd.ConvertValueToString(1) ?? "");
}
finally
{
try
{
await cmd
.ExecuteNonQueryAsync(drop)
.ConfigureAwait(false);
}
catch { }
}
}
}
///
/// Creates a new filelist in the local test database.
///
/// The name of the filelist to create.
/// A task that when awaited returns a new instance of the interface.
public async Task CreateFilelist(string name)
{
return await Filelist.CreateAsync(this, name).ConfigureAwait(false);
}
///
/// Creates a new indexlist in the local test database.
///
/// The name of the indexlist to create.
/// A task that when awaited returns a new instance of the interface.
public async Task CreateIndexlist(string name)
{
return await Indexlist.CreateAsync(this, name).ConfigureAwait(false);
}
///
/// Creates a new blocklist in the local test database.
///
/// The name of the blocklist to create.
/// A task that when awaited returns a new instance of the interface.
public async Task CreateBlocklist(string name)
{
return await Blocklist.CreateAsync(this, name).ConfigureAwait(false);
}
///
/// Creates a new blocklist hash list in the local test database.
///
/// The name of the blocklist hash list to create.
/// A task that when awaited returns a new instance of the interface.
public async Task CreateBlocklistHashList(string name)
{
return await BlocklistHashList.CreateAsync(this, name)
.ConfigureAwait(false);
}
}
}