// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal in the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable using System; using System.Linq; using System.Threading.Tasks; using System.Collections.Generic; using Microsoft.Data.Sqlite; using Duplicati.Library.Utility; namespace Duplicati.Library.Main.Database { /// /// Represents a local database used for testing and verification. /// Provides methods for creating test databases, updating verification counts, and comparing file, index, and block lists /// to support integrity checks and remote volume verification during backup testing. /// internal class LocalTestDatabase : LocalDatabase { /// /// Initializes a new instance of the class. /// /// The path to the database file. /// The size of the page cache in bytes. /// An optional existing instance to use. Used to mimic constructor chaining. /// A task that represents the asynchronous operation. The task result contains the created instance. public static async Task CreateAsync(string path, long pagecachesize, LocalTestDatabase? dbnew = null) { dbnew ??= new LocalTestDatabase(); dbnew = (LocalTestDatabase) await CreateLocalDatabaseAsync(path, "Test", true, pagecachesize, dbnew) .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; return dbnew; } /// /// Creates a new instance of the class using an existing parent database. /// /// The parent database to use for creating the new test database. /// An optional existing instance to use. Used to mimic constructor chaining. /// A task that represents the asynchronous operation. The task result contains the created instance. public static async Task CreateAsync(LocalDatabase dbparent, LocalTestDatabase? dbnew = null) { dbnew ??= new LocalTestDatabase(); return (LocalTestDatabase) await CreateLocalDatabaseAsync(dbparent, dbnew) .ConfigureAwait(false); } /// /// Updates the verification count for a remote volume with the specified name. /// Increments the count by 1, or sets it to the maximum of 1 if it was previously 0 or negative. /// /// The name of the remote volume to update. /// A task that represents the asynchronous operation. public async Task UpdateVerificationCount(string name) { await using var cmd = m_connection.CreateCommand(m_rtr); await cmd.SetCommandAndParameters(@" UPDATE ""RemoteVolume"" SET ""VerificationCount"" = MAX(1, CASE WHEN ""VerificationCount"" <= 0 THEN ( SELECT MAX(""VerificationCount"") FROM ""RemoteVolume"" ) ELSE ""VerificationCount"" + 1 END ) WHERE ""Name"" = @Name ") .SetParameterValue("@Name", name) .ExecuteNonQueryAsync() .ConfigureAwait(false); } /// /// A record representing a remote volume, which implements the interface. /// private record RemoteVolume : IRemoteVolume { /// /// Gets the ID of the remote volume. /// public long ID { get; init; } public string Name { get; init; } public long Size { get; init; } public string Hash { get; init; } /// /// Gets the verification count of the remote volume, indicating how many times it has been verified. /// public long VerificationCount { get; init; } /// /// Initializes a new instance of the record from a SqliteDataReader. /// /// The SqliteDataReader containing the data for the remote volume. public RemoteVolume(SqliteDataReader rd) { ID = rd.ConvertValueToInt64(0); Name = rd.ConvertValueToString(1) ?? ""; Size = rd.ConvertValueToInt64(2); Hash = rd.ConvertValueToString(3) ?? throw new ArgumentNullException("Hash cannot be null"); VerificationCount = rd.ConvertValueToInt64(4); } } /// /// Filters a list of remote volumes based on their verification count. /// The method selects volumes that have not been verified, those with a low verification count, and finally those with a high verification count, ensuring a balanced selection. /// /// The collection of remote volumes to filter. /// The number of samples to select. /// The maximum verification count to consider for filtering. /// A list of remote volumes filtered by verification count. private static List FilterByVerificationCount(IEnumerable volumes, long samples, long maxverification) { var rnd = new Random(); // First round is the new items var res = (from n in volumes where n.VerificationCount == 0 select n).ToList(); while (res.Count > samples) res.RemoveAt(rnd.Next(0, res.Count)); // Quick exit if we are done if (res.Count == samples) return res; // Next is the volumes that are not // verified as much, with preference for low verification count var starved = (from n in volumes where n.VerificationCount != 0 && n.VerificationCount < maxverification orderby n.VerificationCount select n); if (starved.Any()) { var max = starved.Select(x => x.VerificationCount).Max(); var min = starved.Select(x => x.VerificationCount).Min(); for (var i = min; i <= max; i++) { var p = starved.Where(x => x.VerificationCount == i).ToList(); while (res.Count < samples && p.Count > 0) { var n = rnd.Next(0, p.Count); res.Add(p[n]); p.RemoveAt(n); } } // Quick exit if we are done if (res.Count == samples) return res; } if (maxverification > 0) { // Last is the items that are verified mostly var remainder = (from n in volumes where n.VerificationCount >= maxverification select n).ToList(); while (res.Count < samples && remainder.Count > 0) { var n = rnd.Next(0, remainder.Count); res.Add(remainder[n]); remainder.RemoveAt(n); } } return res; } /// /// Asynchronously selects a set of remote volumes to be tested for integrity, based on the specified sample count and selection options. /// This method retrieves candidate remote volumes from the database, prioritizes them according to verification count and state, and yields the selected targets for verification. /// /// The number of remote volumes to select for testing. /// The options that define selection criteria, such as time, version, and verification strategy. /// An asynchronous enumerable of representing the selected test targets. public async IAsyncEnumerable SelectTestTargets(long samples, Options options) { var tp = await GetFilelistWhereClause(options.Time, options.Version) .ConfigureAwait(false); samples = Math.Max(1, samples); await using var cmd = m_connection.CreateCommand(m_rtr); var files = new List(); var max = cmd.ExecuteScalarInt64(@" SELECT MAX(""VerificationCount"") FROM ""RemoteVolume"" ", 0); if (options.FullRemoteVerification != Options.RemoteTestStrategy.IndexesOnly) { // Select any broken items cmd.SetCommandAndParameters(@" SELECT ""ID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"" WHERE (""State"" IN (@States)) AND ( ""Hash"" = '' OR ""Hash"" IS NULL OR ""Size"" <= 0 ) AND (""ArchiveTime"" = 0) ") .ExpandInClauseParameterMssqlite("@States", [ RemoteVolumeState.Verified.ToString(), RemoteVolumeState.Uploaded.ToString() ]); await using (var rd = cmd.ExecuteReader()) while (rd.Read()) yield return new RemoteVolume(rd); //First we select some filesets var whereClause = string.IsNullOrEmpty(tp.Item1) ? " WHERE " : (" " + tp.Item1 + " AND "); await using (var rd = cmd.SetCommandAndParameters(@$" SELECT ""A"".""VolumeID"", ""A"".""Name"", ""A"".""Size"", ""A"".""Hash"", ""A"".""VerificationCount"" FROM ( SELECT ""ID"" AS ""VolumeID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"" WHERE ""ArchiveTime"" = 0 AND ""State"" IN ( @State1, @State2 ) ) ""A"", ""Fileset"" {whereClause} ""A"".""VolumeID"" = ""Fileset"".""VolumeID"" ORDER BY ""Fileset"".""Timestamp"" ") .SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()) .SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()) .SetParameterValues(tp.Item2) .ExecuteReader()) while (rd.Read()) files.Add(new RemoteVolume(rd)); if (files.Count == 0) yield break; if (string.IsNullOrEmpty(tp.Item1)) files = FilterByVerificationCount(files, samples, max).ToList(); foreach (var f in files) yield return f; //Then we select some index files files.Clear(); } cmd.SetCommandAndParameters(@" SELECT ""ID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"" WHERE ""Type"" = @Type AND ""State"" IN (@States) AND ""ArchiveTime"" = 0 ") .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()) .ExpandInClauseParameterMssqlite("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) while (await rd.ReadAsync().ConfigureAwait(false)) files.Add(new RemoteVolume(rd)); foreach (var f in FilterByVerificationCount(files, samples, max)) yield return f; if (options.FullRemoteVerification == Options.RemoteTestStrategy.ListAndIndexes || options.FullRemoteVerification == Options.RemoteTestStrategy.IndexesOnly) yield break; //And finally some block files files.Clear(); cmd.SetCommandAndParameters(@" SELECT ""ID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"" WHERE ""Type"" = @Type AND ""State"" IN (@States) AND ""ArchiveTime"" = 0 ") .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) .ExpandInClauseParameterMssqlite("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); await using (var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false)) while (await rd.ReadAsync().ConfigureAwait(false)) files.Add(new RemoteVolume(rd)); foreach (var f in FilterByVerificationCount(files, samples, max)) yield return f; } /// /// Base class for basic lists used in the local test database. /// Provides methods for creating temporary tables, inserting data, and disposing of resources. /// private abstract class Basiclist : IDisposable, IAsyncDisposable { /// /// The database connection used for executing commands. /// protected LocalDatabase m_db = null!; /// /// The name of the volume associated with this list. /// protected string m_volumename = null!; /// /// The name of the temporary table used for this list. /// protected string m_tablename = null!; /// /// Command used for inserting data into the temporary table. /// protected SqliteCommand m_insertCommand = null!; /// /// Calling this constructor will throw an exception. Use the CreateAsync method instead. /// [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] protected Basiclist(SqliteConnection connection, ReusableTransaction rtr, string volumename, string tablePrefix, string tableFormat, string insertCommand) { throw new NotSupportedException("Use CreateAsync method instead."); } /// /// Protected constructor to allow derived classes to initialize without parameters. /// protected Basiclist() { } /// /// Creates a new instance of the class asynchronously. /// /// The instance of the to initialize. /// The local database to use for the list. /// The name of the volume associated with this list. /// The prefix for the temporary table name. /// The SQL format for creating the temporary table. /// The SQL command for inserting data into the temporary table. /// A task that represents the asynchronous operation. The task result contains the initialized instance. protected static async Task CreateAsync(Basiclist bl, LocalDatabase db, string volumename, string tablePrefix, string tableFormat, string insertCommand) { bl.m_db = db; bl.m_volumename = volumename; var tablename = tablePrefix + "-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); await using (var cmd = bl.m_db.Connection.CreateCommand(bl.m_db.Transaction)) { await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{tablename}"" {tableFormat} ") .ConfigureAwait(false); bl.m_tablename = tablename; } bl.m_insertCommand = await bl.m_db.Connection.CreateCommandAsync($@" INSERT INTO ""{bl.m_tablename}"" {insertCommand} ") .ConfigureAwait(false); return bl; } public void Dispose() { DisposeAsync().AsTask().Await(); } public virtual async ValueTask DisposeAsync() { if (m_tablename != null) try { await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction.Transaction); await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tablename}""") .ConfigureAwait(false); } catch { } finally { m_tablename = null!; } await m_insertCommand.DisposeAsync().ConfigureAwait(false); } } /// /// Interface for a file list used in the local test database. /// Provides methods for adding entries, comparing the list with remote volumes, and disposing of resources. /// public interface IFilelist : IDisposable, IAsyncDisposable { /// /// Asynchronously adds a file entry to the file list. /// /// The path of the file. /// The size of the file in bytes. /// The hash of the file, or null if not applicable. /// The size of the metadata associated with the file. /// The hash of the metadata associated with the file. /// A collection of blocklist hashes associated with the file. /// The type of the file entry. /// The timestamp of the file entry. /// A task that completes when the file entry has been added. Task Add(string path, long size, string hash, long metasize, string metahash, IEnumerable blocklistHashes, FilelistEntryType type, DateTime time); /// /// Asynchronously compares the file list with remote volumes and yields differences. /// /// An asynchronous enumerable of key-value pairs representing the comparison results, where the key is the test entry status and the value is the file path. IAsyncEnumerable> Compare(); } /// /// Implementation of the interface that manages a list of files in a local test database. /// private class Filelist : Basiclist, IFilelist { /// /// The prefix for the temporary table name used for the file list. /// private const string TABLE_PREFIX = "Filelist"; /// /// The SQL format for creating the temporary table used for the file list. /// private const string TABLE_FORMAT = @" ( ""Path"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Hash"" TEXT NULL, ""Metasize"" INTEGER NOT NULL, ""Metahash"" TEXT NOT NULL ) "; /// /// The SQL command for inserting data into the temporary table used for the file list. /// private const string INSERT_COMMAND = @" ( ""Path"", ""Size"", ""Hash"", ""Metasize"", ""Metahash"" ) VALUES ( @Path, @Size, @Hash, @Metasize, @Metahash ) "; /// /// Calling this constructor will throw an exception. Use the CreateAsync method instead. /// [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] public Filelist(SqliteConnection connection, string volumename, ReusableTransaction rtr) : base(connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) { throw new NotSupportedException("Use CreateAsync method instead."); } /// /// Private constructor to allow derived classes to initialize without parameters and to prevent instantiation from outside, which should only be done through the CreateAsync method. /// private Filelist() { } /// /// Asynchronously creates a new instance of the class. /// /// The local database to use for the file list. /// The name of the volume associated with this file list. /// A task that when awaited returns a new instance of the class. public static async Task CreateAsync(LocalDatabase db, string volumename) { var bl = new Filelist(); return (Filelist) await CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) .ConfigureAwait(false); } public async Task Add(string path, long size, string hash, long metasize, string metahash, IEnumerable blocklistHashes, FilelistEntryType type, DateTime time) { await m_insertCommand .SetTransaction(m_db.Transaction) .SetParameterValue("@Path", path) .SetParameterValue("@Size", hash == null ? -1 : size) .SetParameterValue("@Hash", hash) .SetParameterValue("@Metasize", metasize) .SetParameterValue("@Metahash", metahash) .ExecuteNonQueryAsync() .ConfigureAwait(false); } public async IAsyncEnumerable> Compare() { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); var create = $@" CREATE TEMPORARY TABLE ""{cmpName}"" AS SELECT ""A"".""Path"" AS ""Path"", CASE WHEN ""B"".""Fullhash"" IS NULL THEN -1 ELSE ""B"".""Length"" END AS ""Size"", ""B"".""Fullhash"" AS ""Hash"", ""C"".""Length"" AS ""Metasize"", ""C"".""Fullhash"" AS ""Metahash"" FROM ( SELECT ""File"".""Path"", ""File"".""BlocksetID"" AS ""FileBlocksetID"", ""Metadataset"".""BlocksetID"" AS ""MetadataBlocksetID"" FROM ""Remotevolume"", ""Fileset"", ""FilesetEntry"", ""File"", ""Metadataset"" WHERE ""Remotevolume"".""Name"" = @Name AND ""Fileset"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""MetadataID"" = ""Metadataset"".""ID"" ) ""A"" LEFT OUTER JOIN ""Blockset"" ""B"" ON ""B"".""ID"" = ""A"".""FileBlocksetID"" LEFT OUTER JOIN ""Blockset"" ""C"" ON ""C"".""ID""=""A"".""MetadataBlocksetID"" "; var extra = $@" SELECT @TypeExtra AS ""Type"", ""{m_tablename}"".""Path"" AS ""Path"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Path"" NOT IN ( SELECT ""Path"" FROM ""{cmpName}"" )"; var missing = $@" SELECT @TypeMissing AS ""Type"", ""Path"" AS ""Path"" FROM ""{cmpName}"" WHERE ""Path"" NOT IN ( SELECT ""Path"" FROM ""{m_tablename}"" ) "; var modified = $@" SELECT @TypeModified AS ""Type"", ""E"".""Path"" AS ""Path"" FROM ""{m_tablename}"" ""E"", ""{cmpName}"" ""D"" WHERE ""D"".""Path"" = ""E"".""Path"" AND ( ""D"".""Size"" != ""E"".""Size"" OR ""D"".""Hash"" != ""E"".""Hash"" OR ""D"".""Metasize"" != ""E"".""Metasize"" OR ""D"".""Metahash"" != ""E"".""Metahash"" ) "; var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"" "; await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); try { await cmd .SetCommandAndParameters(create) .SetParameterValue("@Name", m_volumename) .ExecuteNonQueryAsync() .ConfigureAwait(false); cmd .SetCommandAndParameters($"{extra} UNION {missing} UNION {modified}") .SetParameterValue("@TypeExtra", (int)Interface.TestEntryStatus.Extra) .SetParameterValue("@TypeMissing", (int)Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Interface.TestEntryStatus.Modified); await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) yield return new KeyValuePair( (Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? "" ); } finally { try { await cmd .ExecuteNonQueryAsync(drop) .ConfigureAwait(false); } catch { } } } } /// /// Interface for an index list used in the local test database. /// Provides methods for adding block links, comparing the index list with remote volumes, and disposing of resources. /// public interface IIndexlist : IDisposable, IAsyncDisposable { /// /// Asynchronously adds a block link to the index list. /// /// The name of the file associated with the block link. /// The hash of the block link. /// The length of the block link in bytes. /// A task that completes when the block link has been added. Task AddBlockLink(string filename, string hash, long length); /// /// Asynchronously compares the index list with remote volumes and yields differences. /// /// An asynchronous enumerable of key-value pairs representing the comparison results, where the key is the test entry status and the value is the file path. IAsyncEnumerable> Compare(); } /// /// Implementation of the interface that manages a list of index entries in a local test database. /// private class Indexlist : Basiclist, IIndexlist { /// /// The prefix for the temporary table name used for the index list. /// private const string TABLE_PREFIX = "Indexlist"; /// /// The SQL format for creating the temporary table used for the index list. /// private const string TABLE_FORMAT = @" ( ""Name"" TEXT NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL ) "; /// /// The SQL command for inserting data into the temporary table used for the index list. /// private const string INSERT_COMMAND = @" ( ""Name"", ""Hash"", ""Size"" ) VALUES ( @Name, @Hash, @Size ) "; /// /// Calling this constructor will throw an exception. Use the CreateAsync method instead. /// [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] public Indexlist(SqliteConnection connection, string volumename, ReusableTransaction rtr) : base(connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) { throw new NotSupportedException("Use CreateAsync method instead."); } /// /// Private constructor to allow derived classes to initialize without parameters and to prevent instantiation from outside, which should only be done through the CreateAsync method. /// private Indexlist() { } /// /// Asynchronously creates a new instance of the class. /// /// The local database to use for the index list. /// The name of the volume associated with this index list. /// A task that when awaited returns a new instance of the class. public static async Task CreateAsync(LocalDatabase db, string volumename) { var bl = new Indexlist(); return (Indexlist) await CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) .ConfigureAwait(false); } public async Task AddBlockLink(string filename, string hash, long length) { await m_insertCommand .SetTransaction(m_db.Transaction) .SetParameterValue("@Name", filename) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", length) .ExecuteNonQueryAsync() .ConfigureAwait(false); } public async IAsyncEnumerable> Compare() { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); var create = $@" CREATE TEMPORARY TABLE ""{cmpName}"" AS SELECT ""A"".""Name"", ""A"".""Hash"", ""A"".""Size"" FROM ""Remotevolume"" ""A"", ""Remotevolume"" ""B"", ""IndexBlockLink"" WHERE ""B"".""Name"" = @Name AND ""A"".""ID"" = ""IndexBlockLink"".""BlockVolumeID"" AND ""B"".""ID"" = ""IndexBlockLink"".""IndexVolumeID"" "; var extra = $@" SELECT @TypeExtra AS ""Type"", ""{m_tablename}"".""Name"" AS ""Name"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Name"" NOT IN ( SELECT ""Name"" FROM ""{cmpName}"" ) "; var missing = $@" SELECT @TypeMissing AS ""Type"", ""Name"" AS ""Name"" FROM ""{cmpName}"" WHERE ""Name"" NOT IN ( SELECT ""Name"" FROM ""{m_tablename}"" ) "; var modified = $@" SELECT @TypeModified AS ""Type"", ""E"".""Name"" AS ""Name"" FROM ""{m_tablename}"" ""E"", ""{cmpName}"" ""D"" WHERE ""D"".""Name"" = ""E"".""Name"" AND ( ""D"".""Hash"" != ""E"".""Hash"" OR ""D"".""Size"" != ""E"".""Size"" ) "; var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"" "; await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); try { await cmd .SetCommandAndParameters(create) .SetParameterValue("@Name", m_volumename) .ExecuteNonQueryAsync() .ConfigureAwait(false); cmd .SetCommandAndParameters($"{extra} UNION {missing} UNION {modified}") .SetParameterValue("@TypeExtra", (int)Interface.TestEntryStatus.Extra) .SetParameterValue("@TypeMissing", (int)Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Interface.TestEntryStatus.Modified); await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) yield return new KeyValuePair((Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); } finally { try { await cmd .ExecuteNonQueryAsync(drop) .ConfigureAwait(false); } catch { } } } } /// /// Interface for a blocklist used in the local test database. /// Provides methods for adding blocks, comparing the blocklist with remote volumes, and disposing of resources. /// public interface IBlocklist : IDisposable, IAsyncDisposable { /// /// Asynchronously adds a block to the blocklist. /// /// The key (hash) of the block. /// The size of the block in bytes. /// A task that completes when the block has been added. Task AddBlock(string key, long value); /// /// Asynchronously compares the blocklist with remote volumes and yields differences. /// /// An asynchronous enumerable of key-value pairs representing the comparison results, where the key is the test entry status and the value is the block hash. IAsyncEnumerable> Compare(); } public interface IBlocklistHashList : IDisposable, IAsyncDisposable { /// /// Asynchronously adds a block hash to the blocklist hash list. /// /// The hash of the block. /// The size of the block in bytes. /// A task that completes when the block hash has been added. Task AddBlockHash(string hash, long size); /// /// Asynchronously compares the blocklist hash list with remote volumes and yields differences. /// /// The number of hashes per block. /// The size of each hash in bytes. /// The size of each block in bytes. /// An asynchronous enumerable of key-value pairs representing the comparison results, where the key is the test entry status and the value is the block hash. IAsyncEnumerable> Compare(int hashesPerBlock, int hashSize, int blockSize); } /// /// Implementation of the interface that manages a list of blocks in a local test database. /// Provides methods for adding blocks, comparing the blocklist with remote volumes, and disposing of resources. /// private class Blocklist : Basiclist, IBlocklist { /// /// The prefix for the temporary table name used for the blocklist. /// private const string TABLE_PREFIX = "Blocklist"; /// /// The SQL format for creating the temporary table used for the blocklist. /// private const string TABLE_FORMAT = @"( ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL )"; /// /// The SQL command for inserting data into the temporary table used for the blocklist. /// private const string INSERT_COMMAND = @"( ""Hash"", ""Size"" ) VALUES ( @Hash, @Size )"; /// /// Calling this constructor will throw an exception. Use the CreateAsync method instead. /// [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] public Blocklist(SqliteConnection connection, string volumename, ReusableTransaction rtr) { throw new NotSupportedException("Use CreateAsync method instead."); } /// /// Private constructor to allow derived classes to initialize without parameters and to prevent instantiation from outside, which should only be done through the CreateAsync method. /// private Blocklist() { } /// /// Asynchronously creates a new instance of the class. /// /// The local database to use for the blocklist. /// The name of the volume associated with this blocklist. public static async Task CreateAsync(LocalDatabase db, string volumename) { var bl = new Blocklist(); return (Blocklist) await Basiclist.CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) .ConfigureAwait(false); } public async Task AddBlock(string hash, long size) { await m_insertCommand .SetTransaction(m_db.Transaction) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) .ExecuteNonQueryAsync() .ConfigureAwait(false); } public async IAsyncEnumerable> Compare() { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); var curBlocks = @" SELECT ""Block"".""Hash"" AS ""Hash"", ""Block"".""Size"" AS ""Size"" FROM ""Remotevolume"", ""Block"" WHERE ""Remotevolume"".""Name"" = @Name AND ""Remotevolume"".""ID"" = ""Block"".""VolumeID"" "; var duplBlocks = @" SELECT ""Block"".""Hash"" AS ""Hash"", ""Block"".""Size"" AS ""Size"" FROM ""DuplicateBlock"", ""Block"" WHERE ""DuplicateBlock"".""VolumeID"" = ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = @Name ) AND ""Block"".""ID"" = ""DuplicateBlock"".""BlockID"" "; var delBlocks = @" SELECT ""DeletedBlock"".""Hash"" AS ""Hash"", ""DeletedBlock"".""Size"" AS ""Size"" FROM ""DeletedBlock"", ""RemoteVolume"" WHERE ""RemoteVolume"".""Name"" = @Name AND ""RemoteVolume"".""ID"" = ""DeletedBlock"".""VolumeID"" "; var create = $@" CREATE TEMPORARY TABLE ""{cmpName}"" AS SELECT DISTINCT ""Hash"" AS ""Hash"", ""Size"" AS ""Size"" FROM ( {curBlocks} UNION {delBlocks} UNION {duplBlocks} ) "; var extra = $@" SELECT @TypeExtra AS ""Type"", ""{m_tablename}"".""Hash"" AS ""Hash"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Hash"" NOT IN ( SELECT ""Hash"" FROM ""{cmpName}"" ) "; var missing = $@" SELECT @TypeMissing AS ""Type"", ""Hash"" AS ""Hash"" FROM ""{cmpName}"" WHERE ""Hash"" NOT IN ( SELECT ""Hash"" FROM ""{m_tablename}"" ) "; var modified = $@" SELECT @TypeModified AS ""Type"", ""E"".""Hash"" AS ""Hash"" FROM ""{m_tablename}"" E, ""{cmpName}"" D WHERE ""D"".""Hash"" = ""E"".""Hash"" AND ""D"".""Size"" != ""E"".""Size"" "; var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"" "; await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); try { await cmd .SetCommandAndParameters(create) .SetParameterValue("@Name", m_volumename) .ExecuteNonQueryAsync() .ConfigureAwait(false); cmd .SetCommandAndParameters($@" {extra} UNION {missing} UNION {modified} ") .SetParameterValue("@TypeExtra", (int)Library.Interface.TestEntryStatus.Extra) .SetParameterValue("@TypeMissing", (int)Library.Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Library.Interface.TestEntryStatus.Modified); await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) yield return new KeyValuePair((Duplicati.Library.Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); } finally { try { await cmd .ExecuteNonQueryAsync(drop) .ConfigureAwait(false); } catch { } } } } /// /// Implementation of the interface that manages a list of block hashes in a local test database. /// Provides methods for adding block hashes, comparing the blocklist hash list with remote volumes, and disposing of resources. /// private class BlocklistHashList : Basiclist, IBlocklistHashList { /// /// The prefix for the temporary table name used for the blocklist hash list. /// private const string TABLE_PREFIX = "BlocklistHashList"; /// /// The SQL format for creating the temporary table used for the blocklist hash list. /// private const string TABLE_FORMAT = @"( ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL )"; /// /// The SQL command for inserting data into the temporary table used for the blocklist hash list. /// private const string INSERT_COMMAND = @"( ""Hash"", ""Size"" ) VALUES ( @Hash, @Size )"; /// /// Calling this constructor will throw an exception. Use the CreateAsync method instead. /// [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] public BlocklistHashList(SqliteConnection connection, string volumename, ReusableTransaction rtr) { throw new NotSupportedException("Use CreateAsync method instead."); } /// /// Private constructor to allow derived classes to initialize without parameters and to prevent instantiation from outside, which should only be done through the CreateAsync method. /// private BlocklistHashList() { } /// /// Asynchronously creates a new instance of the class. /// /// The local database to use for the blocklist hash list. /// The name of the volume associated with this blocklist hash list. /// A task that when awaited returns a new instance of the class. public static async Task CreateAsync(LocalDatabase db, string volumename) { var bl = new BlocklistHashList(); return (BlocklistHashList) await Basiclist.CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) .ConfigureAwait(false); } public async Task AddBlockHash(string hash, long size) { await m_insertCommand .SetTransaction(m_db.Transaction) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) .ExecuteNonQueryAsync() .ConfigureAwait(false); } public async IAsyncEnumerable> Compare(int hashesPerBlock, int hashSize, int blockSize) { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); var create = $@" CREATE TEMPORARY TABLE ""{cmpName}"" ( ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL ); INSERT INTO ""{cmpName}"" ( ""Hash"", ""Size"" ) SELECT ""b"".""Hash"", ""b"".""Size"" FROM ""Block"" ""b"" JOIN ( SELECT ""blh"".""Hash"", CASE WHEN ""blh"".""Index"" = (((""bs"".""Length"" + {blockSize} - 1) / {blockSize} - 1) / {hashesPerBlock}) AND ((""bs"".""Length"" + {blockSize} - 1) / {blockSize}) % {hashesPerBlock} != 0 THEN {hashSize} * ((""bs"".""Length"" + {blockSize} - 1) / {blockSize} % {hashesPerBlock}) ELSE {hashSize} * {hashesPerBlock} END AS ""Size"" FROM ""BlocklistHash"" ""blh"" JOIN ""Blockset"" ""bs"" ON ""bs"".""ID"" = ""blh"".""BlocksetID"" ) ""Expected"" ON ""b"".""Hash"" = ""Expected"".""Hash"" AND ""b"".""Size"" = ""Expected"".""Size"" WHERE ""b"".""VolumeID"" IN ( SELECT ""ibl"".""BlockVolumeID"" FROM ""Remotevolume"" ""idx"" JOIN IndexBlockLink ""ibl"" ON ""ibl"".""IndexVolumeID"" = ""idx"".""ID"" WHERE ""idx"".""Name"" = @Name ); "; var compare = $@" WITH ""Expected"" AS ( SELECT ""Hash"", ""Size"" FROM ""{cmpName}"" ), ""Actual"" AS ( SELECT ""Hash"", ""Size"" FROM ""{m_tablename}"" ), ""Extra"" AS ( SELECT @TypeExtra AS ""Type"", ""a"".""Hash"" FROM ""Actual"" ""a"" LEFT JOIN ""Expected"" ""e"" ON ""a"".""Hash"" = ""e"".""Hash"" AND ""a"".""Size"" = ""e"".""Size"" WHERE ""e"".""Hash"" IS NULL ), ""Missing"" AS ( SELECT @TypeMissing AS ""Type"", ""e"".""Hash"" FROM ""Expected"" ""e"" LEFT JOIN ""Actual"" ""a"" ON ""a"".""Hash"" = ""e"".""Hash"" AND ""a"".""Size"" = ""e"".""Size"" WHERE ""a"".""Hash"" IS NULL ), ""Modified"" AS ( SELECT @TypeModified AS ""Type"", ""a"".""Hash"" FROM ""Actual"" ""a"" JOIN ""Expected"" ""e"" ON ""a"".""Hash"" = ""e"".""Hash"" WHERE ""a"".""Size"" != ""e"".""Size"" AND NOT EXISTS ( SELECT 1 FROM ""Extra"" ""x"" WHERE ""x"".""Hash"" = ""a"".""Hash"" ) ) SELECT * FROM ""Extra"" UNION SELECT * FROM ""Missing"" UNION SELECT * FROM ""Modified""; "; var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"""; await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); try { // Create expected hash+size table filtered by volume await cmd .SetCommandAndParameters(create) .SetParameterValue("@Name", m_volumename) .ExecuteNonQueryAsync() .ConfigureAwait(false); // Compare against actual values inserted into temp table cmd .SetCommandAndParameters(compare) .SetParameterValue("@TypeExtra", (int)Library.Interface.TestEntryStatus.Extra) .SetParameterValue("@TypeMissing", (int)Library.Interface.TestEntryStatus.Missing) .SetParameterValue("@TypeModified", (int)Library.Interface.TestEntryStatus.Modified); await using var rd = await cmd.ExecuteReaderAsync().ConfigureAwait(false); while (await rd.ReadAsync().ConfigureAwait(false)) yield return new KeyValuePair( (Library.Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); } finally { try { await cmd .ExecuteNonQueryAsync(drop) .ConfigureAwait(false); } catch { } } } } /// /// Creates a new filelist in the local test database. /// /// The name of the filelist to create. /// A task that when awaited returns a new instance of the interface. public async Task CreateFilelist(string name) { return await Filelist.CreateAsync(this, name).ConfigureAwait(false); } /// /// Creates a new indexlist in the local test database. /// /// The name of the indexlist to create. /// A task that when awaited returns a new instance of the interface. public async Task CreateIndexlist(string name) { return await Indexlist.CreateAsync(this, name).ConfigureAwait(false); } /// /// Creates a new blocklist in the local test database. /// /// The name of the blocklist to create. /// A task that when awaited returns a new instance of the interface. public async Task CreateBlocklist(string name) { return await Blocklist.CreateAsync(this, name).ConfigureAwait(false); } /// /// Creates a new blocklist hash list in the local test database. /// /// The name of the blocklist hash list to create. /// A task that when awaited returns a new instance of the interface. public async Task CreateBlocklistHashList(string name) { return await BlocklistHashList.CreateAsync(this, name) .ConfigureAwait(false); } } }