// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal in the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable using System; using System.Collections.Generic; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Utility; using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database { /// /// A local database for tracking changes in file lists, such as added, deleted, or modified files. /// internal class LocalListChangesDatabase : LocalDatabase { /// /// Creates a new instance of the class. /// /// The path to the database file. /// An optional existing database instance to use. Used to mimic constructor chaining. /// A cancellation token to cancel the operation. /// A task that when awaited contains a new instance of . public static async Task CreateAsync(string path, LocalListChangesDatabase? dbnew, CancellationToken token) { dbnew ??= new LocalListChangesDatabase(); dbnew = (LocalListChangesDatabase) await CreateLocalDatabaseAsync(path, "ListChanges", false, dbnew, token) .ConfigureAwait(false); dbnew.ShouldCloseConnection = true; return dbnew; } /// /// Interface for storage helper that manages temporary storage of file changes. /// public interface IStorageHelper : IDisposable, IAsyncDisposable { /// /// Adds an element to the temporary storage. /// /// The path of the file or folder. /// The file hash, if applicable. /// The metadata hash. /// The size of the file or folder. /// The type of the element (file, folder, symlink). /// If true, adds to the current table; otherwise, adds to the previous table. /// A cancellation token to cancel the operation. /// A task that completes when the element is added. Task AddElement(string path, string filehash, string metahash, long size, Interface.ListChangesElementType type, bool asNew, CancellationToken token); /// /// Adds elements from the database to the temporary storage. /// /// The ID of the fileset to add. /// If true, adds to the current table; otherwise, adds to the previous table. /// An optional filter to apply when adding elements. /// A cancellation token to cancel the operation. /// A task that completes when the elements are added. Task AddFromDb(long filesetId, bool asNew, IFilter filter, CancellationToken token); /// /// Creates a report containing the count of added, deleted, and modified elements. /// /// A cancellation token to cancel the operation. /// A task that, when awaited, returns an with the change counts. Task CreateChangeCountReport(CancellationToken token); /// /// Creates a report containing the size information for added, deleted, previous, and current elements. /// /// A cancellation token to cancel the operation. /// A task that, when awaited, returns an with the size details. Task CreateChangeSizeReport(CancellationToken token); /// /// Asynchronously generates a report of changed files, yielding tuples that describe the change type, element type, and file path. /// /// A cancellation token to cancel the operation. /// An asynchronous enumerable of tuples containing the change type, element type, and file path. IAsyncEnumerable> CreateChangedFileReport(CancellationToken token); } /// /// Interface for reporting changes in file counts and sizes. /// public interface IChangeCountReport { /// /// Gets the count of added folders. /// long AddedFolders { get; } /// /// Gets the count of added symlinks. /// long AddedSymlinks { get; } /// /// Gets the count of added files. /// long AddedFiles { get; } /// /// Gets the count of deleted folders. /// long DeletedFolders { get; } /// /// Gets the count of deleted symlinks. /// long DeletedSymlinks { get; } /// /// Gets the count of deleted files. /// long DeletedFiles { get; } /// /// Gets the count of modified folders. /// long ModifiedFolders { get; } /// /// Gets the count of modified symlinks. /// long ModifiedSymlinks { get; } /// /// Gets the count of modified files. /// long ModifiedFiles { get; } } /// /// Interface for a report describing changes in file sizes. /// public interface IChangeSizeReport { /// /// Gets the total size of added files. /// long AddedSize { get; } /// /// Gets the total size of deleted files. /// long DeletedSize { get; } /// /// Gets the size of files in the previous state. /// long PreviousSize { get; } /// /// Gets the size of files in the current state. /// long CurrentSize { get; } } /// /// Internal class that implements the interface to report changes in file counts. /// internal class ChangeCountReport : IChangeCountReport { public long AddedFolders { get; internal set; } public long AddedSymlinks { get; internal set; } public long AddedFiles { get; internal set; } public long DeletedFolders { get; internal set; } public long DeletedSymlinks { get; internal set; } public long DeletedFiles { get; internal set; } public long ModifiedFolders { get; internal set; } public long ModifiedSymlinks { get; internal set; } public long ModifiedFiles { get; internal set; } } /// /// Internal class that implements the interface to report changes in file sizes. /// internal class ChangeSizeReport : IChangeSizeReport { public long AddedSize { get; internal set; } public long DeletedSize { get; internal set; } public long PreviousSize { get; internal set; } public long CurrentSize { get; internal set; } } /// /// Helper class for managing temporary storage of file changes. /// Implements the interface. /// private class StorageHelper : IStorageHelper { /// /// The database instance used for storage operations. /// private LocalDatabase m_db = null!; /// /// Command for inserting elements into the previous table. /// private SqliteCommand m_insertPreviousElementCommand = null!; /// /// Command for inserting elements into the current table. /// private SqliteCommand m_insertCurrentElementCommand = null!; /// /// The name of the temporary table for previous elements. /// private string m_previousTable = null!; /// /// The name of the temporary table for current elements. /// private string m_currentTable = null!; /// /// Private constructor to prevent direct instantiation. /// This constructor is obsolete and will throw an exception if called. /// Use the method to create an instance instead. /// [Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")] public StorageHelper(SqliteConnection con) { } /// /// Private constructor to prevent direct instantiation. /// This class should be created using the CreateAsync method. /// private StorageHelper() { } /// /// Asynchronously creates a new instance of the class. /// /// The local database instance to use. /// A cancellation token to cancel the operation. /// A task that, when awaited, returns a new instance of . public static async Task CreateAsync(LocalDatabase db, CancellationToken token) { var sh = new StorageHelper { m_db = db, m_previousTable = $"Previous-{Library.Utility.Utility.GetHexGuid()}", m_currentTable = $"Current-{Library.Utility.Utility.GetHexGuid()}" }; await using (var cmd = sh.m_db.Connection.CreateCommand(db.Transaction)) { await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{sh.m_previousTable}"" ( ""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL ) ", token) .ConfigureAwait(false); await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{sh.m_currentTable}"" ( ""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL ) ", token) .ConfigureAwait(false); } sh.m_insertPreviousElementCommand = await sh.m_db.Connection .CreateCommandAsync($@" INSERT INTO ""{sh.m_previousTable}"" ( ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" ) VALUES ( @Path, @FileHash, @MetaHash, @Size, @Type ) ", token) .ConfigureAwait(false); sh.m_insertCurrentElementCommand = await sh.m_db.Connection .CreateCommandAsync($@" INSERT INTO ""{sh.m_currentTable}"" ( ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" ) VALUES ( @Path, @FileHash, @MetaHash, @Size, @Type ) ", token) .ConfigureAwait(false); return sh; } /// public async Task AddFromDb(long filesetId, bool asNew, IFilter filter, CancellationToken token) { var tablename = asNew ? m_currentTable : m_previousTable; var folders = $@" SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {Library.Utility.Utility.FormatInvariantValue((int)Interface.ListChangesElementType.Folder)} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"", ""FilesetEntry"", ""Metadataset"", ""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -100 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "; var symlinks = $@" SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {Library.Utility.Utility.FormatInvariantValue((int)Interface.ListChangesElementType.Symlink)} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"", ""FilesetEntry"", ""Metadataset"", ""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -200 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "; var files = $@" SELECT ""File"".""Path"" AS ""Path"", ""FB"".""FullHash"" AS ""FileHash"", ""MB"".""Fullhash"" AS ""MetaHash"", ""FB"".""Length"" AS ""Size"", {Library.Utility.Utility.FormatInvariantValue((int)Interface.ListChangesElementType.File)} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"", ""FilesetEntry"", ""Metadataset"", ""Blockset"" MB, ""Blockset"" FB WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" >= 0 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""MB"".""ID"" AND ""File"".""BlocksetID"" = ""FB"".""ID"" "; var combined = $"({folders} UNION {symlinks} UNION {files})"; await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); if (filter == null || filter.Empty) { // Simple case, select everything await cmd.SetCommandAndParameters($@" INSERT INTO ""{tablename}"" ( ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" ) SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {combined} ""A"" WHERE ""A"".""FilesetID"" = @FilesetId ") .SetParameterValue("@FilesetId", filesetId) .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } else if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == Duplicati.Library.Utility.FilterType.Simple) { // File list based // unfortunately we cannot do this if the filesystem is case sensitive as // SQLite only supports ASCII compares var p = expression.GetSimpleList(); var filenamestable = $"Filenames-{Library.Utility.Utility.GetHexGuid()}"; await cmd.ExecuteNonQueryAsync($@" CREATE TEMPORARY TABLE ""{filenamestable}"" ( ""Path"" TEXT NOT NULL ) ", token) .ConfigureAwait(false); await cmd.SetCommandAndParameters($@" INSERT INTO ""{filenamestable}"" (""Path"") VALUES (@Path) ") .PrepareAsync(token) .ConfigureAwait(false); foreach (var s in p) await cmd .SetParameterValue("@Path", s) .ExecuteNonQueryAsync(token) .ConfigureAwait(false); string whereClause; if (expression.Result) { // Include filter whereClause = $@" ""A"".""FilesetID"" = @FilesetId AND ""A"".""Path"" IN ( SELECT DISTINCT ""Path"" FROM ""{filenamestable}"" ) "; } else { // Exclude filter whereClause = $@" ""A"".""FilesetID"" = @FilesetId AND ""A"".""Path"" NOT IN ( SELECT DISTINCT ""Path"" FROM ""{filenamestable}"" ) "; } await cmd.SetCommandAndParameters($@" INSERT INTO ""{tablename}"" ( ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" ) SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {combined} ""A"" WHERE {whereClause} ") .SetParameterValue("@FilesetId", filesetId) .ExecuteNonQueryAsync(token) .ConfigureAwait(false); await cmd .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filenamestable}"" ", token) .ConfigureAwait(false); } else { // Do row-wise iteration var values = new object[5]; await cmd.SetCommandAndParameters($@" SELECT ""A"".""Path"", ""A"".""FileHash"", ""A"".""MetaHash"", ""A"".""Size"", ""A"".""Type"" FROM {combined} ""A"" WHERE ""A"".""FilesetID"" = @FilesetId ") .SetParameterValue("@FilesetId", filesetId) .PrepareAsync(token) .ConfigureAwait(false); await using var cmd2 = m_db.Connection.CreateCommand(m_db.Transaction) .SetCommandAndParameters($@" INSERT INTO ""{tablename}"" ( ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" ) VALUES ( @Path, @FileHash, @MetaHash, @Size, @Type ) "); await cmd2.PrepareAsync(token).ConfigureAwait(false); await using var rd = await cmd .ExecuteReaderAsync(token) .ConfigureAwait(false); while (await rd.ReadAsync(token).ConfigureAwait(false)) { rd.GetValues(values); var path = values[0] as string; if (path != null && FilterExpression.Matches(filter, path.ToString())) { await cmd2 .SetParameterValue("@Path", values[0]) .SetParameterValue("@FileHash", values[1]) .SetParameterValue("@MetaHash", values[2]) .SetParameterValue("@Size", values[3]) .SetParameterValue("@Type", values[4]) .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } } } } /// public async Task AddElement(string path, string filehash, string metahash, long size, Interface.ListChangesElementType type, bool asNew, CancellationToken token) { var cmd = asNew ? m_insertCurrentElementCommand : m_insertPreviousElementCommand; await cmd .SetParameterValue("@Path", path) .SetParameterValue("@FileHash", filehash) .SetParameterValue("@MetaHash", metahash) .SetParameterValue("@Size", size) .SetParameterValue("@Type", (int)type) .ExecuteNonQueryAsync(token) .ConfigureAwait(false); } /// /// Converts a SqliteDataReader to an asynchronous enumerable of strings. /// /// The SqliteDataReader to read from. /// A cancellation token to cancel the operation. /// An asynchronous enumerable of strings, where each string is a value from the first column of the reader. private static async IAsyncEnumerable ReaderToStringList(SqliteDataReader rd, [EnumeratorCancellation] CancellationToken token) { await using (rd) while (await rd.ReadAsync(token).ConfigureAwait(false)) { var v = rd.GetValue(0); if (v == null || v == DBNull.Value) yield return null; else yield return v.ToString(); } } /// /// Retrieves SQL queries for added, deleted, and modified files based on the current and previous tables. /// /// If true, retrieves all types of changes; otherwise, filters by type. /// A tuple containing SQL queries for added, deleted, and modified files. private (string Added, string Deleted, string Modified) GetSqls(bool allTypes) { return ( $@" SELECT ""Path"" FROM ""{m_currentTable}"" WHERE {(allTypes ? "" : @$" ""{m_currentTable}"".""Type"" = @Type AND ")} ""{m_currentTable}"".""Path"" NOT IN ( SELECT ""Path"" FROM ""{m_previousTable}"" ) ", $@" SELECT ""Path"" FROM ""{m_previousTable}"" WHERE {(allTypes ? "" : @$" ""{m_previousTable}"".""Type"" = @Type AND ")} ""{m_previousTable}"".""Path"" NOT IN ( SELECT ""Path"" FROM ""{m_currentTable}"" ) ", $@" SELECT ""{m_currentTable}"".""Path"" FROM ""{m_currentTable}"",""{m_previousTable}"" WHERE {(allTypes ? "" : $@" ""{m_currentTable}"".""Type"" = @Type AND ")} ""{m_currentTable}"".""Path"" = ""{m_previousTable}"".""Path"" AND ( ""{m_currentTable}"".""FileHash"" != ""{m_previousTable}"".""FileHash"" OR ""{m_currentTable}"".""MetaHash"" != ""{m_previousTable}"".""MetaHash"" OR ""{m_currentTable}"".""Type"" != ""{m_previousTable}"".""Type"" ) " ); } /// /// Creates a report of changes in file sizes, including added, deleted, previous, and current sizes. /// /// A cancellation token to cancel the operation. /// A task that, when awaited, returns an with the size details. public async Task CreateChangeSizeReport(CancellationToken token) { var (Added, Deleted, Modified) = GetSqls(true); await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); var result = new ChangeSizeReport { PreviousSize = await cmd.ExecuteScalarInt64Async($@" SELECT SUM(""Size"") FROM ""{m_previousTable}"" ", 0, token) .ConfigureAwait(false), CurrentSize = await cmd.ExecuteScalarInt64Async($@" SELECT SUM(""Size"") FROM ""{m_currentTable}"" ", 0, token) .ConfigureAwait(false), AddedSize = await cmd.ExecuteScalarInt64Async($@" SELECT SUM(""Size"") FROM ""{m_currentTable}"" WHERE ""{m_currentTable}"".""Path"" IN ({Added}) ", 0, token) .ConfigureAwait(false), DeletedSize = await cmd.ExecuteScalarInt64Async($@" SELECT SUM(""Size"") FROM ""{m_previousTable}"" WHERE ""{m_previousTable}"".""Path"" IN ({Deleted}) ", 0, token) .ConfigureAwait(false) }; return result; } /// /// Creates a report containing the count of added, deleted, and modified elements. /// /// A cancellation token to cancel the operation. /// A task that, when awaited, returns an with the change counts. public async Task CreateChangeCountReport(CancellationToken token) { var (Added, Deleted, Modified) = GetSqls(false); var added = @$" SELECT COUNT(*) FROM ({Added}) "; var deleted = @$" SELECT COUNT(*) FROM ({Deleted}) "; var modified = @$" SELECT COUNT(*) FROM ({Modified}) "; await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); var result = new ChangeCountReport { AddedFolders = await cmd .SetCommandAndParameters(added) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Folder) .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false), AddedSymlinks = await cmd .SetCommandAndParameters(added) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Symlink) .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false), AddedFiles = await cmd .SetCommandAndParameters(added) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.File) .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false), DeletedFolders = await cmd .SetCommandAndParameters(deleted) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Folder) .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false), DeletedSymlinks = await cmd .SetCommandAndParameters(deleted) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Symlink) .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false), DeletedFiles = await cmd .SetCommandAndParameters(deleted) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.File) .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false), ModifiedFolders = await cmd .SetCommandAndParameters(modified) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Folder) .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false), ModifiedSymlinks = await cmd .SetCommandAndParameters(modified) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Symlink) .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false), ModifiedFiles = await cmd .SetCommandAndParameters(modified) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.File) .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false) }; return result; } /// /// Asynchronously creates a report of changed files, yielding tuples that describe the change type, element type, and file path. /// /// A cancellation token to cancel the operation. /// An asynchronous enumerable of tuples containing the change type, element type, and file path. public async IAsyncEnumerable> CreateChangedFileReport([EnumeratorCancellation] CancellationToken token) { var (Added, Deleted, Modified) = GetSqls(false); await using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) { var elTypes = new[] { Interface.ListChangesElementType.Folder, Interface.ListChangesElementType.Symlink, Interface.ListChangesElementType.File }; async IAsyncEnumerable> BuildResult(SqliteCommand cmd, string sql, Interface.ListChangesChangeType changeType) { cmd.SetCommandAndParameters(sql); foreach (var type in elTypes) await foreach (var s in ReaderToStringList(await cmd.SetParameterValue("@Type", (int)type).ExecuteReaderAsync(token).ConfigureAwait(false), token).ConfigureAwait(false)) yield return new Tuple(changeType, type, s ?? ""); } await foreach (var r in BuildResult(cmd, Added, Interface.ListChangesChangeType.Added) .ConfigureAwait(false) ) yield return r; await foreach (var r in BuildResult(cmd, Deleted, Interface.ListChangesChangeType.Deleted) .ConfigureAwait(false) ) yield return r; await foreach (var r in BuildResult(cmd, Modified, Interface.ListChangesChangeType.Modified) .ConfigureAwait(false) ) yield return r; } } public void Dispose() { DisposeAsync().AsTask().Await(); } public async ValueTask DisposeAsync() { if (m_insertPreviousElementCommand != null) { try { await m_insertPreviousElementCommand .DisposeAsync() .ConfigureAwait(false); } catch { } finally { m_insertPreviousElementCommand = null!; } } if (m_insertCurrentElementCommand != null) { try { await m_insertCurrentElementCommand .DisposeAsync() .ConfigureAwait(false); } catch { } finally { m_insertCurrentElementCommand = null!; } } try { await m_db.Transaction .RollBackAsync() .ConfigureAwait(false); } catch { } finally { m_previousTable = null!; m_currentTable = null!; } } } /// /// Creates a new instance of the for managing temporary storage of file changes. /// /// A cancellation token to cancel the operation. /// A task that, when awaited, returns an instance of . public async Task CreateStorageHelper(CancellationToken token) { return await StorageHelper.CreateAsync(this, token).ConfigureAwait(false); } } }