// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal in the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable using System; using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database; /// /// Encapsulates the logic for calculating change statistics /// public static class ChangeStatistics { /// /// The tag used for log messages /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(ChangeStatistics)); /// /// Calculates the change statistics for the current and previous fileset. /// /// The database command to use. /// The results object to update with the statistics. /// The ID of the current fileset. /// The ID of the previous fileset. /// The cancellation token to monitor for cancellation requests. /// A task that completes when the statistics are updated. internal static async Task UpdateChangeStatistics(SqliteCommand cmd, BackupResults results, long currentFilesetId, long previousFilesetId, CancellationToken token) { var tmpName = $"TmpFileState_{Library.Utility.Utility.GetHexGuid()}"; try { // Create temp table await cmd.SetCommandAndParameters($@" CREATE TEMP TABLE ""{tmpName}"" AS SELECT ""FL"".""PrefixID"", ""FL"".""Path"", ""FL"".""BlocksetID"", ""BS_Meta"".""Fullhash"" AS ""Metahash"", CASE ""FE"".""FilesetID"" WHEN @LastFilesetId THEN 0 WHEN @CurrentFilesetId THEN 1 END AS ""Source"" FROM ""FileLookup"" ""FL"" JOIN ""FilesetEntry"" ""FE"" ON ""FL"".""ID"" = ""FE"".""FileID"" LEFT JOIN ""Blockset"" ""BS_Data"" ON ""FL"".""BlocksetID"" = ""BS_Data"".""ID"" LEFT JOIN ""Metadataset"" M ON ""FL"".""MetadataID"" = M.""ID"" LEFT JOIN ""Blockset"" ""BS_Meta"" ON M.""BlocksetID"" = ""BS_Meta"".""ID"" WHERE ""FE"".""FilesetID"" IN ( @LastFilesetId, @CurrentFilesetId ); ") .SetParameterValue("@LastFilesetId", previousFilesetId) .SetParameterValue("@CurrentFilesetId", currentFilesetId) .ExecuteNonQueryAsync(token) .ConfigureAwait(false); // Index for fast comparison await cmd.ExecuteNonQueryAsync($@" CREATE INDEX ""idx_{tmpName}"" ON ""{tmpName}"" ( ""PrefixID"", ""Path"" ) ", token) .ConfigureAwait(false); // Added results.AddedFolders = await CountAdded(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID, null, token) .ConfigureAwait(false); results.AddedSymlinks = await CountAdded(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID, null, token) .ConfigureAwait(false); results.AddedFiles = await CountAdded(cmd, tmpName, null, [ LocalDatabase.FOLDER_BLOCKSET_ID, LocalDatabase.SYMLINK_BLOCKSET_ID ], token) .ConfigureAwait(false); // Deleted results.DeletedFolders = await CountDeleted(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID, null, token) .ConfigureAwait(false); results.DeletedSymlinks = await CountDeleted(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID, null, token) .ConfigureAwait(false); results.DeletedFiles = await CountDeleted(cmd, tmpName, null, [ LocalDatabase.FOLDER_BLOCKSET_ID, LocalDatabase.SYMLINK_BLOCKSET_ID ], token) .ConfigureAwait(false); // Modified results.ModifiedFolders = await CountModified(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID, null, token) .ConfigureAwait(false); results.ModifiedSymlinks = await CountModified(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID, null, token) .ConfigureAwait(false); results.ModifiedFiles = await CountModified(cmd, tmpName, null, [ LocalDatabase.FOLDER_BLOCKSET_ID, LocalDatabase.SYMLINK_BLOCKSET_ID ], token) .ConfigureAwait(false); } finally { try { await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmpName}"";", token) .ConfigureAwait(false); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "DropTemp", ex, $"Failed to drop {tmpName}"); } } } /// /// Counts the number of added files or folders. /// /// The database command to use. /// The name of the temporary table. /// The ID of the blockset to count, or null for all. /// The blocksets to exclude from the count. /// The cancellation token to monitor for cancellation requests. /// A task that when awaited contains the number of added files or folders. private static async Task CountAdded(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets, CancellationToken token) { var conditions = $@" ""Source"" = 1 AND NOT EXISTS ( SELECT 1 FROM ""{tmpName}"" ""B"" WHERE ""B"".""Source"" = 0 AND ""B"".""PrefixID"" = ""A"".""PrefixID"" AND ""B"".""Path"" = ""A"".""Path"" ) "; return await CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets, token) .ConfigureAwait(false); } /// /// Counts the number of deleted files or folders. /// /// The database command to use. /// The name of the temporary table. /// The ID of the blockset to count, or null for all. /// The blocksets to exclude from the count. /// The cancellation token to monitor for cancellation requests. /// A task that when awaited contains the number of deleted files or folders. private static async Task CountDeleted(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets, CancellationToken token) { var conditions = @$" ""Source"" = 0 AND NOT EXISTS ( SELECT 1 FROM ""{tmpName}"" ""B"" WHERE ""B"".""Source"" = 1 AND ""B"".""PrefixID"" = ""A"".""PrefixID"" AND ""B"".""Path"" = ""A"".""Path"" ) "; return await CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets, token) .ConfigureAwait(false); } /// /// Counts the number of modified files or folders. /// /// The database command to use. /// The name of the temporary table. /// The ID of the blockset to count, or null for all. /// The blocksets to exclude from the count. /// The cancellation token to monitor for cancellation requests. /// A task that when awaited contains the number of modified files or folders. private static async Task CountModified(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets, CancellationToken token) { string conditions; if (blocksetId == LocalDatabase.FOLDER_BLOCKSET_ID || blocksetId == LocalDatabase.SYMLINK_BLOCKSET_ID) { conditions = @" ""A"".""Source"" = 0 AND ""B"".""Source"" = 1 AND ""A"".""PrefixID"" = ""B"".""PrefixID"" AND ""A"".""Path"" = ""B"".""Path"" AND ""A"".""Metahash"" IS NOT ""B"".""Metahash"" "; } else { conditions = @" ""A"".""Source"" = 0 AND ""B"".""Source"" = 1 AND ""A"".""PrefixID"" = ""B"".""PrefixID"" AND ""A"".""Path"" = ""B"".""Path"" AND ( ""A"".""BlocksetID"" IS NOT ""B"".""BlocksetID"" OR ""A"".""Metahash"" IS NOT ""B"".""Metahash"" ) "; } string blocksetCondition = GetBlocksetCondition("A", blocksetId, excludeBlocksets); if (!string.IsNullOrEmpty(blocksetCondition)) conditions += $" AND {blocksetCondition}"; string sql = $@" SELECT COUNT(*) FROM ""{tmpName}"" ""A"" JOIN ""{tmpName}"" ""B"" ON ""A"".""PrefixID"" = ""B"".""PrefixID"" AND ""A"".""Path"" = ""B"".""Path"" WHERE {conditions} "; return await cmd .SetCommandAndParameters(sql) .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false); } /// /// Counts the number of files or folders with a specific condition. /// /// The database command to use. /// The name of the temporary table. /// The alias for the temporary table. /// The base condition for the count. /// The ID of the blockset to count, or null for all. /// The blocksets to exclude from the count. /// The cancellation token to monitor for cancellation requests. /// A task that when awaited contains the number of files or folders that match the condition. private static async Task CountWithCondition(SqliteCommand cmd, string tmpName, string alias, string baseCondition, long? blocksetId, long[]? excludeBlocksets, CancellationToken token) { var fullCondition = baseCondition; var blocksetCondition = GetBlocksetCondition(alias, blocksetId, excludeBlocksets); if (!string.IsNullOrEmpty(blocksetCondition)) fullCondition += " AND " + blocksetCondition; var sql = $@" SELECT COUNT(*) FROM ""{tmpName}"" ""{alias}"" WHERE {fullCondition} "; return await cmd .SetCommandAndParameters(sql) .ExecuteScalarInt64Async(0, token) .ConfigureAwait(false); } /// /// Generates the blockset condition for the SQL query /// /// The alias for the temporary table /// The ID of the blockset to count, or null for all /// The blocksets to exclude from the count /// >The blockset condition for the SQL query private static string GetBlocksetCondition(string alias, long? blocksetId, long[]? exclude) { if (blocksetId.HasValue) return $@"{alias}.""BlocksetID"" = {Library.Utility.Utility.FormatInvariantValue(blocksetId.Value)}"; if (exclude?.Length > 0) { var formatted_exclude = exclude.Select(x => Library.Utility.Utility.FormatInvariantValue(x)); return $@"{alias}.""BlocksetID"" NOT IN ({string.Join(",", formatted_exclude)})"; } return string.Empty; } }