// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Data.Sqlite;
namespace Duplicati.Library.Main.Database;
///
/// Encapsulates the logic for calculating change statistics
///
public static class ChangeStatistics
{
///
/// The tag used for log messages
///
private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(ChangeStatistics));
///
/// Calculates the change statistics for the current and previous fileset.
///
/// The database command to use.
/// The results object to update with the statistics.
/// The ID of the current fileset.
/// The ID of the previous fileset.
/// The cancellation token to monitor for cancellation requests.
/// A task that completes when the statistics are updated.
internal static async Task UpdateChangeStatistics(SqliteCommand cmd, BackupResults results, long currentFilesetId, long previousFilesetId, CancellationToken token)
{
var tmpName = $"TmpFileState_{Library.Utility.Utility.GetHexGuid()}";
try
{
// Create temp table
await cmd.SetCommandAndParameters($@"
CREATE TEMP TABLE ""{tmpName}"" AS
SELECT
""FL"".""PrefixID"",
""FL"".""Path"",
""FL"".""BlocksetID"",
""BS_Meta"".""Fullhash"" AS ""Metahash"",
CASE ""FE"".""FilesetID""
WHEN @LastFilesetId THEN 0
WHEN @CurrentFilesetId THEN 1
END AS ""Source""
FROM ""FileLookup"" ""FL""
JOIN ""FilesetEntry"" ""FE""
ON ""FL"".""ID"" = ""FE"".""FileID""
LEFT JOIN ""Blockset"" ""BS_Data""
ON ""FL"".""BlocksetID"" = ""BS_Data"".""ID""
LEFT JOIN ""Metadataset"" M
ON ""FL"".""MetadataID"" = M.""ID""
LEFT JOIN ""Blockset"" ""BS_Meta""
ON M.""BlocksetID"" = ""BS_Meta"".""ID""
WHERE ""FE"".""FilesetID"" IN (
@LastFilesetId,
@CurrentFilesetId
);
")
.SetParameterValue("@LastFilesetId", previousFilesetId)
.SetParameterValue("@CurrentFilesetId", currentFilesetId)
.ExecuteNonQueryAsync(token)
.ConfigureAwait(false);
// Index for fast comparison
await cmd.ExecuteNonQueryAsync($@"
CREATE INDEX ""idx_{tmpName}""
ON ""{tmpName}"" (
""PrefixID"",
""Path""
)
", token)
.ConfigureAwait(false);
// Added
results.AddedFolders = await CountAdded(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID, null, token)
.ConfigureAwait(false);
results.AddedSymlinks = await CountAdded(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID, null, token)
.ConfigureAwait(false);
results.AddedFiles = await CountAdded(cmd, tmpName, null, [
LocalDatabase.FOLDER_BLOCKSET_ID,
LocalDatabase.SYMLINK_BLOCKSET_ID
], token)
.ConfigureAwait(false);
// Deleted
results.DeletedFolders = await CountDeleted(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID, null, token)
.ConfigureAwait(false);
results.DeletedSymlinks = await CountDeleted(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID, null, token)
.ConfigureAwait(false);
results.DeletedFiles = await CountDeleted(cmd, tmpName, null, [
LocalDatabase.FOLDER_BLOCKSET_ID,
LocalDatabase.SYMLINK_BLOCKSET_ID
], token)
.ConfigureAwait(false);
// Modified
results.ModifiedFolders = await CountModified(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID, null, token)
.ConfigureAwait(false);
results.ModifiedSymlinks = await CountModified(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID, null, token)
.ConfigureAwait(false);
results.ModifiedFiles = await CountModified(cmd, tmpName, null, [
LocalDatabase.FOLDER_BLOCKSET_ID,
LocalDatabase.SYMLINK_BLOCKSET_ID
], token)
.ConfigureAwait(false);
}
finally
{
try
{
await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmpName}"";", token)
.ConfigureAwait(false);
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(LOGTAG, "DropTemp", ex, $"Failed to drop {tmpName}");
}
}
}
///
/// Counts the number of added files or folders.
///
/// The database command to use.
/// The name of the temporary table.
/// The ID of the blockset to count, or null for all.
/// The blocksets to exclude from the count.
/// The cancellation token to monitor for cancellation requests.
/// A task that when awaited contains the number of added files or folders.
private static async Task CountAdded(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets, CancellationToken token)
{
var conditions = $@"
""Source"" = 1
AND NOT EXISTS (
SELECT 1
FROM ""{tmpName}"" ""B""
WHERE
""B"".""Source"" = 0
AND ""B"".""PrefixID"" = ""A"".""PrefixID""
AND ""B"".""Path"" = ""A"".""Path""
)
";
return await CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets, token)
.ConfigureAwait(false);
}
///
/// Counts the number of deleted files or folders.
///
/// The database command to use.
/// The name of the temporary table.
/// The ID of the blockset to count, or null for all.
/// The blocksets to exclude from the count.
/// The cancellation token to monitor for cancellation requests.
/// A task that when awaited contains the number of deleted files or folders.
private static async Task CountDeleted(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets, CancellationToken token)
{
var conditions = @$"
""Source"" = 0
AND NOT EXISTS (
SELECT 1
FROM ""{tmpName}"" ""B""
WHERE
""B"".""Source"" = 1
AND ""B"".""PrefixID"" = ""A"".""PrefixID""
AND ""B"".""Path"" = ""A"".""Path""
)
";
return await CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets, token)
.ConfigureAwait(false);
}
///
/// Counts the number of modified files or folders.
///
/// The database command to use.
/// The name of the temporary table.
/// The ID of the blockset to count, or null for all.
/// The blocksets to exclude from the count.
/// The cancellation token to monitor for cancellation requests.
/// A task that when awaited contains the number of modified files or folders.
private static async Task CountModified(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets, CancellationToken token)
{
string conditions;
if (blocksetId == LocalDatabase.FOLDER_BLOCKSET_ID || blocksetId == LocalDatabase.SYMLINK_BLOCKSET_ID)
{
conditions = @"
""A"".""Source"" = 0
AND ""B"".""Source"" = 1
AND ""A"".""PrefixID"" = ""B"".""PrefixID""
AND ""A"".""Path"" = ""B"".""Path""
AND ""A"".""Metahash"" IS NOT ""B"".""Metahash""
";
}
else
{
conditions = @"
""A"".""Source"" = 0
AND ""B"".""Source"" = 1
AND ""A"".""PrefixID"" = ""B"".""PrefixID""
AND ""A"".""Path"" = ""B"".""Path""
AND (
""A"".""BlocksetID"" IS NOT ""B"".""BlocksetID""
OR ""A"".""Metahash"" IS NOT ""B"".""Metahash""
)
";
}
string blocksetCondition = GetBlocksetCondition("A", blocksetId, excludeBlocksets);
if (!string.IsNullOrEmpty(blocksetCondition))
conditions += $" AND {blocksetCondition}";
string sql = $@"
SELECT COUNT(*)
FROM ""{tmpName}"" ""A""
JOIN ""{tmpName}"" ""B""
ON ""A"".""PrefixID"" = ""B"".""PrefixID""
AND ""A"".""Path"" = ""B"".""Path""
WHERE {conditions}
";
return await cmd
.SetCommandAndParameters(sql)
.ExecuteScalarInt64Async(0, token)
.ConfigureAwait(false);
}
///
/// Counts the number of files or folders with a specific condition.
///
/// The database command to use.
/// The name of the temporary table.
/// The alias for the temporary table.
/// The base condition for the count.
/// The ID of the blockset to count, or null for all.
/// The blocksets to exclude from the count.
/// The cancellation token to monitor for cancellation requests.
/// A task that when awaited contains the number of files or folders that match the condition.
private static async Task CountWithCondition(SqliteCommand cmd, string tmpName, string alias, string baseCondition, long? blocksetId, long[]? excludeBlocksets, CancellationToken token)
{
var fullCondition = baseCondition;
var blocksetCondition = GetBlocksetCondition(alias, blocksetId, excludeBlocksets);
if (!string.IsNullOrEmpty(blocksetCondition))
fullCondition += " AND " + blocksetCondition;
var sql = $@"
SELECT COUNT(*)
FROM ""{tmpName}"" ""{alias}""
WHERE {fullCondition}
";
return await cmd
.SetCommandAndParameters(sql)
.ExecuteScalarInt64Async(0, token)
.ConfigureAwait(false);
}
///
/// Generates the blockset condition for the SQL query
///
/// The alias for the temporary table
/// The ID of the blockset to count, or null for all
/// The blocksets to exclude from the count
/// >The blockset condition for the SQL query
private static string GetBlocksetCondition(string alias, long? blocksetId, long[]? exclude)
{
if (blocksetId.HasValue)
return $@"{alias}.""BlocksetID"" = {Library.Utility.Utility.FormatInvariantValue(blocksetId.Value)}";
if (exclude?.Length > 0)
{
var formatted_exclude = exclude.Select(x => Library.Utility.Utility.FormatInvariantValue(x));
return $@"{alias}.""BlocksetID"" NOT IN ({string.Join(",", formatted_exclude)})";
}
return string.Empty;
}
}