haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updated ChangeStatistics to use Microsoft.Data.Sqlite and be async

Browse Source
This commit is contained in:
Carl Johnsen
2025-05-19 15:56:16 +02:00
parent f3f00faf8e
commit d08bcbba3a
1 changed files with 141 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Main/Database/ChangeStatistics.cs
+141 -83
View File
@@ -1,27 +1,28 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Data;
using System.Threading.Tasks;
using Microsoft.Data.Sqlite;
namespace Duplicati.Library.Main.Database;
@@ -42,57 +43,85 @@ public static class ChangeStatistics
/// <param name="results">The results object to update with the statistics</param>
/// <param name="currentFilesetId">The ID of the current fileset</param>
/// <param name="previousFilesetId">The ID of the previous fileset</param>
internal static void UpdateChangeStatistics(IDbCommand cmd, BackupResults results, long currentFilesetId, long previousFilesetId)
internal static async Task UpdateChangeStatistics(SqliteCommand cmd, BackupResults results, long currentFilesetId, long previousFilesetId)
{
var tmpName = $"TmpFileState_{Guid.NewGuid():N}";
try
{
// Create temp table
cmd.SetCommandAndParameters(LocalDatabase.FormatInvariant($@"
CREATE TEMP TABLE ""{tmpName}"" AS
SELECT
FL.""PrefixID"",
FL.""Path"",
FL.""BlocksetID"",
BS_Meta.""Fullhash"" AS ""Metahash"",
CASE FE.""FilesetID""
WHEN @LastFilesetId THEN 0
WHEN @CurrentFilesetId THEN 1
END AS Source
FROM ""FileLookup"" FL
JOIN ""FilesetEntry"" FE ON FL.""ID"" = FE.""FileID""
LEFT JOIN ""Blockset"" BS_Data ON FL.""BlocksetID"" = BS_Data.""ID""
LEFT JOIN ""Metadataset"" M ON FL.""MetadataID"" = M.""ID""
LEFT JOIN ""Blockset"" BS_Meta ON M.""BlocksetID"" = BS_Meta.""ID""
WHERE FE.""FilesetID"" IN (@LastFilesetId, @CurrentFilesetId);
"))
.SetParameterValue("@LastFilesetId", previousFilesetId)
.SetParameterValue("@CurrentFilesetId", currentFilesetId)
.ExecuteNonQuery();
await cmd.SetCommandAndParameters($@"
CREATE TEMP TABLE ""{tmpName}"" AS
SELECT
FL.""PrefixID"",
FL.""Path"",
FL.""BlocksetID"",
BS_Meta.""Fullhash"" AS ""Metahash"",
CASE FE.""FilesetID""
WHEN @LastFilesetId THEN 0
WHEN @CurrentFilesetId THEN 1
END AS Source
FROM ""FileLookup"" FL
JOIN ""FilesetEntry"" FE
ON FL.""ID"" = FE.""FileID""
LEFT JOIN ""Blockset"" BS_Data
ON FL.""BlocksetID"" = BS_Data.""ID""
LEFT JOIN ""Metadataset"" M
ON FL.""MetadataID"" = M.""ID""
LEFT JOIN ""Blockset"" BS_Meta
ON M.""BlocksetID"" = BS_Meta.""ID""
WHERE FE.""FilesetID"" IN (
@LastFilesetId,
@CurrentFilesetId
);
")
.SetParameterValue("@LastFilesetId", previousFilesetId)
.SetParameterValue("@CurrentFilesetId", currentFilesetId)
.ExecuteNonQueryAsync();
// Index for fast comparison
cmd.ExecuteNonQuery(LocalDatabase.FormatInvariant($@"CREATE INDEX ""idx_{tmpName}"" ON ""{tmpName}"" (""PrefixID"", ""Path"")"));
await cmd.ExecuteNonQueryAsync($@"
CREATE INDEX ""idx_{tmpName}""
ON ""{tmpName}"" (
""PrefixID"",
""Path""
)
");
// Added
results.AddedFolders = CountAdded(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID);
results.AddedSymlinks = CountAdded(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID);
results.AddedFiles = CountAdded(cmd, tmpName, null, new[] { LocalDatabase.FOLDER_BLOCKSET_ID, LocalDatabase.SYMLINK_BLOCKSET_ID });
results.AddedFolders = await CountAdded(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID);
results.AddedSymlinks = await CountAdded(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID);
results.AddedFiles = await CountAdded(cmd, tmpName, null, [
LocalDatabase.FOLDER_BLOCKSET_ID,
LocalDatabase.SYMLINK_BLOCKSET_ID
]);
// Deleted
results.DeletedFolders = CountDeleted(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID);
results.DeletedSymlinks = CountDeleted(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID);
results.DeletedFiles = CountDeleted(cmd, tmpName, null, new[] { LocalDatabase.FOLDER_BLOCKSET_ID, LocalDatabase.SYMLINK_BLOCKSET_ID });
results.DeletedFolders = await CountDeleted(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID);
results.DeletedSymlinks = await CountDeleted(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID);
results.DeletedFiles = await CountDeleted(cmd, tmpName, null, [
LocalDatabase.FOLDER_BLOCKSET_ID,
LocalDatabase.SYMLINK_BLOCKSET_ID
]);
// Modified
results.ModifiedFolders = CountModified(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID);
results.ModifiedSymlinks = CountModified(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID);
results.ModifiedFiles = CountModified(cmd, tmpName, null, new[] { LocalDatabase.FOLDER_BLOCKSET_ID, LocalDatabase.SYMLINK_BLOCKSET_ID });
results.ModifiedFolders = await CountModified(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID);
results.ModifiedSymlinks = await CountModified(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID);
results.ModifiedFiles = await CountModified(cmd, tmpName, null, [
LocalDatabase.FOLDER_BLOCKSET_ID,
LocalDatabase.SYMLINK_BLOCKSET_ID
]);
}
finally
{
try { cmd.ExecuteNonQuery(LocalDatabase.FormatInvariant($@"DROP TABLE IF EXISTS ""{tmpName}"";")); }
catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "DropTemp", ex, $"Failed to drop {tmpName}"); }
try
{
await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmpName}"";");
}
catch (Exception ex)
{
Logging.Log.WriteWarningMessage(LOGTAG, "DropTemp", ex, $"Failed to drop {tmpName}");
}
}
}
@@ -104,16 +133,21 @@ public static class ChangeStatistics
/// <param name="blocksetId">The ID of the blockset to count, or null for all</param>
/// <param name="excludeBlocksets">The blocksets to exclude from the count</param>
/// <returns>The number of added files or folders</returns>
private static long CountAdded(IDbCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null)
private static async Task<long> CountAdded(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null)
{
var conditions = LocalDatabase.FormatInvariant($@"Source = 1 AND NOT EXISTS (
SELECT 1 FROM ""{tmpName}"" B
WHERE B.Source = 0
AND B.PrefixID = A.PrefixID
AND B.Path = A.Path
)");
var conditions = $@"
Source = 1
AND NOT EXISTS (
SELECT 1
FROM ""{tmpName}"" B
WHERE
B.Source = 0
AND B.PrefixID = A.PrefixID
AND B.Path = A.Path
)
";
return CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets);
return await CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets);
}
/// <summary>
@@ -124,16 +158,21 @@ public static class ChangeStatistics
/// <param name="blocksetId">The ID of the blockset to count, or null for all</param>
/// <param name="excludeBlocksets">The blocksets to exclude from the count</param>
/// <returns>The number of deleted files or folders</returns>
private static long CountDeleted(IDbCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null)
private static async Task<long> CountDeleted(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null)
{
var conditions = @"Source = 0 AND NOT EXISTS (
SELECT 1 FROM """ + tmpName + @""" B
WHERE B.Source = 1
AND B.PrefixID = A.PrefixID
AND B.Path = A.Path
)";
var conditions = @$"
Source = 0
AND NOT EXISTS (
SELECT 1
FROM ""{tmpName}"" B
WHERE
B.Source = 1
AND B.PrefixID = A.PrefixID
AND B.Path = A.Path
)
";
return CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets);
return await CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets);
}
/// <summary>
@@ -145,23 +184,32 @@ public static class ChangeStatistics
/// <param name="hash1">The hash column to compare</param>
/// <param name="excludeBlocksets">The blocksets to exclude from the count</param>
/// <returns>The number of modified files or folders</returns>
private static long CountModified(IDbCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null)
private static async Task<long> CountModified(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null)
{
string conditions;
if (blocksetId == LocalDatabase.FOLDER_BLOCKSET_ID || blocksetId == LocalDatabase.SYMLINK_BLOCKSET_ID)
{
conditions = @"
A.Source = 0 AND B.Source = 1
AND A.PrefixID = B.PrefixID AND A.Path = B.Path
AND A.Metahash IS NOT B.Metahash";
A.Source = 0
AND B.Source = 1
AND A.PrefixID = B.PrefixID
AND A.Path = B.Path
AND A.Metahash IS NOT B.Metahash
";
}
else
{
conditions = @"
A.Source = 0 AND B.Source = 1
AND A.PrefixID = B.PrefixID AND A.Path = B.Path
AND (A.BlocksetID IS NOT B.BlocksetID OR A.Metahash IS NOT B.Metahash)";
A.Source = 0
AND B.Source = 1
AND A.PrefixID = B.PrefixID
AND A.Path = B.Path
AND (
A.BlocksetID IS NOT B.BlocksetID
OR A.Metahash IS NOT B.Metahash
)
";
}
string blocksetCondition = GetBlocksetCondition("A", blocksetId, excludeBlocksets);
@@ -169,12 +217,16 @@ public static class ChangeStatistics
conditions += $" AND {blocksetCondition}";
string sql = $@"
SELECT COUNT(*)
FROM ""{tmpName}"" A
JOIN ""{tmpName}"" B ON A.PrefixID = B.PrefixID AND A.Path = B.Path
WHERE {conditions}";
SELECT COUNT(*)
FROM ""{tmpName}"" A
JOIN ""{tmpName}"" B
ON A.PrefixID = B.PrefixID
AND A.Path = B.Path
WHERE {conditions}
";
return cmd.SetCommandAndParameters(sql).ExecuteScalarInt64(0);
return await cmd.SetCommandAndParameters(sql)
.ExecuteScalarInt64Async(0);
}
/// <summary>
@@ -187,15 +239,21 @@ public static class ChangeStatistics
/// <param name="blocksetId">The ID of the blockset to count, or null for all</param>
/// <param name="excludeBlocksets">The blocksets to exclude from the count</param>
/// <returns>The number of files or folders that match the condition</returns>
private static long CountWithCondition(IDbCommand cmd, string tmpName, string alias, string baseCondition, long? blocksetId, long[]? excludeBlocksets)
private static async Task<long> CountWithCondition(SqliteCommand cmd, string tmpName, string alias, string baseCondition, long? blocksetId, long[]? excludeBlocksets)
{
var fullCondition = baseCondition;
var blocksetCondition = GetBlocksetCondition(alias, blocksetId, excludeBlocksets);
if (!string.IsNullOrEmpty(blocksetCondition))
fullCondition += " AND " + blocksetCondition;
var sql = LocalDatabase.FormatInvariant($@"SELECT COUNT(*) FROM ""{tmpName}"" {alias} WHERE {fullCondition}");
return cmd.SetCommandAndParameters(sql).ExecuteScalarInt64(0);
var sql = $@"
SELECT COUNT(*)
FROM ""{tmpName}"" {alias}
WHERE {fullCondition}
";
return await cmd.SetCommandAndParameters(sql)
.ExecuteScalarInt64Async(0);
}
/// <summary>
@@ -208,9 +266,9 @@ public static class ChangeStatistics
private static string GetBlocksetCondition(string alias, long? blocksetId, long[]? exclude)
{
if (blocksetId.HasValue)
return LocalDatabase.FormatInvariant($"{alias}.\"BlocksetID\" = {blocksetId.Value}");
return @$"{alias}.""BlocksetID"" = {blocksetId.Value}";
if (exclude?.Length > 0)
return LocalDatabase.FormatInvariant($"{alias}.\"BlocksetID\" NOT IN ({string.Join(",", exclude)})");
return @$"{alias}.""BlocksetID"" NOT IN ({string.Join(",", exclude)})";
return string.Empty;
}
}