// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal in the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Data; using System.Collections.Generic; using Duplicati.Library.Utility; namespace Duplicati.Library.Main.Database { internal class LocalListChangesDatabase : LocalDatabase { public LocalListChangesDatabase(string path) : base(path, "ListChanges", false) { ShouldCloseConnection = true; } public interface IStorageHelper : IDisposable { void AddElement(string path, string filehash, string metahash, long size, Interface.ListChangesElementType type, bool asNew); void AddFromDb(long filesetId, bool asNew, IFilter filter); IChangeCountReport CreateChangeCountReport(); IChangeSizeReport CreateChangeSizeReport(); IEnumerable> CreateChangedFileReport(); } public interface IChangeCountReport { long AddedFolders { get; } long AddedSymlinks { get; } long AddedFiles { get; } long DeletedFolders { get; } long DeletedSymlinks { get; } long DeletedFiles { get; } long ModifiedFolders { get; } long ModifiedSymlinks { get; } long ModifiedFiles { get; } } public interface IChangeSizeReport { long AddedSize { get; } long DeletedSize { get; } long PreviousSize { get; } long CurrentSize { get; } } internal class ChangeCountReport : IChangeCountReport { public long AddedFolders { get; internal set; } public long AddedSymlinks { get; internal set; } public long AddedFiles { get; internal set; } public long DeletedFolders { get; internal set; } public long DeletedSymlinks { get; internal set; } public long DeletedFiles { get; internal set; } public long ModifiedFolders { get; internal set; } public long ModifiedSymlinks { get; internal set; } public long ModifiedFiles { get; internal set; } } internal class ChangeSizeReport : IChangeSizeReport { public long AddedSize { get; internal set; } public long DeletedSize { get; internal set; } public long PreviousSize { get; internal set; } public long CurrentSize { get; internal set; } } private class StorageHelper : IStorageHelper { private readonly IDbConnection m_connection; private IDbTransaction m_transaction; private IDbCommand m_insertPreviousElementCommand; private IDbCommand m_insertCurrentElementCommand; private string m_previousTable; private string m_currentTable; public StorageHelper(IDbConnection con) { m_connection = con; m_previousTable = "Previous-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); m_currentTable = "Current-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); m_transaction = m_connection.BeginTransaction(); using (var cmd = m_connection.CreateCommand(m_transaction)) { cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_previousTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ")); cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_currentTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ")); } m_insertPreviousElementCommand = m_connection.CreateCommand(m_transaction, FormatInvariant($@"INSERT INTO ""{m_previousTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)")); m_insertCurrentElementCommand = m_connection.CreateCommand(FormatInvariant($@"INSERT INTO ""{m_currentTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)")); } public void AddFromDb(long filesetId, bool asNew, IFilter filter) { var tablename = asNew ? m_currentTable : m_previousTable; var folders = FormatInvariant($@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {(int)Interface.ListChangesElementType.Folder} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -100 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "); var symlinks = FormatInvariant($@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {(int)Interface.ListChangesElementType.Symlink} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -200 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "); var files = FormatInvariant($@"SELECT ""File"".""Path"" AS ""Path"", ""FB"".""FullHash"" AS ""FileHash"", ""MB"".""Fullhash"" AS ""MetaHash"", ""FB"".""Length"" AS ""Size"", {(int)Interface.ListChangesElementType.File} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" MB, ""Blockset"" FB WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" >= 0 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""MB"".""ID"" AND ""File"".""BlocksetID"" = ""FB"".""ID"" "); var combined = "(" + folders + " UNION " + symlinks + " UNION " + files + ")"; using (var cmd = m_connection.CreateCommand(m_transaction)) { if (filter == null || filter.Empty) { // Simple case, select everything cmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {combined} A WHERE ""A"".""FilesetID"" = ? "), filesetId); } else if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == Duplicati.Library.Utility.FilterType.Simple) { // File list based // unfortunately we cannot do this if the filesystem is case sensitive as // SQLite only supports ASCII compares var p = expression.GetSimpleList(); var filenamestable = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{filenamestable}"" (""Path"" TEXT NOT NULL) ")); cmd.CommandText = FormatInvariant($@"INSERT INTO ""{filenamestable}"" (""Path"") VALUES (?)"); cmd.AddParameter(); foreach (var s in p) { cmd.SetParameterValue(0, s); cmd.ExecuteNonQuery(); } string whereClause; if (expression.Result) { // Include filter whereClause = FormatInvariant($@"""A"".""FilesetID"" = ? AND ""A"".""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{filenamestable}"")"); } else { // Exclude filter whereClause = FormatInvariant($@"""A"".""FilesetID"" = ? AND ""A"".""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""{filenamestable}"")"); } cmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {combined} A WHERE {whereClause} "), filesetId); cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{filenamestable}"" ")); } else { // Do row-wise iteration var values = new object[5]; using (var cmd2 = m_connection.CreateCommand(m_transaction, FormatInvariant($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)"))) using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT ""A"".""Path"", ""A"".""FileHash"", ""A"".""MetaHash"", ""A"".""Size"", ""A"".""Type"" FROM {combined} A WHERE ""A"".""FilesetID"" = ?"), filesetId)) while (rd.Read()) { rd.GetValues(values); if (values[0] != null && values[0] != DBNull.Value && FilterExpression.Matches(filter, values[0].ToString())) { cmd2.SetParameterValue(0, values[0]); cmd2.SetParameterValue(1, values[1]); cmd2.SetParameterValue(2, values[2]); cmd2.SetParameterValue(3, values[3]); cmd2.SetParameterValue(4, values[4]); cmd2.ExecuteNonQuery(); } } } } } public void AddElement(string path, string filehash, string metahash, long size, Interface.ListChangesElementType type, bool asNew) { var cmd = asNew ? m_insertCurrentElementCommand : m_insertPreviousElementCommand; cmd.SetParameterValue(0, path); cmd.SetParameterValue(1, filehash); cmd.SetParameterValue(2, metahash); cmd.SetParameterValue(3, size); cmd.SetParameterValue(4, (int)type); cmd.ExecuteNonQuery(); } private static IEnumerable ReaderToStringList(IDataReader rd) { using (rd) while (rd.Read()) { var v = rd.GetValue(0); if (v == null || v == DBNull.Value) yield return null; else yield return v.ToString(); } } private Tuple GetSqls(bool allTypes) { return new Tuple( FormatInvariant($@"SELECT ""Path"" FROM ""{m_currentTable}"" WHERE {(allTypes ? "" : FormatInvariant(@$" ""{m_currentTable}"".""Type"" = ? AND "))} ""{m_currentTable}"".""Path"" NOT IN (SELECT ""Path"" FROM ""{m_previousTable}"")"), FormatInvariant($@"SELECT ""Path"" FROM ""{m_previousTable}"" WHERE {(allTypes ? "" : FormatInvariant(@$" ""{m_previousTable}"".""Type"" = ? AND "))} ""{m_previousTable}"".""Path"" NOT IN (SELECT ""Path"" FROM ""{m_currentTable}"")"), FormatInvariant($@"SELECT ""{m_currentTable}"".""Path"" FROM ""{m_currentTable}"",""{m_previousTable}"" WHERE {(allTypes ? "" : FormatInvariant($@" ""{m_currentTable}"".""Type"" = ? AND "))} ""{m_currentTable}"".""Path"" = ""{m_previousTable}"".""Path"" AND (""{m_currentTable}"".""FileHash"" != ""{m_previousTable}"".""FileHash"" OR ""{m_currentTable}"".""MetaHash"" != ""{m_previousTable}"".""MetaHash"" OR ""{m_currentTable}"".""Type"" != ""{m_previousTable}"".""Type"") ") ); } public IChangeSizeReport CreateChangeSizeReport() { var sqls = GetSqls(true); var added = sqls.Item1; var deleted = sqls.Item2; //var modified = sqls.Item3; using (var cmd = m_connection.CreateCommand(m_transaction)) { var result = new ChangeSizeReport(); result.PreviousSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_previousTable}"" "), 0); result.CurrentSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_currentTable}"" "), 0); result.AddedSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_currentTable}"" WHERE ""{m_currentTable}"".""Path"" IN ({added}) "), 0); result.DeletedSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_previousTable}"" WHERE ""{m_previousTable}"".""Path"" IN ({deleted}) "), 0); return result; } } public IChangeCountReport CreateChangeCountReport() { var sqls = GetSqls(false); var added = @"SELECT COUNT(*) FROM (" + sqls.Item1 + ")"; var deleted = @"SELECT COUNT(*) FROM (" + sqls.Item2 + ")"; var modified = @"SELECT COUNT(*) FROM (" + sqls.Item3 + ")"; using (var cmd = m_connection.CreateCommand(m_transaction)) { var result = new ChangeCountReport(); result.AddedFolders = cmd.ExecuteScalarInt64(added, 0, (int)Interface.ListChangesElementType.Folder); result.AddedSymlinks = cmd.ExecuteScalarInt64(added, 0, (int)Interface.ListChangesElementType.Symlink); result.AddedFiles = cmd.ExecuteScalarInt64(added, 0, (int)Interface.ListChangesElementType.File); result.DeletedFolders = cmd.ExecuteScalarInt64(deleted, 0, (int)Interface.ListChangesElementType.Folder); result.DeletedSymlinks = cmd.ExecuteScalarInt64(deleted, 0, (int)Interface.ListChangesElementType.Symlink); result.DeletedFiles = cmd.ExecuteScalarInt64(deleted, 0, (int)Interface.ListChangesElementType.File); result.ModifiedFolders = cmd.ExecuteScalarInt64(modified, 0, (int)Interface.ListChangesElementType.Folder); result.ModifiedSymlinks = cmd.ExecuteScalarInt64(modified, 0, (int)Interface.ListChangesElementType.Symlink); result.ModifiedFiles = cmd.ExecuteScalarInt64(modified, 0, (int)Interface.ListChangesElementType.File); return result; } } public IEnumerable> CreateChangedFileReport() { var sqls = GetSqls(false); var added = sqls.Item1; var deleted = sqls.Item2; var modified = sqls.Item3; using (var cmd = m_connection.CreateCommand(m_transaction)) { foreach (var s in ReaderToStringList(cmd.ExecuteReader(added, (int)Interface.ListChangesElementType.Folder))) yield return new Tuple(Interface.ListChangesChangeType.Added, Interface.ListChangesElementType.Folder, s); foreach (var s in ReaderToStringList(cmd.ExecuteReader(added, (int)Interface.ListChangesElementType.Symlink))) yield return new Tuple(Interface.ListChangesChangeType.Added, Interface.ListChangesElementType.Symlink, s); foreach (var s in ReaderToStringList(cmd.ExecuteReader(added, (int)Interface.ListChangesElementType.File))) yield return new Tuple(Interface.ListChangesChangeType.Added, Interface.ListChangesElementType.File, s); foreach (var s in ReaderToStringList(cmd.ExecuteReader(deleted, (int)Interface.ListChangesElementType.Folder))) yield return new Tuple(Interface.ListChangesChangeType.Deleted, Interface.ListChangesElementType.Folder, s); foreach (var s in ReaderToStringList(cmd.ExecuteReader(deleted, (int)Interface.ListChangesElementType.Symlink))) yield return new Tuple(Interface.ListChangesChangeType.Deleted, Interface.ListChangesElementType.Symlink, s); foreach (var s in ReaderToStringList(cmd.ExecuteReader(deleted, (int)Interface.ListChangesElementType.File))) yield return new Tuple(Interface.ListChangesChangeType.Deleted, Interface.ListChangesElementType.File, s); foreach (var s in ReaderToStringList(cmd.ExecuteReader(modified, (int)Interface.ListChangesElementType.Folder))) yield return new Tuple(Interface.ListChangesChangeType.Modified, Interface.ListChangesElementType.Folder, s); foreach (var s in ReaderToStringList(cmd.ExecuteReader(modified, (int)Interface.ListChangesElementType.Symlink))) yield return new Tuple(Interface.ListChangesChangeType.Modified, Interface.ListChangesElementType.Symlink, s); foreach (var s in ReaderToStringList(cmd.ExecuteReader(modified, (int)Interface.ListChangesElementType.File))) yield return new Tuple(Interface.ListChangesChangeType.Modified, Interface.ListChangesElementType.File, s); } } public void Dispose() { if (m_insertPreviousElementCommand != null) { try { m_insertPreviousElementCommand.Dispose(); } catch { } finally { m_insertPreviousElementCommand = null; } } if (m_insertCurrentElementCommand != null) { try { m_insertCurrentElementCommand.Dispose(); } catch { } finally { m_insertCurrentElementCommand = null; } } if (m_transaction != null) { try { m_transaction.Rollback(); } catch { } finally { m_previousTable = null; m_currentTable = null; m_transaction = null; } } } } public IStorageHelper CreateStorageHelper() { return new StorageHelper(m_connection); } } }