duplicati/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs

// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
// 
// Permission is hereby granted, free of charge, to any person obtaining a 
// copy of this software and associated documentation files (the "Software"), 
// to deal in the Software without restriction, including without limitation 
// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
// and/or sell copies of the Software, and to permit persons to whom the 
// Software is furnished to do so, subject to the following conditions:
// 
// The above copyright notice and this permission notice shall be included in 
// all copies or substantial portions of the Software.
// 
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
// DEALINGS IN THE SOFTWARE.

using System;
using System.Data;
using System.Collections.Generic;
using Duplicati.Library.Utility;

namespace Duplicati.Library.Main.Database
{
    internal class LocalListChangesDatabase : LocalDatabase
    {
        public LocalListChangesDatabase(string path)
            : base(path, "ListChanges", false)
        {
            ShouldCloseConnection = true;
        }

        public interface IStorageHelper : IDisposable
        {
            void AddElement(string path, string filehash, string metahash, long size, Interface.ListChangesElementType type, bool asNew);

            void AddFromDb(long filesetId, bool asNew, IFilter filter);

            IChangeCountReport CreateChangeCountReport();
            IChangeSizeReport CreateChangeSizeReport();
            IEnumerable<Tuple<Interface.ListChangesChangeType, Interface.ListChangesElementType, string>> CreateChangedFileReport();
        }

        public interface IChangeCountReport
        {
            long AddedFolders { get; }
            long AddedSymlinks { get; }
            long AddedFiles { get; }

            long DeletedFolders { get; }
            long DeletedSymlinks { get; }
            long DeletedFiles { get; }

            long ModifiedFolders { get; }
            long ModifiedSymlinks { get; }
            long ModifiedFiles { get; }
        }

        public interface IChangeSizeReport
        {
            long AddedSize { get; }
            long DeletedSize { get; }

            long PreviousSize { get; }
            long CurrentSize { get; }
        }

        internal class ChangeCountReport : IChangeCountReport
        {
            public long AddedFolders { get; internal set; }
            public long AddedSymlinks { get; internal set; }
            public long AddedFiles { get; internal set; }

            public long DeletedFolders { get; internal set; }
            public long DeletedSymlinks { get; internal set; }
            public long DeletedFiles { get; internal set; }

            public long ModifiedFolders { get; internal set; }
            public long ModifiedSymlinks { get; internal set; }
            public long ModifiedFiles { get; internal set; }
        }

        internal class ChangeSizeReport : IChangeSizeReport
        {
            public long AddedSize { get; internal set; }
            public long DeletedSize { get; internal set; }

            public long PreviousSize { get; internal set; }
            public long CurrentSize { get; internal set; }
        }

        private class StorageHelper : IStorageHelper
        {
            private readonly IDbConnection m_connection;
            private IDbTransaction m_transaction;

            private IDbCommand m_insertPreviousElementCommand;
            private IDbCommand m_insertCurrentElementCommand;

            private string m_previousTable;
            private string m_currentTable;

            public StorageHelper(IDbConnection con)
            {
                m_connection = con;
                m_previousTable = "Previous-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
                m_currentTable = "Current-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());

                m_transaction = m_connection.BeginTransaction();

                using (var cmd = m_connection.CreateCommand(m_transaction))
                {
                    cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_previousTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) "));
                    cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_currentTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) "));
                }

                m_insertPreviousElementCommand = m_connection.CreateCommand(m_transaction, FormatInvariant($@"INSERT INTO ""{m_previousTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)"));
                m_insertCurrentElementCommand = m_connection.CreateCommand(FormatInvariant($@"INSERT INTO ""{m_currentTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)"));
            }

            public void AddFromDb(long filesetId, bool asNew, IFilter filter)
            {
                var tablename = asNew ? m_currentTable : m_previousTable;

                var folders = FormatInvariant($@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {(int)Interface.ListChangesElementType.Folder} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -100 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" ");
                var symlinks = FormatInvariant($@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {(int)Interface.ListChangesElementType.Symlink} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -200 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" ");
                var files = FormatInvariant($@"SELECT ""File"".""Path"" AS ""Path"", ""FB"".""FullHash"" AS ""FileHash"", ""MB"".""Fullhash"" AS ""MetaHash"", ""FB"".""Length"" AS ""Size"", {(int)Interface.ListChangesElementType.File} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" MB, ""Blockset"" FB WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" >= 0 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""MB"".""ID"" AND ""File"".""BlocksetID"" = ""FB"".""ID"" ");
                var combined = "(" + folders + " UNION " + symlinks + " UNION " + files + ")";


                using (var cmd = m_connection.CreateCommand(m_transaction))
                {
                    if (filter == null || filter.Empty)
                    {
                        // Simple case, select everything
                        cmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {combined} A WHERE ""A"".""FilesetID"" = ? "), filesetId);
                    }
                    else if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == Duplicati.Library.Utility.FilterType.Simple)
                    {
                        // File list based
                        // unfortunately we cannot do this if the filesystem is case sensitive as
                        // SQLite only supports ASCII compares
                        var p = expression.GetSimpleList();
                        var filenamestable = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
                        cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{filenamestable}"" (""Path"" TEXT NOT NULL) "));
                        cmd.CommandText = FormatInvariant($@"INSERT INTO ""{filenamestable}"" (""Path"") VALUES (?)");
                        cmd.AddParameter();

                        foreach (var s in p)
                        {
                            cmd.SetParameterValue(0, s);
                            cmd.ExecuteNonQuery();
                        }

                        string whereClause;
                        if (expression.Result)
                        {
                            // Include filter
                            whereClause = FormatInvariant($@"""A"".""FilesetID"" = ? AND ""A"".""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{filenamestable}"")");
                        }
                        else
                        {
                            // Exclude filter
                            whereClause = FormatInvariant($@"""A"".""FilesetID"" = ? AND ""A"".""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""{filenamestable}"")");
                        }
                        cmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {combined} A WHERE {whereClause} "), filesetId);
                        cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{filenamestable}"" "));
                    }
                    else
                    {
                        // Do row-wise iteration
                        var values = new object[5];
                        using (var cmd2 = m_connection.CreateCommand(m_transaction, FormatInvariant($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)")))
                        using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT ""A"".""Path"", ""A"".""FileHash"", ""A"".""MetaHash"", ""A"".""Size"", ""A"".""Type"" FROM {combined} A WHERE ""A"".""FilesetID"" = ?"), filesetId))
                            while (rd.Read())
                            {
                                rd.GetValues(values);
                                if (values[0] != null && values[0] != DBNull.Value && FilterExpression.Matches(filter, values[0].ToString()))
                                {
                                    cmd2.SetParameterValue(0, values[0]);
                                    cmd2.SetParameterValue(1, values[1]);
                                    cmd2.SetParameterValue(2, values[2]);
                                    cmd2.SetParameterValue(3, values[3]);
                                    cmd2.SetParameterValue(4, values[4]);
                                    cmd2.ExecuteNonQuery();
                                }
                            }
                    }
                }
            }

            public void AddElement(string path, string filehash, string metahash, long size, Interface.ListChangesElementType type, bool asNew)
            {
                var cmd = asNew ? m_insertCurrentElementCommand : m_insertPreviousElementCommand;
                cmd.SetParameterValue(0, path);
                cmd.SetParameterValue(1, filehash);
                cmd.SetParameterValue(2, metahash);
                cmd.SetParameterValue(3, size);
                cmd.SetParameterValue(4, (int)type);
                cmd.ExecuteNonQuery();
            }

            private static IEnumerable<string> ReaderToStringList(IDataReader rd)
            {
                using (rd)
                    while (rd.Read())
                    {
                        var v = rd.GetValue(0);
                        if (v == null || v == DBNull.Value)
                            yield return null;
                        else
                            yield return v.ToString();
                    }
            }

            private Tuple<string, string, string> GetSqls(bool allTypes)
            {
                return new Tuple<string, string, string>(
                    FormatInvariant($@"SELECT ""Path"" FROM ""{m_currentTable}"" WHERE {(allTypes ? "" : FormatInvariant(@$" ""{m_currentTable}"".""Type"" = ? AND "))} ""{m_currentTable}"".""Path"" NOT IN (SELECT ""Path"" FROM ""{m_previousTable}"")"),
                    FormatInvariant($@"SELECT ""Path"" FROM ""{m_previousTable}"" WHERE {(allTypes ? "" : FormatInvariant(@$" ""{m_previousTable}"".""Type"" = ? AND "))} ""{m_previousTable}"".""Path"" NOT IN (SELECT ""Path"" FROM ""{m_currentTable}"")"),
                    FormatInvariant($@"SELECT ""{m_currentTable}"".""Path"" FROM ""{m_currentTable}"",""{m_previousTable}"" WHERE {(allTypes ? "" : FormatInvariant($@" ""{m_currentTable}"".""Type"" = ? AND "))} ""{m_currentTable}"".""Path"" = ""{m_previousTable}"".""Path"" AND (""{m_currentTable}"".""FileHash"" != ""{m_previousTable}"".""FileHash"" OR ""{m_currentTable}"".""MetaHash"" != ""{m_previousTable}"".""MetaHash"" OR ""{m_currentTable}"".""Type"" != ""{m_previousTable}"".""Type"") ")
                );
            }

            public IChangeSizeReport CreateChangeSizeReport()
            {
                var sqls = GetSqls(true);
                var added = sqls.Item1;
                var deleted = sqls.Item2;
                //var modified = sqls.Item3;

                using (var cmd = m_connection.CreateCommand(m_transaction))
                {
                    var result = new ChangeSizeReport();

                    result.PreviousSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_previousTable}"" "), 0);
                    result.CurrentSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_currentTable}"" "), 0);

                    result.AddedSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_currentTable}"" WHERE ""{m_currentTable}"".""Path"" IN ({added}) "), 0);
                    result.DeletedSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_previousTable}"" WHERE ""{m_previousTable}"".""Path"" IN ({deleted}) "), 0);

                    return result;
                }
            }

            public IChangeCountReport CreateChangeCountReport()
            {
                var sqls = GetSqls(false);
                var added = @"SELECT COUNT(*) FROM (" + sqls.Item1 + ")";
                var deleted = @"SELECT COUNT(*) FROM (" + sqls.Item2 + ")";
                var modified = @"SELECT COUNT(*) FROM (" + sqls.Item3 + ")";

                using (var cmd = m_connection.CreateCommand(m_transaction))
                {
                    var result = new ChangeCountReport();
                    result.AddedFolders = cmd.ExecuteScalarInt64(added, 0, (int)Interface.ListChangesElementType.Folder);
                    result.AddedSymlinks = cmd.ExecuteScalarInt64(added, 0, (int)Interface.ListChangesElementType.Symlink);
                    result.AddedFiles = cmd.ExecuteScalarInt64(added, 0, (int)Interface.ListChangesElementType.File);

                    result.DeletedFolders = cmd.ExecuteScalarInt64(deleted, 0, (int)Interface.ListChangesElementType.Folder);
                    result.DeletedSymlinks = cmd.ExecuteScalarInt64(deleted, 0, (int)Interface.ListChangesElementType.Symlink);
                    result.DeletedFiles = cmd.ExecuteScalarInt64(deleted, 0, (int)Interface.ListChangesElementType.File);

                    result.ModifiedFolders = cmd.ExecuteScalarInt64(modified, 0, (int)Interface.ListChangesElementType.Folder);
                    result.ModifiedSymlinks = cmd.ExecuteScalarInt64(modified, 0, (int)Interface.ListChangesElementType.Symlink);
                    result.ModifiedFiles = cmd.ExecuteScalarInt64(modified, 0, (int)Interface.ListChangesElementType.File);

                    return result;
                }
            }

            public IEnumerable<Tuple<Interface.ListChangesChangeType, Interface.ListChangesElementType, string>> CreateChangedFileReport()
            {
                var sqls = GetSqls(false);
                var added = sqls.Item1;
                var deleted = sqls.Item2;
                var modified = sqls.Item3;

                using (var cmd = m_connection.CreateCommand(m_transaction))
                {
                    foreach (var s in ReaderToStringList(cmd.ExecuteReader(added, (int)Interface.ListChangesElementType.Folder)))
                        yield return new Tuple<Interface.ListChangesChangeType, Interface.ListChangesElementType, string>(Interface.ListChangesChangeType.Added, Interface.ListChangesElementType.Folder, s);
                    foreach (var s in ReaderToStringList(cmd.ExecuteReader(added, (int)Interface.ListChangesElementType.Symlink)))
                        yield return new Tuple<Interface.ListChangesChangeType, Interface.ListChangesElementType, string>(Interface.ListChangesChangeType.Added, Interface.ListChangesElementType.Symlink, s);
                    foreach (var s in ReaderToStringList(cmd.ExecuteReader(added, (int)Interface.ListChangesElementType.File)))
                        yield return new Tuple<Interface.ListChangesChangeType, Interface.ListChangesElementType, string>(Interface.ListChangesChangeType.Added, Interface.ListChangesElementType.File, s);

                    foreach (var s in ReaderToStringList(cmd.ExecuteReader(deleted, (int)Interface.ListChangesElementType.Folder)))
                        yield return new Tuple<Interface.ListChangesChangeType, Interface.ListChangesElementType, string>(Interface.ListChangesChangeType.Deleted, Interface.ListChangesElementType.Folder, s);
                    foreach (var s in ReaderToStringList(cmd.ExecuteReader(deleted, (int)Interface.ListChangesElementType.Symlink)))
                        yield return new Tuple<Interface.ListChangesChangeType, Interface.ListChangesElementType, string>(Interface.ListChangesChangeType.Deleted, Interface.ListChangesElementType.Symlink, s);
                    foreach (var s in ReaderToStringList(cmd.ExecuteReader(deleted, (int)Interface.ListChangesElementType.File)))
                        yield return new Tuple<Interface.ListChangesChangeType, Interface.ListChangesElementType, string>(Interface.ListChangesChangeType.Deleted, Interface.ListChangesElementType.File, s);

                    foreach (var s in ReaderToStringList(cmd.ExecuteReader(modified, (int)Interface.ListChangesElementType.Folder)))
                        yield return new Tuple<Interface.ListChangesChangeType, Interface.ListChangesElementType, string>(Interface.ListChangesChangeType.Modified, Interface.ListChangesElementType.Folder, s);
                    foreach (var s in ReaderToStringList(cmd.ExecuteReader(modified, (int)Interface.ListChangesElementType.Symlink)))
                        yield return new Tuple<Interface.ListChangesChangeType, Interface.ListChangesElementType, string>(Interface.ListChangesChangeType.Modified, Interface.ListChangesElementType.Symlink, s);
                    foreach (var s in ReaderToStringList(cmd.ExecuteReader(modified, (int)Interface.ListChangesElementType.File)))
                        yield return new Tuple<Interface.ListChangesChangeType, Interface.ListChangesElementType, string>(Interface.ListChangesChangeType.Modified, Interface.ListChangesElementType.File, s);
                }
            }

            public void Dispose()
            {
                if (m_insertPreviousElementCommand != null)
                {
                    try { m_insertPreviousElementCommand.Dispose(); }
                    catch { }
                    finally { m_insertPreviousElementCommand = null; }
                }

                if (m_insertCurrentElementCommand != null)
                {
                    try { m_insertCurrentElementCommand.Dispose(); }
                    catch { }
                    finally { m_insertCurrentElementCommand = null; }
                }

                if (m_transaction != null)
                {
                    try { m_transaction.Rollback(); }
                    catch { }
                    finally
                    {
                        m_previousTable = null;
                        m_currentTable = null;
                        m_transaction = null;
                    }
                }
            }
        }

        public IStorageHelper CreateStorageHelper()
        {
            return new StorageHelper(m_connection);
        }
    }
}