// Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal in the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable using System; using System.Data; using System.Linq; using System.Collections.Generic; using Duplicati.Library.Common.IO; using Duplicati.Library.Utility; using Duplicati.Library.Interface; namespace Duplicati.Library.Main.Database { internal class LocalListDatabase : LocalDatabase { public LocalListDatabase(string path, long pagecachesize) : base(path, "List", false, pagecachesize) { ShouldCloseConnection = true; } public interface IFileversion { string? Path { get; } IEnumerable Sizes { get; } } public interface IFileset { long Version { get; } int IsFullBackup { get; set; } DateTime Time { get; } long FileCount { get; } long FileSizes { get; } } public interface IFileSets : IDisposable { IEnumerable Sets { get; } IEnumerable QuickSets { get; } IEnumerable SelectFiles(IFilter filter); IEnumerable GetLargestPrefix(IFilter filter); IEnumerable SelectFolderContents(IFilter filter); void TakeFirst(); } private class FileSets : IFileSets { private readonly IDbConnection m_connection; private string m_tablename; private readonly KeyValuePair[] m_filesets; public FileSets(LocalListDatabase owner, DateTime time, long[] versions) { m_connection = owner.m_connection; m_filesets = owner.FilesetTimes.ToArray(); m_tablename = "Filesets-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); var tmp = owner.GetFilelistWhereClause(time, versions, m_filesets); var query = tmp.Item1; var args = tmp.Item2; using (var cmd = m_connection.CreateCommand()) { cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tablename}"" AS SELECT DISTINCT ""ID"" AS ""FilesetID"", ""IsFullBackup"" AS ""IsFullBackup"" , ""Timestamp"" AS ""Timestamp"" FROM ""Fileset"" {query}"), args); cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tablename}_FilesetIDTimestampIndex"" ON ""{m_tablename}"" (""FilesetID"", ""Timestamp"" DESC)")); } } private class Fileset : IFileset { public long Version { get; private set; } public int IsFullBackup { get; set; } public DateTime Time { get; private set; } public long FileCount { get; private set; } public long FileSizes { get; private set; } public Fileset(long version, int isFullBackup, DateTime time, long filecount, long filesizes) { Version = version; IsFullBackup = isFullBackup; Time = time; FileCount = filecount; FileSizes = filesizes; } } private class Fileversion : IFileversion { private readonly IDataReader m_reader; public string? Path { get; private set; } public bool More { get; private set; } public Fileversion(IDataReader reader) { m_reader = reader; Path = reader.ConvertValueToString(0); More = true; } public IEnumerable Sizes { get { while (More && Path == m_reader.ConvertValueToString(0)) { yield return m_reader.ConvertValueToInt64(1, -1); More = m_reader.Read(); } } } } private class FileversionFixed : IFileversion { public string? Path { get; internal set; } public IEnumerable Sizes { get { return new long[0]; } } } public IEnumerable GetLargestPrefix(IFilter filter) { return GetLargestPrefix(filter, null); } private IEnumerable GetLargestPrefix(IFilter filter, string? prefixrule) { using (var tmpnames = new FilteredFilenameTable(m_connection, filter, null)) using (var cmd = m_connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets cmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) ")); //If we have a prefix rule, apply it if (!string.IsNullOrWhiteSpace(prefixrule)) cmd.SetCommandAndParameters(FormatInvariant($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE SUBSTR(""Path"", 1, {prefixrule.Length}) != @Rule")) .SetParameterValue("@Rule", prefixrule) .ExecuteNonQuery(); // Then we recursively find the largest prefix var v0 = cmd.ExecuteScalar(FormatInvariant($@"SELECT ""Path"" FROM ""{tmpnames.Tablename}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1")); var maxpath = ""; if (v0 != null) maxpath = v0.ToString() ?? ""; var dirsep = Util.GuessDirSeparator(maxpath); var filecount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""{tmpnames.Tablename}"""), 0); var foundfiles = -1L; //TODO: Handle FS case-sensitive? cmd.SetCommandAndParameters(FormatInvariant($@"SELECT COUNT(*) FROM ""{tmpnames.Tablename}"" WHERE SUBSTR(""Path"", 1, @PrefixLength) = @Prefix")); while (filecount != foundfiles && maxpath.Length > 0) { var mp = Util.AppendDirSeparator(maxpath, dirsep); foundfiles = cmd.SetParameterValue("@PrefixLength", mp.Length) .SetParameterValue("@Prefix", mp) .ExecuteScalarInt64(0); if (filecount != foundfiles) { var oldlen = maxpath.Length; var lix = maxpath.LastIndexOf(dirsep, maxpath.Length - 2, StringComparison.Ordinal); maxpath = maxpath.Substring(0, lix + 1); if (string.IsNullOrWhiteSpace(maxpath) || maxpath.Length == oldlen || maxpath == "\\\\") maxpath = ""; } } // Special handling for Windows and multi-drive/UNC backups as they do not have a single common root if (string.IsNullOrWhiteSpace(maxpath) && string.IsNullOrWhiteSpace(prefixrule)) { var paths = cmd.ExecuteReaderEnumerable(FormatInvariant($@"SELECT Path FROM ""{tmpnames.Tablename}""")).Select(x => x.ConvertValueToString(0) ?? "").ToArray(); var roots = paths.Select(x => x.Substring(0, 1)).Distinct().Where(x => x != "\\").ToArray(); //unc path like \\server.domain\ var regexUNCPrefix = new System.Text.RegularExpressions.Regex(@"^\\\\.*?\\"); var rootsUNC = paths.Select(x => regexUNCPrefix.Match(x)).Where(x => x.Success).Select(x => x.Value).Distinct().ToArray(); return roots.Concat(rootsUNC).Select(x => GetLargestPrefix(filter, x).First()).Distinct().ToArray(); } return [new FileversionFixed { Path = maxpath == "" ? "" : Util.AppendDirSeparator(maxpath, dirsep) }]; } } private IEnumerable SelectFolderEntries(IDbCommand cmd, string prefix, string table) { if (!string.IsNullOrEmpty(prefix)) prefix = Util.AppendDirSeparator(prefix, Util.GuessDirSeparator(prefix)); var ppl = prefix.Length; using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT DISTINCT ""Path"" FROM ""{table}"" "))) while (rd.Read()) { var s = rd.ConvertValueToString(0) ?? ""; if (!s.StartsWith(prefix, StringComparison.Ordinal)) continue; var dirsep = Util.GuessDirSeparator(s); s = s.Substring(ppl); var ix = s.IndexOf(dirsep, StringComparison.Ordinal); if (ix > 0 && ix != s.Length - 1) s = s.Substring(0, ix + 1); yield return prefix + s; } } public IEnumerable SelectFolderContents(IFilter filter) { var tbname = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); try { string pathprefix; if (filter == null || filter.Empty) pathprefix = ""; else if (filter as FilterExpression == null || ((FilterExpression)filter).Type != FilterType.Simple || ((FilterExpression)filter).GetSimpleList().Length != 1) throw new ArgumentException("Filter for list-folder-contents must be a path prefix with no wildcards", nameof(filter)); else pathprefix = ((FilterExpression)filter).GetSimpleList().First(); var dirsep = Util.GuessDirSeparator(pathprefix); if (pathprefix.Length > 0 || dirsep == "/") pathprefix = Util.AppendDirSeparator(pathprefix, dirsep); using (var tmpnames = new FilteredFilenameTable(m_connection, new FilterExpression(new string[] { pathprefix + "*" }, true), null)) using (var cmd = m_connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets cmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) ")); // If we had instr support this would work: /*var distinctPaths = @"SELECT DISTINCT :1 || " + @"CASE(INSTR(SUBSTR(""Path"", :2), '/')) " + @"WHEN 0 THEN SUBSTR(""Path"", :2) " + @"ELSE SUBSTR(""Path"", :2, INSTR(SUBSTR(path, :2), '/')) " + @"END AS ""Path"", ""FilesetID"" " + @" FROM (" + cartesianPathFileset + @")";*/ // Instead we manually iterate the paths cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tbname}"" (""Path"" TEXT NOT NULL)")); using (var c2 = m_connection.CreateCommand()) { c2.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{tbname}"" (""Path"") VALUES (@Path)")); foreach (var n in SelectFolderEntries(cmd, pathprefix, tmpnames.Tablename).Distinct()) c2.SetParameterValue("@Path", n) .ExecuteNonQuery(); c2.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{tbname}_PathIndex"" ON ""{tbname}"" (""Path"")")); } //Then we select the matching results var filesets = FormatInvariant($@"SELECT ""FilesetID"", ""Timestamp"" FROM ""{m_tablename}"" ORDER BY ""Timestamp"" DESC"); var cartesianPathFileset = FormatInvariant($@"SELECT ""A"".""Path"", ""B"".""FilesetID"" FROM ""{tbname}"" A, ({filesets}) B ORDER BY ""A"".""Path"" ASC, ""B"".""Timestamp"" DESC"); var filesWithSizes = FormatInvariant($@"SELECT ""Length"", ""FilesetEntry"".""FilesetID"", ""File"".""Path"" FROM ""Blockset"", ""FilesetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND FilesetEntry.""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") "); var query = @"SELECT ""C"".""Path"", ""D"".""Length"", ""C"".""FilesetID"" FROM (" + cartesianPathFileset + @") C LEFT OUTER JOIN (" + filesWithSizes + @") D ON ""C"".""FilesetID"" = ""D"".""FilesetID"" AND ""C"".""Path"" = ""D"".""Path"""; using (var rd = cmd.ExecuteReader(query)) if (rd.Read()) { bool more; do { var f = new Fileversion(rd); if (!(string.IsNullOrWhiteSpace(f.Path) || f.Path == pathprefix)) { yield return f; more = f.More; } else { more = rd.Read(); } } while (more); } } } finally { try { using (var c = m_connection.CreateCommand()) c.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tbname}""")); } catch { } } } public IEnumerable SelectFiles(IFilter filter) { using (var tmpnames = new FilteredFilenameTable(m_connection, filter, null)) using (var cmd = m_connection.CreateCommand()) { //First we trim the filelist to exclude filenames not found in any of the filesets cmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) ")); //Then we select the matching results var filesets = FormatInvariant($@"SELECT ""FilesetID"", ""Timestamp"" FROM ""{m_tablename}"" ORDER BY ""Timestamp"" DESC"); var cartesianPathFileset = FormatInvariant($@"SELECT ""A"".""Path"", ""B"".""FilesetID"" FROM ""{tmpnames.Tablename}"" A, ({filesets}) B ORDER BY ""A"".""Path"" ASC, ""B"".""Timestamp"" DESC"); var filesWithSizes = FormatInvariant($@"SELECT ""Length"", ""FilesetEntry"".""FilesetID"", ""File"".""Path"" FROM ""Blockset"", ""FilesetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND FilesetEntry.""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") "); var query = @"SELECT ""C"".""Path"", ""D"".""Length"", ""C"".""FilesetID"" FROM (" + cartesianPathFileset + @") C LEFT OUTER JOIN (" + filesWithSizes + @") D ON ""C"".""FilesetID"" = ""D"".""FilesetID"" AND ""C"".""Path"" = ""D"".""Path"""; using (var rd = cmd.ExecuteReader(query)) if (rd.Read()) { bool more; do { var f = new Fileversion(rd); yield return f; more = f.More; } while (more); } } } public void TakeFirst() { using (var cmd = m_connection.CreateCommand()) cmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{m_tablename}"" WHERE ""FilesetID"" NOT IN (SELECT ""FilesetID"" FROM ""{m_tablename}"" ORDER BY ""Timestamp"" DESC LIMIT 1 )")); } public IEnumerable QuickSets { get { var dict = new Dictionary(); for (var i = 0; i < m_filesets.Length; i++) dict[m_filesets[i].Key] = i; using (var cmd = m_connection.CreateCommand()) using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""ID"", ""IsFullBackup"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC ")) while (rd.Read()) { var id = rd.ConvertValueToInt64(0); var backupType = rd.GetInt32(1); var e = dict[id]; yield return new Fileset(e, backupType, m_filesets[e].Value, -1L, -1L); } } } public IEnumerable Sets { get { var dict = new Dictionary(); for (var i = 0; i < m_filesets.Length; i++) dict[m_filesets[i].Key] = i; var summation = FormatInvariant( $@"SELECT ""A"".""FilesetID"" AS ""FilesetID"", COUNT(*) AS ""FileCount"", SUM(""C"".""Length"") AS ""FileSizes"" FROM ""FilesetEntry"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" AND ""A"".""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") GROUP BY ""A"".""FilesetID"" "); using (var cmd = m_connection.CreateCommand()) using (var rd = cmd.ExecuteReader(FormatInvariant( $@"SELECT DISTINCT ""A"".""FilesetID"", ""A"".""IsFullBackup"", ""B"".""FileCount"", ""B"".""FileSizes"" FROM ""{m_tablename}"" A LEFT OUTER JOIN ( {summation} ) B ON ""A"".""FilesetID"" = ""B"".""FilesetID"" ORDER BY ""A"".""Timestamp"" DESC ")) ) while (rd.Read()) { var id = rd.ConvertValueToInt64(0); var isFullBackup = rd.GetInt32(1); var e = dict[id]; var filecount = rd.ConvertValueToInt64(2, -1L); var filesizes = rd.ConvertValueToInt64(3, -1L); yield return new Fileset(e, isFullBackup, m_filesets[e].Value, filecount, filesizes); } } } public void Dispose() { if (m_tablename != null) { try { using (var cmd = m_connection.CreateCommand()) cmd.ExecuteNonQuery(FormatInvariant(@$"DROP TABLE IF EXISTS ""{m_tablename}"" ")); } catch { } finally { m_tablename = null!; } } } } public IFileSets SelectFileSets(DateTime time, long[] versions) { return new FileSets(this, time, versions); } /// /// Represents a fileset entry in the list fileset results. /// /// The fileset version /// Flag indicating if the backup is a full backup or synthetic backup /// The timestamp of the fileset /// The number of files in the fileset /// >The total size of files in the fileset private sealed record FilesetEntry(long Version, bool? IsFullBackup, DateTime Time, long? FileCount, long? FileSizes) : IListFilesetResultFileset; /// /// Lists all filesets with summary data /// /// A collection of fileset entries public IEnumerable ListFilesetsExtended() { const string query = @" SELECT f.""ID"" AS ""FilesetID"", f.""Timestamp"", f.""IsFullBackup"", COUNT(fe.""FileID"") AS ""NumberOfFiles"", COALESCE(SUM(b.""Length""), 0) AS ""TotalFileSize"" FROM ""Fileset"" f LEFT JOIN ""FilesetEntry"" fe ON f.""ID"" = fe.""FilesetID"" LEFT JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" LEFT JOIN ""Blockset"" b ON fl.""BlocksetID"" = b.""ID"" GROUP BY f.""ID"" ORDER BY f.""Timestamp"" DESC; "; using (var cmd = m_connection.CreateCommand(query)) foreach (var rd in cmd.ExecuteReaderEnumerable()) { var id = rd.ConvertValueToInt64(0); var time = ParseFromEpochSeconds(rd.ConvertValueToInt64(1)); var isFullBackup = rd.GetInt32(2) == BackupType.FULL_BACKUP; var filecount = rd.ConvertValueToInt64(3, -1L); var filesizes = rd.ConvertValueToInt64(4, -1L); yield return new FilesetEntry(id, isFullBackup, time, filecount, filesizes); } } /// /// Represents a folder entry in the list folder results. /// /// The path of the folder entry. /// The size of the folder entry. /// Indicates if the entry is a directory. /// Indicates if the entry is a symbolic link. /// The last modified date of the folder entry. private sealed record FolderEntry(string Path, long Size, bool IsDirectory, bool IsSymlink, DateTime LastModified) : IListFolderEntry; /// /// Lists the folder entries for a given fileset and prefix IDs. /// /// The prefix IDs to filter the folder entries. /// The fileset ID to filter the folder entries. /// The offset for pagination. /// >The limit for pagination. /// A paginated result set of folder entries. public IPaginatedResults ListFolder(IEnumerable prefixIds, long filesetid, long offset, long limit) { if (offset != 0 && limit <= 0) throw new ArgumentException("Cannot use offset without limit specified.", nameof(offset)); if (limit <= 0) limit = long.MaxValue; using var cmd = m_connection.CreateCommand(); if (prefixIds.Count() == 0) return new PaginatedResults(0, (int)limit, 0, 0, Enumerable.Empty()); // Then query the matching files cmd.SetCommandAndParameters($@" SELECT pp.""Prefix"" || fl.""Path"" AS ""FullPath"", b.""Length"", (CASE WHEN fl.""BlocksetID"" = @FolderBlocksetId THEN 1 ELSE 0 END) AS ""IsDirectory"", (CASE WHEN fl.""BlocksetID"" = @SymlinkBlocksetId THEN 1 ELSE 0 END) AS ""IsSymlink"", fe.""Lastmodified"" FROM ""FilesetEntry"" fe INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" INNER JOIN ""PathPrefix"" pp ON fl.""PrefixID"" = pp.""ID"" LEFT JOIN ""Blockset"" b ON fl.""BlocksetID"" = b.""ID"" WHERE fe.""FilesetID"" = @filesetid AND fl.""PrefixID"" IN (@PrefixIds) ORDER BY pp.""Prefix"" ASC, fl.""Path"" ASC LIMIT @limit OFFSET @offset ") .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) .ExpandInClauseParameter("@PrefixIds", prefixIds) .SetParameterValue("@filesetid", filesetid) .SetParameterValue("@limit", limit) .SetParameterValue("@offset", offset); var results = new List(); foreach (var rd in cmd.ExecuteReaderEnumerable()) { var path = rd.ConvertValueToString(0) ?? string.Empty; var size = rd.ConvertValueToInt64(-1); var isDir = rd.GetInt32(2) != 0; var isSymlink = rd.GetInt32(3) != 0; var lastModified = new DateTime(rd.ConvertValueToInt64(4, 0), DateTimeKind.Utc); results.Add(new FolderEntry(path, size, isDir, isSymlink, lastModified)); } long totalCount; if (offset != 0) { // Only calculate the total for the first query totalCount = -1; } else if (results.Count <= limit) { // If we have all results, we already know the total totalCount = results.Count; } else { // Get the total count of files for the given fileset and prefix IDs totalCount = cmd.SetCommandAndParameters($@" SELECT COUNT(*) FROM FilesetEntry fe INNER JOIN FileLookup fl ON fe.FileID = fl.ID WHERE fe.FilesetID = @filesetid AND fl.PrefixID IN (@PrefixIds) ") .ExpandInClauseParameter("@PrefixIds", prefixIds) .SetParameterValue("@filesetid", filesetid) .ExecuteScalarInt64(0); } // Return paginated results var intLimit = limit > int.MaxValue ? int.MaxValue : (int)limit; return new PaginatedResults((int)(offset / limit), (int)intLimit, (int)((totalCount + limit - 1) / limit), totalCount, results); } /// /// Lists the folder entries for a given fileset and prefix IDs. /// /// The prefix IDs to return entries for. /// The offset for pagination. /// >The limit for pagination. /// A paginated result set of folder entries. public IPaginatedResults ListFilesetEntries(IEnumerable prefixIds, long filesetid, long offset, long limit) { if (offset != 0 && limit <= 0) throw new ArgumentException("Cannot use offset without limit specified.", nameof(offset)); if (limit <= 0) limit = long.MaxValue; using var cmd = m_connection.CreateCommand(); if (!prefixIds.Any()) return new PaginatedResults(0, (int)limit, 0, 0, Enumerable.Empty()); // Fetch entries cmd.SetCommandAndParameters(@" SELECT pp.""Prefix"" || fl.""Path"" AS ""FullPath"", b.""Length"", CASE WHEN fl.""BlocksetID"" = @FolderBlocksetId THEN 1 ELSE 0 END AS ""IsDirectory"", CASE WHEN fl.""BlocksetID"" = @SymlinkBlocksetId THEN 1 ELSE 0 END AS ""IsSymlink"", fe.""Lastmodified"" FROM ""FilesetEntry"" fe INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" INNER JOIN ""PathPrefix"" pp ON fl.""PrefixID"" = pp.""ID"" LEFT JOIN ""Blockset"" b ON fl.""BlocksetID"" = b.""ID"" WHERE fe.""FilesetId"" = @FilesetId AND fl.""PrefixId"" IN (@PrefixIds) ORDER BY pp.""Prefix"" ASC, fl.""Path"" ASC LIMIT @limit OFFSET @offset ") .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) .SetParameterValue("@FilesetId", filesetid) .ExpandInClauseParameter("@PrefixIds", prefixIds) .SetParameterValue("@limit", limit) .SetParameterValue("@offset", offset); var results = new List(); foreach (var rd in cmd.ExecuteReaderEnumerable()) { var path = rd.ConvertValueToString(0) ?? string.Empty; var size = rd.ConvertValueToInt64(-1); var isDir = rd.GetInt32(2) != 0; var isSymlink = rd.GetInt32(3) != 0; var lastModified = new DateTime(rd.ConvertValueToInt64(4, 0), DateTimeKind.Utc); results.Add(new FolderEntry(path, size, isDir, isSymlink, lastModified)); } long totalCount; if (offset != 0) { // Only calculate the total for the first query totalCount = -1; } else if (results.Count <= limit) { // If we have all results, we already know the total totalCount = results.Count; } else { totalCount = cmd.SetCommandAndParameters(@" SELECT COUNT(*) FROM ""FilesetEntry"" fe INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" WHERE fe.""FilesetId"" = @FilesetId AND fl.""PrefixId"" IN (@PrefixIds) ") .SetParameterValue("@FilesetId", filesetid) .ExpandInClauseParameter("@PrefixIds", prefixIds) .ExecuteScalarInt64(0); } var intLimit = limit > int.MaxValue ? int.MaxValue : (int)limit; return new PaginatedResults((int)(offset / limit), intLimit, (int)((totalCount + limit - 1) / limit), totalCount, results); } /// /// Gets the prefix IDs for a given set of folder-prefixes. /// /// The prefixes to get the IDs for. /// >A list of prefix IDs. public IEnumerable GetPrefixIds(IEnumerable prefixes) { using var cmd = m_connection.CreateCommand(); // Map folders to prefix IDs cmd.SetCommandAndParameters($@"SELECT ""ID"" FROM ""PathPrefix"" WHERE ""Prefix"" IN (@Prefixes)") .ExpandInClauseParameter("@Prefixes", prefixes); return cmd.ExecuteReaderEnumerable() .Select(x => x.ConvertValueToInt64(0)) .ToList(); } /// /// Gathers root prefixes from the fileset entries. /// /// The filesetId to get the prefixes for /// A list of root prefixes public IEnumerable GetRootPrefixes(long filesetid) { using var cmd = m_connection.CreateCommand(); cmd.SetCommandAndParameters(@" WITH Prefixes AS ( SELECT DISTINCT fl.""PrefixID"", pp.""Prefix"" FROM ""FilesetEntry"" fe INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" INNER JOIN ""PathPrefix"" pp ON fl.""PrefixID"" = pp.""ID"" WHERE fe.""FilesetID"" = @filesetid ) SELECT p1.""PrefixID"" FROM Prefixes p1 WHERE NOT EXISTS ( SELECT 1 FROM Prefixes p2 WHERE p2.""PrefixID"" != p1.""PrefixID"" AND p1.""Prefix"" LIKE p2.""Prefix"" || '%' AND LENGTH(p1.""Prefix"") > LENGTH(p2.""Prefix"") ) ORDER BY p1.""Prefix"" ASC ") .SetParameterValue("@filesetid", filesetid); return cmd.ExecuteReaderEnumerable() .Select(x => x.ConvertValueToInt64(0, -1)) .Where(x => x != -1) .ToList(); } /// /// Gathers all prefixes and their parent prefix IDs from the fileset entries. /// /// The filesetId to get the prefixes for /// A list of (PrefixID, ParentPrefixID) pairs public IEnumerable<(long PrefixId, long ParentPrefixId)> GetRootPrefixesWithParents(long filesetid) { using var cmd = m_connection.CreateCommand(); var dirSeparator = Util.GuessDirSeparator(cmd.SetCommandAndParameters(@"SELECT ""Prefix"" FROM ""PathPrefix"" LIMIT 1") .ExecuteScalar()?.ToString()); cmd.SetCommandAndParameters(@" WITH Prefixes AS ( SELECT DISTINCT fl.""PrefixID"", pp.""Prefix"" FROM ""FilesetEntry"" fe INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" INNER JOIN ""PathPrefix"" pp ON fl.""PrefixID"" = pp.""ID"" WHERE fe.""FilesetID"" = @filesetid ), PrefixParents AS ( SELECT child.""PrefixID"" AS ChildPrefixID, parent.""PrefixID"" AS ParentPrefixID, LENGTH(parent.""Prefix"") AS ParentLength FROM Prefixes child JOIN Prefixes parent ON LENGTH(child.""Prefix"") > LENGTH(parent.""Prefix"") AND SUBSTR(child.""Prefix"", 1, LENGTH(parent.""Prefix"")) = parent.""Prefix"" AND (SUBSTR(child.""Prefix"", LENGTH(parent.""Prefix"") + 1, 1) = @dirSeparator) ) SELECT child.""PrefixID"", ( SELECT ParentPrefixID FROM PrefixParents pp WHERE pp.ChildPrefixID = child.""PrefixID"" ORDER BY pp.ParentLength DESC LIMIT 1 ) AS ParentPrefixID FROM Prefixes child ORDER BY child.""Prefix"" ASC ") .SetParameterValue("@dirSeparator", dirSeparator) .SetParameterValue("@filesetid", filesetid); return cmd.ExecuteReaderEnumerable() .Select(x => ( PrefixId: x.ConvertValueToInt64(0, -1), ParentPrefixId: x.ConvertValueToInt64(1, -1) )); } /// /// Lists all versions of specific file paths, optionally filtered by fileset IDs. /// /// Paths to match exactly. /// Optional fileset IDs to restrict the search to. /// Pagination offset. /// Pagination limit. /// All matching file versions ordered by path and then version time. public IPaginatedResults ListFileVersions(IEnumerable paths, long[]? filesetIds, long offset, long limit) { if (paths == null || !paths.Any()) return new PaginatedResults(0, (int)(limit > 0 ? limit : long.MaxValue), 0, 0, Enumerable.Empty()); if (offset != 0 && limit <= 0) throw new ArgumentException("Cannot use offset without limit specified.", nameof(offset)); if (limit <= 0) limit = long.MaxValue; using var cmd = m_connection.CreateCommand(); var countWhere = "WHERE fl.\"Path\" IN (@Paths)"; if (filesetIds != null && filesetIds.Length > 0) countWhere += " AND fe.\"FilesetID\" IN (@FilesetIds)"; cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC"); var versionMap = cmd.ExecuteReaderEnumerable() .Select((x, i) => (Version: i, FilesetId: x.ConvertValueToInt64(0))) .ToDictionary(x => x.FilesetId, x => x.Version); // Then fetch the actual data var fetchSql = @" SELECT fe.""FilesetID"", f.""Timestamp"", fl.""Path"", COALESCE(b.""Length"", 0) AS ""Size"", CASE WHEN fl.""BlocksetID"" = @FolderBlocksetId THEN 1 ELSE 0 END AS ""IsDirectory"", CASE WHEN fl.""BlocksetID"" = @SymlinkBlocksetId THEN 1 ELSE 0 END AS ""IsSymlink"", fe.""Lastmodified"" FROM ""FilesetEntry"" fe INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" INNER JOIN ""Fileset"" f ON fe.""FilesetID"" = f.""ID"" LEFT JOIN ""Blockset"" b ON fl.""BlocksetID"" = b.""ID"" " + countWhere + @" ORDER BY fl.""Path"" ASC, f.""Timestamp"" ASC LIMIT @limit OFFSET @offset"; cmd.SetCommandAndParameters(fetchSql) .ExpandInClauseParameter("@Paths", paths) .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) .SetParameterValue("@limit", limit) .SetParameterValue("@offset", offset); if (filesetIds != null && filesetIds.Length > 0) cmd.ExpandInClauseParameter("@FilesetIds", filesetIds); var results = new List(); foreach (var rd in cmd.ExecuteReaderEnumerable()) { var version = versionMap[rd.ConvertValueToInt64(0, -1)]; var time = new DateTime(rd.ConvertValueToInt64(1, 0), DateTimeKind.Utc); var path = rd.ConvertValueToString(2) ?? string.Empty; var size = rd.ConvertValueToInt64(3, 0); var isDirectory = rd.GetInt32(4) != 0; var isSymlink = rd.GetInt32(5) != 0; var lastModified = new DateTime(rd.ConvertValueToInt64(6, 0), DateTimeKind.Utc); results.Add(new ListFileVersion(version, time, path, size, isDirectory, isSymlink, lastModified)); } long totalCount; if (offset != 0) { // Only calculate the total for the first query totalCount = -1; } else if (results.Count <= limit) { // If we have all results, we already know the total totalCount = results.Count; } else { // Get total count var countSql = @" SELECT COUNT(*) FROM ""FilesetEntry"" fe INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" "; cmd.SetCommandAndParameters(countSql + "\n" + countWhere) .ExpandInClauseParameter("@Paths", paths); if (filesetIds != null && filesetIds.Length > 0) cmd.ExpandInClauseParameter("@FilesetIds", filesetIds); totalCount = cmd.ExecuteScalarInt64(0); } var intLimit = limit > int.MaxValue ? int.MaxValue : (int)limit; var totalPages = (int)((totalCount + limit - 1) / limit); return new PaginatedResults( (int)(offset / limit), intLimit, totalPages, totalCount, results ); } /// /// Searches for file versions matching a filter in the specified path prefixes. /// /// The path prefixes to search in, or search in all if empty. /// The filter to match against file paths. /// Optional fileset IDs to restrict the search to. /// The offset for pagination. /// The limit for pagination. /// All matching file versions ordered by path and then version time. public IPaginatedResults SearchEntries(IEnumerable? pathprefixes, IFilter filter, long[]? filesetIds, long offset, long limit) { if (offset != 0 && limit <= 0) throw new ArgumentException("Cannot use offset without limit specified.", nameof(offset)); if (limit <= 0) limit = long.MaxValue; using var cmd = m_connection.CreateCommand(); // Analyze the filter to determine the default behavior FilterExpression.AnalyzeFilters(filter, out var includes, out var excludes); var defaultExclude = includes && !excludes; // true = exclude unmatched, false = include unmatched var defaultBehavior = defaultExclude ? 1 : 0; var caseSensitive = false; //TODO: Should we expose this? Or use: Library.Utility.Utility.IsFSCaseSensitive? var filterProps = new Dictionary(); var caseWhenParts = new List(); var filterExpressions = new List(); if (filter is FilterExpression fe) { filterExpressions.Add(fe); } else if (filter is JoinedFilterExpression jfe) { var work = new Queue([jfe.First, jfe.Second]); while (work.Count > 0) { var current = work.Dequeue(); if (current is FilterExpression fe1) { filterExpressions.Add(fe1); } else if (current is JoinedFilterExpression jfe1) { work.Enqueue(jfe1.First); work.Enqueue(jfe1.Second); } else { throw new UserInformationException($"Filter type {current.GetType()} is not supported.", "FilterTypeNotSupported"); } } } foreach (var filterExpression in filterExpressions) { if (filterExpression.Type == FilterType.Empty) { // No additional CASE needed } else if (filterExpression.Type == FilterType.Simple) { var parts = filterExpression.GetSimpleList(); foreach (var part in parts) { var propName = $"@Part{filterProps.Count}"; filterProps[propName] = part; if (caseSensitive) caseWhenParts.Add($@"WHEN instr(LOWER(pp.""Prefix"" || fl.""Path""), LOWER({propName})) > 0 THEN {(filterExpression.Result ? 0 : 1)}"); else caseWhenParts.Add($@"WHEN instr(pp.""Prefix"" || fl.""Path"", {propName}) > 0 THEN {(filterExpression.Result ? 0 : 1)}"); } } else if (filterExpression.Type == FilterType.Wildcard) { var parts = filterExpression.GetSimpleList(); foreach (var part in parts) { var propName = $"@Part{filterProps.Count}"; filterProps[propName] = part; if (caseSensitive) caseWhenParts.Add($@"WHEN LOWER(pp.""Prefix"" || fl.""Path"") GLOB LOWER({propName}) THEN {(filterExpression.Result ? 0 : 1)}"); else caseWhenParts.Add($@"WHEN pp.""Prefix"" || fl.""Path"" GLOB {propName} THEN {(filterExpression.Result ? 0 : 1)}"); } } else { throw new UserInformationException($"Filter type {filterExpression.Type} is not supported.", "FilterTypeNotSupported"); } } // Build WHERE clauses var whereClauses = new List(); if (pathprefixes != null && pathprefixes.Any()) whereClauses.Add(@"pp.""Prefix"" IN (@PathPrefixes)"); if (filesetIds != null && filesetIds.Length > 0) whereClauses.Add(@"fe.""FilesetID"" IN (@FilesetIds)"); if (caseWhenParts.Any()) { whereClauses.Add($@"COALESCE( CASE {string.Join("\n ", caseWhenParts)} ELSE NULL END, @DefaultBehavior ) = 0"); } else { // No CASE parts, fallback to default behavior directly if (defaultBehavior == 1) { // Default is EXCLUDE -> exclude everything whereClauses.Add("0 = 1"); } } var whereClause = string.Join("\n AND ", whereClauses); cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC"); var versionMap = cmd.ExecuteReaderEnumerable() .Select((x, i) => (Version: i, FilesetId: x.ConvertValueToInt64(0))) .ToDictionary(x => x.FilesetId, x => x.Version); // Fetch results var fetchSql = $@" SELECT fe.""FilesetID"", f.""Timestamp"", pp.""Prefix"" || fl.""Path"" AS ""FullPath"", COALESCE(b.""Length"", 0) AS ""Size"", CASE WHEN fl.""BlocksetID"" = @FolderBlocksetId THEN 1 ELSE 0 END AS ""IsDirectory"", CASE WHEN fl.""BlocksetID"" = @SymlinkBlocksetId THEN 1 ELSE 0 END AS ""IsSymlink"", fe.""Lastmodified"" FROM ""FilesetEntry"" fe INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" INNER JOIN ""Fileset"" f ON fe.""FilesetID"" = f.""ID"" LEFT JOIN ""Blockset"" b ON fl.""BlocksetID"" = b.""ID"" INNER JOIN ""PathPrefix"" pp ON fl.""PrefixID"" = pp.""ID"" WHERE {whereClause} ORDER BY pp.""Prefix"" ASC, fl.""Path"" ASC, f.""Timestamp"" ASC LIMIT @limit OFFSET @offset "; cmd.SetCommandAndParameters(fetchSql) .SetParameterValues(filterProps) .SetParameterValue("@DefaultBehavior", defaultBehavior) .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) .SetParameterValue("@limit", limit) .SetParameterValue("@offset", offset); if (pathprefixes != null && pathprefixes.Any()) cmd.ExpandInClauseParameter("@PathPrefixes", pathprefixes); if (filesetIds != null && filesetIds.Length > 0) cmd.ExpandInClauseParameter("@FilesetIds", filesetIds); var results = new List(); foreach (var rd in cmd.ExecuteReaderEnumerable()) { var version = versionMap[rd.ConvertValueToInt64(0, -1)]; var time = new DateTime(rd.ConvertValueToInt64(1, 0), DateTimeKind.Utc); var path = rd.ConvertValueToString(2) ?? string.Empty; var size = rd.ConvertValueToInt64(3, 0); var isDirectory = rd.GetInt32(4) != 0; var isSymlink = rd.GetInt32(5) != 0; var lastModified = new DateTime(rd.ConvertValueToInt64(6, 0), DateTimeKind.Utc); // We cannot know exactly where the match occurred here // (unless we do another pass), so just use Range(0,0) for now results.Add(new SearchFileVersion(version, time, path, size, isDirectory, isSymlink, lastModified, new Range(0, 0))); } long totalCount; if (offset != 0) { // Only calculate the total for the first query totalCount = -1; } else if (results.Count <= limit) { // If we have all results, we already know the total totalCount = results.Count; } else { // Calculate the total var countSql = $@" SELECT COUNT(*) FROM ""FilesetEntry"" fe INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" INNER JOIN ""PathPrefix"" pp ON fl.""PrefixID"" = pp.""ID"" WHERE {whereClause} "; cmd.SetCommandAndParameters(countSql) .SetParameterValues(filterProps) .SetParameterValue("@DefaultBehavior", defaultBehavior); if (pathprefixes != null && pathprefixes.Any()) cmd.ExpandInClauseParameter("@PathPrefixes", pathprefixes); if (filesetIds != null && filesetIds.Length > 0) cmd.ExpandInClauseParameter("@FilesetIds", filesetIds); totalCount = cmd.ExecuteScalarInt64(0); } var intLimit = limit > int.MaxValue ? int.MaxValue : (int)limit; var totalPages = (int)((totalCount + limit - 1) / limit); return new PaginatedResults( (int)(offset / limit), intLimit, totalPages, totalCount, results ); } } }