From 9e3aa72d00c19c006355b48663317c8854aa6ea0 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 3 Apr 2025 11:31:36 +0200 Subject: [PATCH] Added LocalBackupDatabase --- .../Library/Main/Database/ExtensionMethods.cs | 20 +- .../Main/Database/LocalBackupDatabase.cs | 493 ++++++++++-------- .../Main/Operation/Backup/BackupDatabase.cs | 4 +- .../Operation/Backup/FilePreFilterProcess.cs | 4 +- 4 files changed, 294 insertions(+), 227 deletions(-) diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index 9db25a8ac..87a555ca9 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -194,6 +194,20 @@ public static class ExtensionMethods return self; } + /// + /// Sets the transaction for the command. + /// + /// The type of the command + /// The command to set the transaction for + /// The transaction to set for the command + /// >The command with the transaction set + public static T SetTransaction(this T self, IDbTransaction? transaction) + where T : IDbCommand + { + self.Transaction = transaction; + return self; + } + /// /// Gets the printable command text for the given command. /// The command string to execute /// The values to use as parameters. The parameters must already be added. /// The scalar value of the first row - public static object? ExecuteScalar(this IDbCommand self, bool writeLog, string cmd, params object[] values) + public static object? ExecuteScalar(this IDbCommand self, bool writeLog, string? cmd, params object[] values) { if (cmd != null) self.SetCommandAndParameters(cmd); @@ -474,7 +488,7 @@ public static class ExtensionMethods /// The command string to execute /// The values to use as parameters. The parameters must already be added. /// A instance - public static IDataReader ExecuteReader(this IDbCommand self, bool writeLog, string cmd, params object[] values) + public static IDataReader ExecuteReader(this IDbCommand self, bool writeLog, string? cmd, params object[] values) { if (cmd != null) self.SetCommandAndParameters(cmd); @@ -494,7 +508,7 @@ public static class ExtensionMethods /// The command string to execute /// The values to use as parameters. The parameters must already be added. /// A instance - public static IDataReader ExecuteReader(this IDbCommand self, bool writeLog, string cmd, Dictionary values) + public static IDataReader ExecuteReader(this IDbCommand self, bool writeLog, string? cmd, Dictionary values) { if (cmd != null) self.SetCommandAndParameters(cmd); diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 4bc61e3ae..fda37cd9f 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -18,6 +18,8 @@ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +#nullable enable + using System; using System.Collections.Generic; using System.Data; @@ -43,7 +45,6 @@ namespace Duplicati.Library.Main.Database private readonly IDbCommand m_insertfileCommand; private readonly IDbCommand m_insertblocksetCommand; - private readonly IDbCommand m_insertblocksetentryFastCommand; private readonly IDbCommand m_insertblocksetentryCommand; private readonly IDbCommand m_insertblocklistHashesCommand; @@ -53,7 +54,6 @@ namespace Duplicati.Library.Main.Database private readonly IDbCommand m_selectfilelastmodifiedCommand; private readonly IDbCommand m_selectfilelastmodifiedWithSizeCommand; private readonly IDbCommand m_selectfileHashCommand; - private readonly IDbCommand m_selectblocklistHashesCommand; private readonly IDbCommand m_insertfileOperationCommand; private readonly IDbCommand m_selectfilemetadatahashandsizeCommand; @@ -76,41 +76,20 @@ namespace Duplicati.Library.Main.Database { m_logQueries = options.ProfileAllDatabaseQueries; - m_findblockCommand = m_connection.CreateCommand(); - m_insertblockCommand = m_connection.CreateCommand(); - m_insertfileCommand = m_connection.CreateCommand(); - m_insertblocksetCommand = m_connection.CreateCommand(); - m_insertmetadatasetCommand = m_connection.CreateCommand(); - m_findblocksetCommand = m_connection.CreateCommand(); - m_findmetadatasetCommand = m_connection.CreateCommand(); - m_findfilesetCommand = m_connection.CreateCommand(); - m_insertblocksetentryCommand = m_connection.CreateCommand(); - m_insertblocklistHashesCommand = m_connection.CreateCommand(); - m_selectblocklistHashesCommand = m_connection.CreateCommand(); - m_insertfileOperationCommand = m_connection.CreateCommand(); - m_findfileCommand = m_connection.CreateCommand(); - m_selectfilelastmodifiedCommand = m_connection.CreateCommand(); - m_selectfilelastmodifiedWithSizeCommand = m_connection.CreateCommand(); - m_selectfileHashCommand = m_connection.CreateCommand(); - m_insertblocksetentryFastCommand = m_connection.CreateCommand(); - m_selectfilemetadatahashandsizeCommand = m_connection.CreateCommand(); - m_getfirstfilesetwithblockinblockset = m_connection.CreateCommand(); - - m_findblockCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?"); - m_findblocksetCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Fullhash"" = ? AND ""Length"" = ?"); - m_findmetadatasetCommand = m_connection.CreateCommand(@"SELECT ""A"".""ID"" FROM ""Metadataset"" A, ""BlocksetEntry"" B, ""Block"" C WHERE ""A"".""BlocksetID"" = ""B"".""BlocksetID"" AND ""B"".""BlockID"" = ""C"".""ID"" AND ""C"".""Hash"" = ? AND ""C"".""Size"" = ?"); - m_findfilesetCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""FileLookup"" WHERE ""BlocksetID"" = ? AND ""MetadataID"" = ? AND ""Path"" = ? AND ""PrefixID"" = ?"); - m_insertblockCommand = m_connection.CreateCommand(@"INSERT INTO ""Block"" (""Hash"", ""VolumeID"", ""Size"") VALUES (?, ?, ?); SELECT last_insert_rowid();"); - m_insertfileOperationCommand = m_connection.CreateCommand(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (?, ?, ?)"); - m_insertfileCommand = m_connection.CreateCommand(@"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"",""BlocksetID"", ""MetadataID"") VALUES (?, ?, ? ,?); SELECT last_insert_rowid();"); - m_insertblocksetCommand = m_connection.CreateCommand(@"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (?, ?); SELECT last_insert_rowid();"); - m_insertblocksetentryFastCommand = m_connection.CreateCommand(@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") VALUES (?,?,?)"); - m_insertblocksetentryCommand = m_connection.CreateCommand(@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") SELECT ? AS A, ? AS B, ""ID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?"); - m_insertblocklistHashesCommand = m_connection.CreateCommand(@"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (?, ?, ?)"); - m_insertmetadatasetCommand = m_connection.CreateCommand(@"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (?); SELECT last_insert_rowid();"); - m_selectfilelastmodifiedCommand = m_connection.CreateCommand(@"SELECT ""A"".""ID"", ""B"".""LastModified"" FROM (SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = ? AND ""Path"" = ?) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = ?"); - m_selectfilelastmodifiedWithSizeCommand = m_connection.CreateCommand(@"SELECT ""C"".""ID"", ""C"".""LastModified"", ""D"".""Length"" FROM (SELECT ""A"".""ID"", ""B"".""LastModified"", ""A"".""BlocksetID"" FROM (SELECT ""ID"", ""BlocksetID"" FROM ""FileLookup"" WHERE ""PrefixID"" = ? AND ""Path"" = ?) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = ?) AS ""C"", ""Blockset"" AS ""D"" WHERE ""C"".""BlocksetID"" == ""D"".""ID"" "); - m_selectfilemetadatahashandsizeCommand = m_connection.CreateCommand(@"SELECT ""Blockset"".""Length"", ""Blockset"".""FullHash"" FROM ""Blockset"", ""Metadataset"", ""File"" WHERE ""File"".""ID"" = ? AND ""Blockset"".""ID"" = ""Metadataset"".""BlocksetID"" AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" "); + m_findblockCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @hash AND ""Size"" = @Size"); + m_findblocksetCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Fullhash"" = @Fullhash AND ""Length"" = @Length"); + m_findmetadatasetCommand = m_connection.CreateCommand(@"SELECT ""A"".""ID"" FROM ""Metadataset"" A, ""BlocksetEntry"" B, ""Block"" C WHERE ""A"".""BlocksetID"" = ""B"".""BlocksetID"" AND ""B"".""BlockID"" = ""C"".""ID"" AND ""C"".""Hash"" = @Hash AND ""C"".""Size"" = @Size"); + m_findfilesetCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""FileLookup"" WHERE ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId AND ""Path"" = @Path AND ""PrefixID"" = @PrefixId"); + m_insertblockCommand = m_connection.CreateCommand(@"INSERT INTO ""Block"" (""Hash"", ""VolumeID"", ""Size"") VALUES (@Hash, @VolumeId, @Size); SELECT last_insert_rowid();"); + m_insertfileOperationCommand = m_connection.CreateCommand(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (@FilesetId, @FileId, @LastModified)"); + m_insertfileCommand = m_connection.CreateCommand(@"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"",""BlocksetID"", ""MetadataID"") VALUES (@PrefixId, @Path, @BlocksetId, @MetadataId); SELECT last_insert_rowid();"); + m_insertblocksetCommand = m_connection.CreateCommand(@"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (@Length, @Fullhash); SELECT last_insert_rowid();"); + m_insertblocksetentryCommand = m_connection.CreateCommand(@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") SELECT @BlocksetId AS A, @Index AS B, ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); + m_insertblocklistHashesCommand = m_connection.CreateCommand(@"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (@BlocksetId, @Index, @Hash)"); + m_insertmetadatasetCommand = m_connection.CreateCommand(@"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (@BlocksetId); SELECT last_insert_rowid();"); + m_selectfilelastmodifiedCommand = m_connection.CreateCommand(@"SELECT ""A"".""ID"", ""B"".""LastModified"" FROM (SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = @FilesetId"); + m_selectfilelastmodifiedWithSizeCommand = m_connection.CreateCommand(@"SELECT ""C"".""ID"", ""C"".""LastModified"", ""D"".""Length"" FROM (SELECT ""A"".""ID"", ""B"".""LastModified"", ""A"".""BlocksetID"" FROM (SELECT ""ID"", ""BlocksetID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = @FilesetId) AS ""C"", ""Blockset"" AS ""D"" WHERE ""C"".""BlocksetID"" == ""D"".""ID"" "); + m_selectfilemetadatahashandsizeCommand = m_connection.CreateCommand(@"SELECT ""Blockset"".""Length"", ""Blockset"".""FullHash"" FROM ""Blockset"", ""Metadataset"", ""File"" WHERE ""File"".""ID"" = @FileId AND ""Blockset"".""ID"" = ""Metadataset"".""BlocksetID"" AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" "); // Allow users to test on real-world data // to get feedback on potential performance @@ -137,11 +116,11 @@ namespace Duplicati.Library.Main.Database findQuery = @" SELECT ""FileLookup"".""ID"" AS ""FileID"", ""FilesetEntry"".""Lastmodified"", ""FileBlockset"".""Length"", ""MetaBlockset"".""Fullhash"" AS ""Metahash"", ""MetaBlockset"".""Length"" AS ""Metasize"" " + @" FROM ""FileLookup"", ""FilesetEntry"", ""Fileset"", ""Blockset"" ""FileBlockset"", ""Metadataset"", ""Blockset"" ""MetaBlockset"" " + - @" WHERE ""FileLookup"".""PrefixID"" = ? AND ""FileLookup"".""Path"" = ? " + + @" WHERE ""FileLookup"".""PrefixID"" = @PrefixId AND ""FileLookup"".""Path"" = @Path " + @" AND ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" " + @" AND ""FileBlockset"".""ID"" = ""FileLookup"".""BlocksetID"" " + @" AND ""Metadataset"".""ID"" = ""FileLookup"".""MetadataID"" AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"" " + - @" AND ? IS NOT NULL" + + @" AND @FilesetId IS NOT NULL" + @" ORDER BY ""Fileset"".""Timestamp"" DESC " + @" LIMIT 1 "; break; @@ -151,10 +130,10 @@ namespace Duplicati.Library.Main.Database case 2: var getLastFileEntryForPath = @"SELECT ""A"".""ID"", ""B"".""LastModified"", ""A"".""BlocksetID"", ""A"".""MetadataID"" " + - @" FROM (SELECT ""ID"", ""BlocksetID"", ""MetadataID"" FROM ""FileLookup"" WHERE ""PrefixID"" = ? AND ""Path"" = ?) ""A"" " + + @" FROM (SELECT ""ID"", ""BlocksetID"", ""MetadataID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path) ""A"" " + @" CROSS JOIN ""FilesetEntry"" ""B"" " + @" WHERE ""A"".""ID"" = ""B"".""FileID"" " + - @" AND ""B"".""FilesetID"" = ? "; + @" AND ""B"".""FilesetID"" = @FilesetId "; findQuery = FormatInvariant($@"SELECT ""C"".""ID"" AS ""FileID"", ""C"".""LastModified"", ""D"".""Length"", ""E"".""FullHash"" as ""Metahash"", ""E"".""Length"" AS ""Metasize"" FROM ({getLastFileEntryForPath}) AS ""C"", ""Blockset"" AS ""D"", ""Blockset"" AS ""E"", ""Metadataset"" ""F"" @@ -173,7 +152,7 @@ LIMIT 1"); @"INNER JOIN Metadataset ON (Metadataset.ID = FileLookup.MetadataID) " + @"INNER JOIN Blockset AS MetaBlockset ON (MetaBlockset.ID = Metadataset.BlocksetID) " + @" LEFT JOIN Blockset AS FileBlockset ON (FileBlockset.ID = FileLookup.BlocksetID) " + - @" WHERE FileLookup.PrefixID = ? AND FileLookup.Path = ? AND FilesetID = ? " + + @" WHERE FileLookup.PrefixID = @PrefixId AND FileLookup.Path = @Path AND FilesetID = @FilesetId " + @" LIMIT 1 "; break; @@ -182,8 +161,8 @@ LIMIT 1"); findQuery = @" SELECT ""FileLookup"".""ID"" AS ""FileID"", ""FilesetEntry"".""Lastmodified"", ""FileBlockset"".""Length"", ""MetaBlockset"".""Fullhash"" AS ""Metahash"", ""MetaBlockset"".""Length"" AS ""Metasize"" " + @" FROM ""FileLookup"", ""FilesetEntry"", ""Fileset"", ""Blockset"" ""FileBlockset"", ""Metadataset"", ""Blockset"" ""MetaBlockset"" " + - @" WHERE ""FileLookup"".""PrefixID"" = ? AND ""FileLookup"".""Path"" = ? " + - @" AND ""Fileset"".""ID"" = ? " + + @" WHERE ""FileLookup"".""PrefixID"" = @PrefixId AND ""FileLookup"".""Path"" = @Path " + + @" AND ""Fileset"".""ID"" = @FilesetId " + @" AND ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" " + @" AND ""FileBlockset"".""ID"" = ""FileLookup"".""BlocksetID"" " + @" AND ""Metadataset"".""ID"" = ""FileLookup"".""MetadataID"" AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"" " + @@ -194,11 +173,10 @@ LIMIT 1"); m_findfileCommand = m_connection.CreateCommand(findQuery); - m_selectfileHashCommand = m_connection.CreateCommand(@"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"", ""FileLookup"" WHERE ""Blockset"".""ID"" = ""FileLookup"".""BlocksetID"" AND ""FileLookup"".""ID"" = ? "); - m_selectblocklistHashesCommand = m_connection.CreateCommand(@"SELECT ""Hash"" FROM ""BlocklistHash"" WHERE ""BlocksetID"" = ? ORDER BY ""Index"" ASC "); + m_selectfileHashCommand = m_connection.CreateCommand(@"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"", ""FileLookup"" WHERE ""Blockset"".""ID"" = ""FileLookup"".""BlocksetID"" AND ""FileLookup"".""ID"" = @FileId "); m_getfirstfilesetwithblockinblockset = m_connection.CreateCommand(@"SELECT MIN(""FilesetEntry"".""FilesetID"") FROM ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" IN ( SELECT ""File"".""ID"" FROM ""File"" WHERE ""File"".""BlocksetID"" IN( -SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""BlocklistHash"".""Hash"" = ?))"); +SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""BlocklistHash"".""Hash"" = @Hash))"); m_blocklistHashes = new HashSet(); } @@ -209,12 +187,12 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The block key /// The size of the block /// True if the block should be added to the current output - public long FindBlockID(string key, long size, IDbTransaction transaction = null) + public long FindBlockID(string key, long size, IDbTransaction? transaction = null) { - m_findblockCommand.Transaction = transaction; - m_findblockCommand.SetParameterValue(0, key); - m_findblockCommand.SetParameterValue(1, size); - return m_findblockCommand.ExecuteScalarInt64(m_logQueries, -1); + return m_findblockCommand.SetTransaction(transaction) + .SetParameterValue("@Hash", key) + .SetParameterValue("@Size", size) + .ExecuteScalarInt64(m_logQueries, -1); } /// @@ -223,20 +201,18 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The block key /// The size of the block /// True if the block should be added to the current output - public bool AddBlock(string key, long size, long volumeid, IDbTransaction transaction = null) + public bool AddBlock(string key, long size, long volumeid, IDbTransaction? transaction = null) { - m_findblockCommand.Transaction = transaction; - m_findblockCommand.SetParameterValue(0, key); - m_findblockCommand.SetParameterValue(1, size); - var r = m_findblockCommand.ExecuteScalarInt64(m_logQueries, -1); - + var r = FindBlockID(key, size, transaction); if (r == -1L) { - m_insertblockCommand.Transaction = transaction; - m_insertblockCommand.SetParameterValue(0, key); - m_insertblockCommand.SetParameterValue(1, volumeid); - m_insertblockCommand.SetParameterValue(2, size); - m_insertblockCommand.ExecuteScalarInt64(m_logQueries); + var ins = m_insertblockCommand.SetTransaction(transaction) + .SetParameterValue("@Hash", key) + .SetParameterValue("@VolumeId", volumeid) + .SetParameterValue("@Size", size) + .ExecuteNonQuery(m_logQueries); + if (ins != 1) + throw new Exception($"Failed to insert block {key} with size {size}, result count: {ins}"); return true; } else @@ -255,58 +231,64 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The list of hashes /// The id of the blockset, new or old /// True if the blockset was created, false otherwise - public bool AddBlockset(string filehash, long size, int blocksize, IEnumerable hashes, IEnumerable blocklistHashes, out long blocksetid, IDbTransaction transaction = null) + public bool AddBlockset(string filehash, long size, int blocksize, IEnumerable hashes, IEnumerable blocklistHashes, out long blocksetid, IDbTransaction? transaction = null) { - m_findblocksetCommand.Transaction = transaction; - blocksetid = m_findblocksetCommand.ExecuteScalarInt64(m_logQueries, null, -1, filehash, size); + blocksetid = m_findblocksetCommand.SetTransaction(transaction) + .SetParameterValue("@Fullhash", filehash) + .SetParameterValue("@Length", size) + .ExecuteScalarInt64(m_logQueries, null, -1); if (blocksetid != -1) return false; //Found it using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) { - m_insertblocksetCommand.Transaction = tr.Parent; - m_insertblocksetCommand.SetParameterValue(0, size); - m_insertblocksetCommand.SetParameterValue(1, filehash); - blocksetid = m_insertblocksetCommand.ExecuteScalarInt64(m_logQueries); + blocksetid = m_insertblocksetCommand.SetTransaction(tr.Parent) + .SetParameterValue("@Length", size) + .SetParameterValue("@Fullhash", filehash) + .ExecuteScalarInt64(m_logQueries); long ix = 0; if (blocklistHashes != null) { - m_insertblocklistHashesCommand.SetParameterValue(0, blocksetid); - m_insertblocklistHashesCommand.Transaction = tr.Parent; + m_insertblocklistHashesCommand.SetTransaction(tr.Parent) + .SetParameterValue("@BlocksetId", blocksetid); + foreach (var bh in blocklistHashes) { - m_insertblocklistHashesCommand.SetParameterValue(1, ix); - m_insertblocklistHashesCommand.SetParameterValue(2, bh); - m_insertblocklistHashesCommand.ExecuteNonQuery(m_logQueries); + var c = m_insertblocklistHashesCommand.SetParameterValue("@Index", ix) + .SetParameterValue("@Hash", bh) + .ExecuteNonQuery(m_logQueries); + if (c != 1) + throw new Exception($"Failed to insert blocklist hash {bh} for blockset {blocksetid}, result count: {c}"); ix++; } } - m_insertblocksetentryCommand.SetParameterValue(0, blocksetid); - m_insertblocksetentryCommand.Transaction = tr.Parent; - - m_insertblocksetentryFastCommand.SetParameterValue(0, blocksetid); - m_insertblocksetentryFastCommand.Transaction = tr.Parent; + m_insertblocksetentryCommand.SetTransaction(tr.Parent) + .SetParameterValue("@BlocksetId", blocksetid); ix = 0; long remainsize = size; foreach (var h in hashes) { var exsize = remainsize < blocksize ? remainsize : blocksize; - m_insertblocksetentryCommand.SetParameterValue(1, ix); - m_insertblocksetentryCommand.SetParameterValue(2, h); - m_insertblocksetentryCommand.SetParameterValue(3, exsize); - var c = m_insertblocksetentryCommand.ExecuteNonQuery(m_logQueries); + var c = m_insertblocksetentryCommand.SetParameterValue("@Index", ix) + .SetParameterValue("@Hash", h) + .SetParameterValue("@Size", exsize) + .ExecuteNonQuery(m_logQueries); if (c != 1) { Logging.Log.WriteErrorMessage(LOGTAG, "CheckingErrorsForIssue1400", null, "Checking errors, related to #1400. Unexpected result count: {0}, expected {1}, hash: {2}, size: {3}, blocksetid: {4}, ix: {5}, fullhash: {6}, fullsize: {7}", c, 1, h, exsize, blocksetid, ix, filehash, size); using (var cmd = m_connection.CreateCommand(tr.Parent)) { - var bid = cmd.ExecuteScalarInt64(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = ?", -1, h); + var bid = cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash") + .SetParameterValue("@Hash", h) + .ExecuteScalarInt64(-1); if (bid == -1) throw new Exception($"Could not find any blocks with the given hash: {h}"); - foreach (var rd in cmd.ExecuteReaderEnumerable(@"SELECT ""Size"" FROM ""Block"" WHERE ""Hash"" = ?", h)) + cmd.SetCommandAndParameters(@"SELECT ""Size"" FROM ""Block"" WHERE ""Hash"" = @Hash") + .SetParameterValue("@Hash", h); + foreach (var rd in cmd.ExecuteReaderEnumerable()) Logging.Log.WriteErrorMessage(LOGTAG, "FoundIssue1400Error", null, "Found block with ID {0} and hash {1} and size {2}", bid, h, rd.ConvertValueToInt64(0, -1)); } @@ -331,12 +313,14 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The size of the metadata. /// The ID of the metadataset. /// An optional transaction. - public bool GetMetadatasetID(string filehash, long size, out long metadataid, IDbTransaction transaction = null) + public bool GetMetadatasetID(string filehash, long size, out long metadataid, IDbTransaction? transaction = null) { if (size > 0) { - m_findmetadatasetCommand.Transaction = transaction; - metadataid = m_findmetadatasetCommand.ExecuteScalarInt64(m_logQueries, null, -1, filehash, size); + metadataid = m_findmetadatasetCommand.SetTransaction(transaction) + .SetParameterValue("@Hash", filehash) + .SetParameterValue("@Size", size) + .ExecuteScalarInt64(m_logQueries, null, -1); return metadataid != -1; } @@ -353,16 +337,16 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The id of the blockset to add /// The id of the metadata set /// True if the set was added to the database, false otherwise - public bool AddMetadataset(string filehash, long size, long blocksetid, out long metadataid, IDbTransaction transaction = null) + public bool AddMetadataset(string filehash, long size, long blocksetid, out long metadataid, IDbTransaction? transaction = null) { if (GetMetadatasetID(filehash, size, out metadataid, transaction)) return false; using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) { - m_insertmetadatasetCommand.Transaction = tr.Parent; - m_insertmetadatasetCommand.SetParameterValue(0, blocksetid); - metadataid = m_insertmetadatasetCommand.ExecuteScalarInt64(m_logQueries); + metadataid = m_insertmetadatasetCommand.SetTransaction(tr.Parent) + .SetParameterValue("@BlocksetId", blocksetid) + .ExecuteScalarInt64(m_logQueries); tr.Commit(); return true; } @@ -377,35 +361,30 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The ID of the hashkey for the file /// The ID for the metadata /// The transaction to use for insertion, or null for no transaction - public void AddFile(long pathprefixid, string filename, DateTime lastmodified, long blocksetID, long metadataID, IDbTransaction transaction) + public void AddFile(long pathprefixid, string filename, DateTime lastmodified, long blocksetID, long metadataID, IDbTransaction? transaction) { - var fileidobj = -1L; - m_findfilesetCommand.Transaction = transaction; - m_findfilesetCommand.SetParameterValue(0, blocksetID); - m_findfilesetCommand.SetParameterValue(1, metadataID); - m_findfilesetCommand.SetParameterValue(2, filename); - m_findfilesetCommand.SetParameterValue(3, pathprefixid); - fileidobj = m_findfilesetCommand.ExecuteScalarInt64(m_logQueries); + var fileidobj = m_findfilesetCommand.SetTransaction(transaction) + .SetParameterValue("@BlocksetId", blocksetID) + .SetParameterValue("@MetadataId", metadataID) + .SetParameterValue("@Path", filename) + .SetParameterValue("@PrefixId", pathprefixid) + .ExecuteScalarInt64(m_logQueries); if (fileidobj == -1) { using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) { - m_insertfileCommand.Transaction = tr.Parent; - m_insertfileCommand.SetParameterValue(0, pathprefixid); - m_insertfileCommand.SetParameterValue(1, filename); - m_insertfileCommand.SetParameterValue(2, blocksetID); - m_insertfileCommand.SetParameterValue(3, metadataID); - fileidobj = m_insertfileCommand.ExecuteScalarInt64(m_logQueries); + fileidobj = m_insertfileCommand.SetTransaction(tr.Parent) + .SetParameterValue("@PrefixId", pathprefixid) + .SetParameterValue("@Path", filename) + .SetParameterValue("@BlocksetId", blocksetID) + .SetParameterValue("@MetadataId", metadataID) + .ExecuteScalarInt64(m_logQueries); tr.Commit(); } } - m_insertfileOperationCommand.Transaction = transaction; - m_insertfileOperationCommand.SetParameterValue(0, m_filesetId); - m_insertfileOperationCommand.SetParameterValue(1, fileidobj); - m_insertfileOperationCommand.SetParameterValue(2, lastmodified.ToUniversalTime().Ticks); - m_insertfileOperationCommand.ExecuteNonQuery(m_logQueries); + AddKnownFile(fileidobj, lastmodified, transaction); } /// @@ -416,39 +395,45 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The ID of the hashkey for the file /// The ID for the metadata /// The transaction to use for insertion, or null for no transaction - public void AddFile(string filename, DateTime lastmodified, long blocksetID, long metadataID, IDbTransaction transaction) + public void AddFile(string filename, DateTime lastmodified, long blocksetID, long metadataID, IDbTransaction? transaction) { var split = SplitIntoPrefixAndName(filename); AddFile(GetOrCreatePathPrefix(split.Key, transaction), split.Value, lastmodified, blocksetID, metadataID, transaction); } - public void AddUnmodifiedFile(long fileid, DateTime lastmodified, IDbTransaction transaction = null) + /// + /// Adds a known file to the fileset + /// + /// Id of the file + /// The time the file was modified + /// The transaction to use for insertion, or null for no transaction + public void AddKnownFile(long fileid, DateTime lastmodified, IDbTransaction? transaction = null) { - m_insertfileOperationCommand.Transaction = transaction; - m_insertfileOperationCommand.SetParameterValue(0, m_filesetId); - m_insertfileOperationCommand.SetParameterValue(1, fileid); - m_insertfileOperationCommand.SetParameterValue(2, lastmodified.ToUniversalTime().Ticks); - m_insertfileOperationCommand.ExecuteNonQuery(m_logQueries); + m_insertfileOperationCommand.SetTransaction(transaction) + .SetParameterValue("@FilesetId", m_filesetId) + .SetParameterValue("@FileId", fileid) + .SetParameterValue("@LastModified", lastmodified.ToUniversalTime().Ticks) + .ExecuteNonQuery(m_logQueries); } - public void AddDirectoryEntry(string path, long metadataID, DateTime lastmodified, IDbTransaction transaction = null) + public void AddDirectoryEntry(string path, long metadataID, DateTime lastmodified, IDbTransaction? transaction = null) { AddFile(path, lastmodified, FOLDER_BLOCKSET_ID, metadataID, transaction); } - public void AddSymlinkEntry(string path, long metadataID, DateTime lastmodified, IDbTransaction transaction = null) + public void AddSymlinkEntry(string path, long metadataID, DateTime lastmodified, IDbTransaction? transaction = null) { AddFile(path, lastmodified, SYMLINK_BLOCKSET_ID, metadataID, transaction); } - public long GetFileLastModified(long prefixid, string path, long filesetid, bool includeLength, out DateTime oldModified, out long length, IDbTransaction transaction = null) + public long GetFileLastModified(long prefixid, string path, long filesetid, bool includeLength, out DateTime oldModified, out long length, IDbTransaction? transaction = null) { if (includeLength) { - m_selectfilelastmodifiedWithSizeCommand.Transaction = transaction; - m_selectfilelastmodifiedWithSizeCommand.SetParameterValue(0, prefixid); - m_selectfilelastmodifiedWithSizeCommand.SetParameterValue(1, path); - m_selectfilelastmodifiedWithSizeCommand.SetParameterValue(2, filesetid); + m_selectfilelastmodifiedWithSizeCommand.SetTransaction(transaction) + .SetParameterValue("@PrefixId", prefixid) + .SetParameterValue("@Path", path) + .SetParameterValue("@FilesetId", filesetid); using (var rd = m_selectfilelastmodifiedWithSizeCommand.ExecuteReader(m_logQueries, null)) if (rd.Read()) { @@ -459,10 +444,10 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } else { - m_selectfilelastmodifiedCommand.Transaction = transaction; - m_selectfilelastmodifiedCommand.SetParameterValue(0, prefixid); - m_selectfilelastmodifiedCommand.SetParameterValue(1, path); - m_selectfilelastmodifiedCommand.SetParameterValue(2, filesetid); + m_selectfilelastmodifiedCommand.SetTransaction(transaction) + .SetParameterValue("@PrefixId", prefixid) + .SetParameterValue("@Path", path) + .SetParameterValue("@FilesetId", filesetid); using (var rd = m_selectfilelastmodifiedCommand.ExecuteReader(m_logQueries, null)) if (rd.Read()) { @@ -478,12 +463,12 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } - public long GetFileEntry(long prefixid, string path, long filesetid, out DateTime oldModified, out long lastFileSize, out string oldMetahash, out long oldMetasize, IDbTransaction transaction) + public long GetFileEntry(long prefixid, string path, long filesetid, out DateTime oldModified, out long lastFileSize, out string? oldMetahash, out long oldMetasize, IDbTransaction transaction) { - m_findfileCommand.SetParameterValue(0, prefixid); - m_findfileCommand.SetParameterValue(1, path); - m_findfileCommand.SetParameterValue(2, filesetid); - m_findfileCommand.Transaction = transaction; + m_findfileCommand.SetTransaction(transaction) + .SetParameterValue("@PrefixId", prefixid) + .SetParameterValue("@Path", path) + .SetParameterValue("@FilesetId", filesetid); using (var rd = m_findfileCommand.ExecuteReader()) if (rd.Read()) @@ -504,24 +489,24 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } } - public Tuple GetMetadataHashAndSizeForFile(long fileid, IDbTransaction transaction) + public (long Size, string MetadataHash)? GetMetadataHashAndSizeForFile(long fileid, IDbTransaction transaction) { - m_selectfilemetadatahashandsizeCommand.SetParameterValue(0, fileid); - m_selectfilemetadatahashandsizeCommand.Transaction = transaction; + m_selectfilemetadatahashandsizeCommand.SetTransaction(transaction) + .SetParameterValue("@FileId", fileid); - using (var rd = m_findfileCommand.ExecuteReader()) + using (var rd = m_selectfilemetadatahashandsizeCommand.ExecuteReader()) if (rd.Read()) - return new Tuple(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1)); + return (rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? throw new InvalidOperationException("Metadata hash is null")); return null; } - public string GetFileHash(long fileid, IDbTransaction transaction) + public string? GetFileHash(long fileid, IDbTransaction transaction) { - m_selectfileHashCommand.SetParameterValue(0, fileid); - m_selectfileHashCommand.Transaction = transaction; - var r = m_selectfileHashCommand.ExecuteScalar(m_logQueries, null); + var r = m_selectfileHashCommand.SetTransaction(transaction) + .SetParameterValue("@FileId", fileid) + .ExecuteScalar(m_logQueries, null); if (r == null || r == DBNull.Value) return null; @@ -535,13 +520,16 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist private long GetPreviousFilesetID(IDbCommand cmd) { - return GetPreviousFilesetID(cmd, OperationTimestamp, m_filesetId); + return GetPreviousFilesetID(cmd, OperationTimestamp, m_filesetId, cmd.Transaction); } - private long GetPreviousFilesetID(IDbCommand cmd, DateTime timestamp, long filesetid) + private long GetPreviousFilesetID(IDbCommand cmd, DateTime timestamp, long filesetid, IDbTransaction? transaction) { - var lastFilesetId = cmd.ExecuteScalarInt64(@"SELECT ""ID"" FROM ""Fileset"" WHERE ""Timestamp"" < ? AND ""ID"" != ? ORDER BY ""Timestamp"" DESC ", -1, Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(timestamp), filesetid); - return lastFilesetId; + return cmd.SetTransaction(transaction) + .SetCommandAndParameters(@"SELECT ""ID"" FROM ""Fileset"" WHERE ""Timestamp"" < @Timestamp AND ""ID"" != @FilesetId ORDER BY ""Timestamp"" DESC ") + .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(timestamp)) + .SetParameterValue("@FilesetId", filesetid) + .ExecuteScalarInt64(-1); } internal Tuple GetLastBackupFileCountAndSize() @@ -549,8 +537,17 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist using (var cmd = m_connection.CreateCommand()) { var lastFilesetId = cmd.ExecuteScalarInt64(@"SELECT ""ID"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC LIMIT 1"); - var count = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""FileLookup"" INNER JOIN ""FilesetEntry"" ON ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""FileLookup"".""BlocksetID"" NOT IN (?, ?)", -1, lastFilesetId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); - var size = cmd.ExecuteScalarInt64(@"SELECT SUM(""Blockset"".""Length"") FROM ""FileLookup"", ""FilesetEntry"", ""Blockset"" WHERE ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""FileLookup"".""BlocksetID"" NOT IN (?, ?)", -1, lastFilesetId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); + var count = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""FileLookup"" INNER JOIN ""FilesetEntry"" ON ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""FileLookup"".""BlocksetID"" NOT IN (@FolderBlocksetId, @SymlinkBlocksetId)") + .SetParameterValue("@FilesetId", lastFilesetId) + .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) + .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) + .ExecuteScalarInt64(-1); + + var size = cmd.SetCommandAndParameters(@"SELECT SUM(""Blockset"".""Length"") FROM ""FileLookup"", ""FilesetEntry"", ""Blockset"" WHERE ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""FileLookup"".""BlocksetID"" NOT IN (@FolderBlocksetId, @SymlinkBlocksetId)") + .SetParameterValue("@FilesetId", lastFilesetId) + .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) + .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) + .ExecuteScalarInt64(-1); return new Tuple(count, size); } @@ -562,32 +559,68 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist { // TODO: Optimize these queries to not use the "File" view var lastFilesetId = GetPreviousFilesetID(cmd); - results.AddedFolders = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" = ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)", 0, m_filesetId, FOLDER_BLOCKSET_ID, lastFilesetId); - results.AddedSymlinks = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" = ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)", 0, m_filesetId, SYMLINK_BLOCKSET_ID, lastFilesetId); + results.AddedFolders = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @CurrentFilesetId AND ""File"".""BlocksetID"" = @BlocksetId AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @LastFilsetId)") + .SetParameterValue("@CurrentFilesetId", m_filesetId) + .SetParameterValue("@BlocksetId", FOLDER_BLOCKSET_ID) + .SetParameterValue("@LastFilsetId", lastFilesetId) + .ExecuteScalarInt64(0); + results.AddedSymlinks = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @CurrentFilesetId AND ""File"".""BlocksetID"" = @BlocksetId AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @LastFilsetId)") + .SetParameterValue("@CurrentFilesetId", m_filesetId) + .SetParameterValue("@BlocksetId", SYMLINK_BLOCKSET_ID) + .SetParameterValue("@LastFilsetId", lastFilesetId) + .ExecuteScalarInt64(0); - results.DeletedFolders = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" = ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)", 0, lastFilesetId, FOLDER_BLOCKSET_ID, m_filesetId); - results.DeletedSymlinks = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" = ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)", 0, lastFilesetId, SYMLINK_BLOCKSET_ID, m_filesetId); + results.DeletedFolders = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @LastFilsetId AND ""File"".""BlocksetID"" = @BlocksetId AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @CurrentFilesetId)") + .SetParameterValue("@LastFilsetId", lastFilesetId) + .SetParameterValue("@BlocksetId", FOLDER_BLOCKSET_ID) + .SetParameterValue("@CurrentFilesetId", m_filesetId) + .ExecuteScalarInt64(0); + results.DeletedSymlinks = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @LastFilsetId AND ""File"".""BlocksetID"" = @BlocksetId AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @CurrentFilesetId)") + .SetParameterValue("@LastFilsetId", lastFilesetId) + .SetParameterValue("@BlocksetId", SYMLINK_BLOCKSET_ID) + .SetParameterValue("@CurrentFilesetId", m_filesetId) + .ExecuteScalarInt64(0); - var subqueryNonFiles = @"SELECT ""File"".""Path"", ""Blockset"".""Fullhash"" FROM ""File"", ""FilesetEntry"", ""Metadataset"", ""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" AND ""File"".""BlocksetID"" = ? AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FilesetID"" = ? "; - results.ModifiedFolders = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM (" + subqueryNonFiles + @") A, (" + subqueryNonFiles + @") B WHERE ""A"".""Path"" = ""B"".""Path"" AND ""A"".""Fullhash"" != ""B"".""Fullhash"" ", 0, lastFilesetId, FOLDER_BLOCKSET_ID, m_filesetId, FOLDER_BLOCKSET_ID); - results.ModifiedSymlinks = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM (" + subqueryNonFiles + @") A, (" + subqueryNonFiles + @") B WHERE ""A"".""Path"" = ""B"".""Path"" AND ""A"".""Fullhash"" != ""B"".""Fullhash"" ", 0, lastFilesetId, SYMLINK_BLOCKSET_ID, m_filesetId, SYMLINK_BLOCKSET_ID); + var subqueryNonFiles = @"SELECT ""File"".""Path"", ""Blockset"".""Fullhash"" FROM ""File"", ""FilesetEntry"", ""Metadataset"", ""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" AND ""File"".""BlocksetID"" = @BlocksetId AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FilesetID"" = "; + var subQueryNonFilesPrevious = subqueryNonFiles + "@LastFilesetId"; + var subQueryNonFilesCurrent = subqueryNonFiles + "@CurrentFilesetId"; + + results.ModifiedFolders = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM (" + subQueryNonFilesPrevious + @") A, (" + subQueryNonFilesCurrent + @") B WHERE ""A"".""Path"" = ""B"".""Path"" AND ""A"".""Fullhash"" != ""B"".""Fullhash"" ") + .SetParameterValue("@LastFilesetId", lastFilesetId) + .SetParameterValue("@CurrentFilesetId", m_filesetId) + .SetParameterValue("@BlocksetId", FOLDER_BLOCKSET_ID) + .ExecuteScalarInt64(0); + + results.ModifiedSymlinks = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM (" + subQueryNonFilesPrevious + @") A, (" + subQueryNonFilesCurrent + @") B WHERE ""A"".""Path"" = ""B"".""Path"" AND ""A"".""Fullhash"" != ""B"".""Fullhash"" ") + .SetParameterValue("@LastFilesetId", lastFilesetId) + .SetParameterValue("@CurrentFilesetId", m_filesetId) + .SetParameterValue("@BlocksetId", SYMLINK_BLOCKSET_ID) + .ExecuteScalarInt64(0); var tmpName1 = "TmpFileList-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); var tmpName2 = "TmpFileList-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); try { - var subqueryFiles = @"SELECT ""File"".""Path"" AS ""Path"", ""A"".""Fullhash"" AS ""Filehash"", ""B"".""Fullhash"" AS ""Metahash"" FROM ""File"", ""FilesetEntry"", ""Blockset"" A, ""Blockset"" B, ""Metadataset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""A"".""ID"" = ""File"".""BlocksetID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""B"".""ID"" "; + var subqueryFiles = @"SELECT ""File"".""Path"" AS ""Path"", ""A"".""Fullhash"" AS ""Filehash"", ""B"".""Fullhash"" AS ""Metahash"" FROM ""File"", ""FilesetEntry"", ""Blockset"" A, ""Blockset"" B, ""Metadataset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""A"".""ID"" = ""File"".""BlocksetID"" AND ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""File"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""B"".""ID"" "; - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tmpName1}"" AS {subqueryFiles}"), lastFilesetId); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tmpName2}"" AS {subqueryFiles}"), m_filesetId); + cmd.SetCommandAndParameters(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tmpName1}"" AS {subqueryFiles}")) + .SetParameterValue("@FilesetId", lastFilesetId) + .ExecuteNonQuery(); + cmd.SetCommandAndParameters(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tmpName2}"" AS {subqueryFiles}")) + .SetParameterValue("@FilesetId", m_filesetId) + .ExecuteNonQuery(); cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""nn_tmpName1"" ON ""{tmpName1}"" (""Path"")")); cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""nn_tmpName2"" ON ""{tmpName2}"" (""Path"")")); - - results.AddedFiles = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" != ? AND ""File"".""BlocksetID"" != ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""{tmpName1}"")"), 0, m_filesetId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); - results.DeletedFiles = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""{tmpName1}"" WHERE ""{tmpName1}"".""Path"" NOT IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)"), 0, m_filesetId); + results.AddedFiles = cmd.SetCommandAndParameters(FormatInvariant($@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""File"".""BlocksetID"" != @FolderBlocksetId AND ""File"".""BlocksetID"" != @SymlinkBlocksetId AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""{tmpName1}"")")) + .SetParameterValue("@FilesetId", m_filesetId) + .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) + .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) + .ExecuteScalarInt64(0); + results.DeletedFiles = cmd.SetCommandAndParameters(FormatInvariant($@"SELECT COUNT(*) FROM ""{tmpName1}"" WHERE ""{tmpName1}"".""Path"" NOT IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId)")) + .SetParameterValue("@FilesetId", m_filesetId) + .ExecuteScalarInt64(0); results.ModifiedFiles = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""{tmpName1}"" A, ""{tmpName2}"" B WHERE ""A"".""Path"" = ""B"".""Path"" AND (""A"".""Filehash"" != ""B"".""Filehash"" OR ""A"".""Metahash"" != ""B"".""Metahash"")"), 0); - } finally { @@ -605,7 +638,7 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// /// Transaction /// List of deleted paths, or null - public void AppendFilesFromPreviousSet(IDbTransaction transaction, IEnumerable deleted = null) + public void AppendFilesFromPreviousSet(IDbTransaction transaction, IEnumerable? deleted = null) { AppendFilesFromPreviousSet(transaction, deleted, m_filesetId, -1, OperationTimestamp); } @@ -619,29 +652,28 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// Current file-set ID /// Source file-set ID /// If filesetid == -1, used to locate previous file-set - public void AppendFilesFromPreviousSet(IDbTransaction transaction, IEnumerable deleted, long filesetid, long prevId, DateTime timestamp) + public void AppendFilesFromPreviousSet(IDbTransaction transaction, IEnumerable? deleted, long filesetid, long prevId, DateTime timestamp) { using (var cmd = m_connection.CreateCommand()) using (var cmdDelete = m_connection.CreateCommand()) using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) { - long lastFilesetId = prevId < 0 ? GetPreviousFilesetID(cmd, timestamp, filesetid) : prevId; + long lastFilesetId = prevId < 0 ? GetPreviousFilesetID(cmd, timestamp, filesetid, tr.Parent) : prevId; - cmd.Transaction = tr.Parent; - cmd.ExecuteNonQuery(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") SELECT ? AS ""FilesetID"", ""FileID"", ""Lastmodified"" FROM (SELECT DISTINCT ""FilesetID"", ""FileID"", ""Lastmodified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ? AND ""FileID"" NOT IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ?)) ", filesetid, lastFilesetId, filesetid); + cmd.SetTransaction(tr.Parent) + .SetCommandAndParameters(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") SELECT @CurrentFilesetId AS ""FilesetID"", ""FileID"", ""Lastmodified"" FROM (SELECT DISTINCT ""FilesetID"", ""FileID"", ""Lastmodified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @PreviousFilesetId AND ""FileID"" NOT IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @CurrentFilesetId)) ") + .SetParameterValue("@CurrentFilesetId", filesetid) + .SetParameterValue("@PreviousFilesetId", lastFilesetId) + .ExecuteNonQuery(m_logQueries); if (deleted != null) { - cmdDelete.Transaction = tr.Parent; - cmdDelete.CommandText = @"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = ? AND ""FileID"" IN (SELECT ""ID"" FROM ""File"" WHERE ""Path"" = ?) "; - cmdDelete.AddParameters(2); - cmdDelete.SetParameterValue(0, filesetid); - - foreach (string s in deleted) - { - cmdDelete.SetParameterValue(1, s); - cmdDelete.ExecuteNonQuery(); - } + using var tmplist = new TemporaryDbValueList(m_connection, tr.Parent, deleted); + cmdDelete.SetTransaction(tr.Parent) + .SetCommandAndParameters(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId AND ""FileID"" IN (SELECT ""ID"" FROM ""File"" WHERE ""Path"" IN (@Paths)) ") + .SetParameterValue("@FilesetId", filesetid) + .ExpandInClauseParameter("@Paths", tmplist) + .ExecuteNonQuery(m_logQueries); } tr.Commit(); @@ -683,18 +715,20 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist using (var cmdDelete = m_connection.CreateCommand()) using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) { - long lastFilesetId = prevFileSetId < 0 ? GetPreviousFilesetID(cmd, timestamp, fileSetId) : prevFileSetId; + long lastFilesetId = prevFileSetId < 0 ? GetPreviousFilesetID(cmd, timestamp, fileSetId, tr.Parent) : prevFileSetId; // copy entries from previous file set into a temporary table, except those file IDs already added by the current backup var tempFileSetTable = "FilesetEntry-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); cmd.Transaction = tr.Parent; - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tempFileSetTable}"" AS SELECT ""FileID"", ""Lastmodified"" FROM (SELECT DISTINCT ""FilesetID"", ""FileID"", ""Lastmodified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ? AND ""FileID"" NOT IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ?))"), lastFilesetId, fileSetId); + cmd.SetCommandAndParameters(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tempFileSetTable}"" AS SELECT ""FileID"", ""Lastmodified"" FROM (SELECT DISTINCT ""FilesetID"", ""FileID"", ""Lastmodified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @PreviousFilesetId AND ""FileID"" NOT IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @CurrentFilesetId))")) + .SetParameterValue("@PreviousFilesetId", lastFilesetId) + .SetParameterValue("@CurrentFilesetId", fileSetId) + .ExecuteNonQuery(); // now we need to remove, from the above, any entries that were enumerated by the // UNC-driven backup - cmdDelete.Transaction = tr.Parent; - cmdDelete.CommandText = FormatInvariant($@"DELETE FROM ""{tempFileSetTable}"" WHERE ""FileID"" = ?"); - cmdDelete.AddParameters(1); + cmdDelete.SetTransaction(tr.Parent) + .SetCommandAndParameters(FormatInvariant($@"DELETE FROM ""{tempFileSetTable}"" WHERE ""FileID"" = @FileId")); // enumerate files from new temporary file set, and remove any entries handled by UNC cmd.Transaction = tr.Parent; @@ -704,19 +738,19 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist LEFT JOIN ""File"" AS f ON fs.""FileID"" = f.""ID"" LEFT JOIN ""Blockset"" AS bs ON f.""BlocksetID"" = bs.""ID"";"))) { - var path = row.GetString(0); - var size = row.GetInt64(3); + var path = row.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for path"); + var size = row.ConvertValueToInt64(3); if (exclusionPredicate(path, size)) - { - cmdDelete.SetParameterValue(0, row.GetInt64(1)); - cmdDelete.ExecuteNonQuery(); - } + cmdDelete.SetParameterValue("@FileId", row.ConvertValueToInt64(1)) + .ExecuteNonQuery(); } // now copy the temporary table into the FileSetEntry table - cmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") - SELECT ?, ""FileID"", ""Lastmodified"" FROM ""{tempFileSetTable}"""), fileSetId); + cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") + SELECT @FilesetId, ""FileID"", ""Lastmodified"" FROM ""{tempFileSetTable}""")) + .SetParameterValue("@FilesetId", fileSetId) + .ExecuteNonQuery(); tr.Commit(); } @@ -728,12 +762,12 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// The ID of the fileset volume to update /// The timestamp of the operation to create /// An optional external transaction - public override long CreateFileset(long volumeid, DateTime timestamp, IDbTransaction transaction = null) + public override long CreateFileset(long volumeid, DateTime timestamp, IDbTransaction? transaction = null) { return m_filesetId = base.CreateFileset(volumeid, timestamp, transaction); } - public IEnumerable GetTemporaryFilelistVolumeNames(bool latestOnly, IDbTransaction transaction = null) + public IEnumerable GetTemporaryFilelistVolumeNames(bool latestOnly, IDbTransaction? transaction = null) { var incompleteFilesetIDs = GetIncompleteFilesets(transaction).OrderBy(x => x.Value).Select(x => x.Key).ToArray(); @@ -752,10 +786,14 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist public IEnumerable GetMissingIndexFiles(IDbTransaction transaction) { - using (var cmd = m_connection.CreateCommand(transaction)) - using (var rd = cmd.ExecuteReader(@"SELECT ""Name"" FROM ""RemoteVolume"" WHERE ""Type"" = ? AND NOT ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"") AND ""State"" IN (?,?)", RemoteVolumeType.Blocks.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString())) + using var cmd = m_connection.CreateCommand(transaction) + .SetCommandAndParameters(@"SELECT ""Name"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND NOT ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"") AND ""State"" IN (@States)") + .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) + .ExpandInClauseParameter("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); + + using (var rd = cmd.ExecuteReader()) while (rd.Read()) - yield return rd.GetValue(0).ToString(); + yield return rd.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for volume name"); } public void MoveBlockToVolume(string blockkey, long size, long sourcevolumeid, long targetvolumeid, IDbTransaction transaction) @@ -763,7 +801,12 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist using (var cmd = m_connection.CreateCommand()) { cmd.Transaction = transaction; - var c = cmd.ExecuteNonQuery(@"UPDATE ""Block"" SET ""VolumeID"" = ? WHERE ""Hash"" = ? AND ""Size"" = ? AND ""VolumeID"" = ? ", targetvolumeid, blockkey, size, sourcevolumeid); + var c = cmd.SetCommandAndParameters(@"UPDATE ""Block"" SET ""VolumeID"" = @NewVolumeId WHERE ""Hash"" = @Hash AND ""Size"" = @Size AND ""VolumeID"" = @PreviousVolumeId ") + .SetParameterValue("@NewVolumeId", targetvolumeid) + .SetParameterValue("@Hash", blockkey) + .SetParameterValue("@Size", size) + .SetParameterValue("@PreviousVolumeId", sourcevolumeid) + .ExecuteNonQuery(); if (c != 1) throw new Exception($"Failed to move block {blockkey}:{size} from volume {sourcevolumeid}, count: {c}"); } @@ -775,7 +818,9 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist using (var cmd = m_connection.CreateCommand(transaction)) { - var c = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Block"" WHERE ""VolumeID"" = ? ", -1, volumeid); + var c = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""Block"" WHERE ""VolumeID"" = @VolumeId ") + .SetParameterValue("@VolumeId", volumeid) + .ExecuteScalarInt64(-1); if (c != 0) throw new Exception($"Failed to safe-delete volume {name}, blocks: {c}"); @@ -789,14 +834,15 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist using (var cmd = m_connection.CreateCommand(transaction)) { // Grab the strings and return as array to avoid concurrent access to the IEnumerable - return cmd.ExecuteReaderEnumerable( - @"SELECT DISTINCT ""Block"".""Hash"" FROM ""Block"" WHERE ""Block"".""VolumeID"" = ? AND ""Block"".""Hash"" IN (SELECT ""Hash"" FROM ""BlocklistHash"")", volumeid) - .Select(x => x.ConvertValueToString(0)) + cmd.SetCommandAndParameters(@"SELECT DISTINCT ""Block"".""Hash"" FROM ""Block"" WHERE ""Block"".""VolumeID"" = @VolumeId AND ""Block"".""Hash"" IN (SELECT ""Hash"" FROM ""BlocklistHash"")") + .SetParameterValue("@VolumeId", volumeid); + return cmd.ExecuteReaderEnumerable() + .Select(x => x.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for blocklist hash")) .ToArray(); } } - public string GetFirstPath() + public string? GetFirstPath() { using (var cmd = m_connection.CreateCommand()) { @@ -817,11 +863,9 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist { var data = new List(); - using (var cmd = m_connection.CreateCommand()) - using (var rd = - cmd.ExecuteReader( - @"SELECT ""VolumeName"", ""JournalID"", ""NextUSN"", ""ConfigHash"" FROM ""ChangeJournalData"" WHERE ""FilesetID"" = ?", - fileSetId)) + using var cmd = m_connection.CreateCommand(@"SELECT ""VolumeName"", ""JournalID"", ""NextUSN"", ""ConfigHash"" FROM ""ChangeJournalData"" WHERE ""FilesetID"" = @FilesetId") + .SetParameterValue("@FilesetId", fileSetId); + using (var rd = cmd.ExecuteReader()) { while (rd.Read()) { @@ -844,7 +888,7 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// /// Data to add /// An optional external transaction - public void CreateChangeJournalData(IEnumerable data, IDbTransaction transaction = null) + public void CreateChangeJournalData(IEnumerable data, IDbTransaction? transaction = null) { using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) { @@ -853,9 +897,14 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist using (var cmd = m_connection.CreateCommand()) { cmd.Transaction = tr.Parent; - var c = cmd.ExecuteNonQuery( - @"INSERT INTO ""ChangeJournalData"" (""FilesetID"", ""VolumeName"", ""JournalID"", ""NextUSN"", ""ConfigHash"") VALUES (?, ?, ?, ?, ?);", - m_filesetId, entry.Volume, entry.JournalId, entry.NextUsn, entry.ConfigHash); + var c = cmd.SetCommandAndParameters( + @"INSERT INTO ""ChangeJournalData"" (""FilesetID"", ""VolumeName"", ""JournalID"", ""NextUSN"", ""ConfigHash"") VALUES (@FilesetId, @VolumeName, @JournalId, @NextUsn, @ConfigHash);") + .SetParameterValue("@FilesetId", m_filesetId) + .SetParameterValue("@VolumeName", entry.Volume) + .SetParameterValue("@JournalId", entry.JournalId) + .SetParameterValue("@NextUsn", entry.NextUsn) + .SetParameterValue("@ConfigHash", entry.ConfigHash) + .ExecuteNonQuery(); if (c != 1) throw new Exception("Unable to add change journal entry"); @@ -872,7 +921,7 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// Data to add /// Existing file set to update /// An optional external transaction - public void UpdateChangeJournalData(IEnumerable data, long fileSetId, IDbTransaction transaction = null) + public void UpdateChangeJournalData(IEnumerable data, long fileSetId, IDbTransaction? transaction = null) { using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) { @@ -880,10 +929,14 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist { using (var cmd = m_connection.CreateCommand()) { - cmd.Transaction = tr.Parent; - cmd.ExecuteNonQuery( - @"UPDATE ""ChangeJournalData"" SET ""NextUSN"" = ? WHERE ""FilesetID"" = ? AND ""VolumeName"" = ? AND ""JournalID"" = ?;", - entry.NextUsn, fileSetId, entry.Volume, entry.JournalId); + cmd.SetTransaction(tr.Parent) + .SetCommandAndParameters( + @"UPDATE ""ChangeJournalData"" SET ""NextUSN"" = @NextUsn WHERE ""FilesetID"" = @FilesetId AND ""VolumeName"" = @VolumeName AND ""JournalID"" = @JournalId;") + .SetParameterValue("@NextUsn", entry.NextUsn) + .SetParameterValue("@FilesetId", fileSetId) + .SetParameterValue("@VolumeName", entry.Volume) + .SetParameterValue("@JournalId", entry.JournalId) + .ExecuteNonQuery(); } } @@ -899,9 +952,9 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// True if the hash is known, false otherwise public bool IsBlocklistHashKnown(string hash, IDbTransaction transaction) { - m_getfirstfilesetwithblockinblockset.Transaction = transaction; - m_getfirstfilesetwithblockinblockset.SetParameterValue(0, hash); - var res = m_getfirstfilesetwithblockinblockset.ExecuteScalarInt64(); + var res = m_getfirstfilesetwithblockinblockset.SetTransaction(transaction) + .SetParameterValue("@Hash", hash) + .ExecuteScalarInt64(); if (res != -1 && res != m_filesetId) return true; else diff --git a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs index cc7eb1b39..ccaa96d5e 100644 --- a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs +++ b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs @@ -100,7 +100,7 @@ namespace Duplicati.Library.Main.Operation.Backup return RunOnMain(() => m_database.AddSymlinkEntry(filename, metadataid, lastModified, GetTransaction())); } - public Task> GetMetadataHashAndSizeForFileAsync(long fileid) + public Task<(long Size, string MetadataHash)?> GetMetadataHashAndSizeForFileAsync(long fileid) { return RunOnMain(() => m_database.GetMetadataHashAndSizeForFile(fileid, GetTransaction())); } @@ -154,7 +154,7 @@ namespace Duplicati.Library.Main.Operation.Backup public Task AddUnmodifiedAsync(long fileid, DateTime lastModified) { - return RunOnMain(() => m_database.AddUnmodifiedFile(fileid, lastModified, GetTransaction())); + return RunOnMain(() => m_database.AddKnownFile(fileid, lastModified, GetTransaction())); } diff --git a/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs b/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs index 03e651c17..1574bff60 100644 --- a/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs @@ -124,8 +124,8 @@ namespace Duplicati.Library.Main.Operation.Backup var tp = await database.GetMetadataHashAndSizeForFileAsync(e.OldId); if (tp != null) { - e.OldMetaSize = tp.Item1; - e.OldMetaHash = tp.Item2; + e.OldMetaSize = tp.Value.Size; + e.OldMetaHash = tp.Value.MetadataHash; } }