haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added LocalBackupDatabase

Browse Source
This commit is contained in:
Kenneth Skovhede
2025-04-03 11:31:36 +02:00
parent 7c8571ed33
commit 9e3aa72d00
4 changed files with 294 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Main/Database/ExtensionMethods.cs
+17 -3
View File
@@ -194,6 +194,20 @@ public static class ExtensionMethods
return self;
}
/// <summary>
/// Sets the transaction for the command.
/// </summary>
/// <typeparam name="T">The type of the command</typeparam>
/// <param name="self">The command to set the transaction for</param>
/// <param name="transaction">The transaction to set for the command</param>
/// <returns>>The command with the transaction set</returns>
public static T SetTransaction<T>(this T self, IDbTransaction? transaction)
where T : IDbCommand
{
self.Transaction = transaction;
return self;
}
/// <summary>
/// Gets the printable command text for the given command.
/// </summary
@@ -330,7 +344,7 @@ public static class ExtensionMethods
/// <param name="cmd">The command string to execute</param>
/// <param name="values">The values to use as parameters. The parameters must already be added.</param>
/// <returns>The scalar value of the first row</returns>
public static object? ExecuteScalar(this IDbCommand self, bool writeLog, string cmd, params object[] values)
public static object? ExecuteScalar(this IDbCommand self, bool writeLog, string? cmd, params object[] values)
{
if (cmd != null)
self.SetCommandAndParameters(cmd);
@@ -474,7 +488,7 @@ public static class ExtensionMethods
/// <param name="cmd">The command string to execute</param>
/// <param name="values">The values to use as parameters. The parameters must already be added.</param>
/// <returns>A <see cref="IDataReader"/> instance</returns>
public static IDataReader ExecuteReader(this IDbCommand self, bool writeLog, string cmd, params object[] values)
public static IDataReader ExecuteReader(this IDbCommand self, bool writeLog, string? cmd, params object[] values)
{
if (cmd != null)
self.SetCommandAndParameters(cmd);
@@ -494,7 +508,7 @@ public static class ExtensionMethods
/// <param name="cmd">The command string to execute</param>
/// <param name="values">The values to use as parameters. The parameters must already be added.</param>
/// <returns>A <see cref="IDataReader"/> instance</returns>
public static IDataReader ExecuteReader(this IDbCommand self, bool writeLog, string cmd, Dictionary<string, object?> values)
public static IDataReader ExecuteReader(this IDbCommand self, bool writeLog, string? cmd, Dictionary<string, object?> values)
{
if (cmd != null)
self.SetCommandAndParameters(cmd);
Duplicati/Library/Main/Database/LocalBackupDatabase.cs
+273 -220
View File
@@ -18,6 +18,8 @@
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Collections.Generic;
using System.Data;
@@ -43,7 +45,6 @@ namespace Duplicati.Library.Main.Database
private readonly IDbCommand m_insertfileCommand;
private readonly IDbCommand m_insertblocksetCommand;
private readonly IDbCommand m_insertblocksetentryFastCommand;
private readonly IDbCommand m_insertblocksetentryCommand;
private readonly IDbCommand m_insertblocklistHashesCommand;
@@ -53,7 +54,6 @@ namespace Duplicati.Library.Main.Database
private readonly IDbCommand m_selectfilelastmodifiedCommand;
private readonly IDbCommand m_selectfilelastmodifiedWithSizeCommand;
private readonly IDbCommand m_selectfileHashCommand;
private readonly IDbCommand m_selectblocklistHashesCommand;
private readonly IDbCommand m_insertfileOperationCommand;
private readonly IDbCommand m_selectfilemetadatahashandsizeCommand;
@@ -76,41 +76,20 @@ namespace Duplicati.Library.Main.Database
{
m_logQueries = options.ProfileAllDatabaseQueries;
m_findblockCommand = m_connection.CreateCommand();
m_insertblockCommand = m_connection.CreateCommand();
m_insertfileCommand = m_connection.CreateCommand();
m_insertblocksetCommand = m_connection.CreateCommand();
m_insertmetadatasetCommand = m_connection.CreateCommand();
m_findblocksetCommand = m_connection.CreateCommand();
m_findmetadatasetCommand = m_connection.CreateCommand();
m_findfilesetCommand = m_connection.CreateCommand();
m_insertblocksetentryCommand = m_connection.CreateCommand();
m_insertblocklistHashesCommand = m_connection.CreateCommand();
m_selectblocklistHashesCommand = m_connection.CreateCommand();
m_insertfileOperationCommand = m_connection.CreateCommand();
m_findfileCommand = m_connection.CreateCommand();
m_selectfilelastmodifiedCommand = m_connection.CreateCommand();
m_selectfilelastmodifiedWithSizeCommand = m_connection.CreateCommand();
m_selectfileHashCommand = m_connection.CreateCommand();
m_insertblocksetentryFastCommand = m_connection.CreateCommand();
m_selectfilemetadatahashandsizeCommand = m_connection.CreateCommand();
m_getfirstfilesetwithblockinblockset = m_connection.CreateCommand();
m_findblockCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?");
m_findblocksetCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Fullhash"" = ? AND ""Length"" = ?");
m_findmetadatasetCommand = m_connection.CreateCommand(@"SELECT ""A"".""ID"" FROM ""Metadataset"" A, ""BlocksetEntry"" B, ""Block"" C WHERE ""A"".""BlocksetID"" = ""B"".""BlocksetID"" AND ""B"".""BlockID"" = ""C"".""ID"" AND ""C"".""Hash"" = ? AND ""C"".""Size"" = ?");
m_findfilesetCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""FileLookup"" WHERE ""BlocksetID"" = ? AND ""MetadataID"" = ? AND ""Path"" = ? AND ""PrefixID"" = ?");
m_insertblockCommand = m_connection.CreateCommand(@"INSERT INTO ""Block"" (""Hash"", ""VolumeID"", ""Size"") VALUES (?, ?, ?); SELECT last_insert_rowid();");
m_insertfileOperationCommand = m_connection.CreateCommand(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (?, ?, ?)");
m_insertfileCommand = m_connection.CreateCommand(@"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"",""BlocksetID"", ""MetadataID"") VALUES (?, ?, ? ,?); SELECT last_insert_rowid();");
m_insertblocksetCommand = m_connection.CreateCommand(@"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (?, ?); SELECT last_insert_rowid();");
m_insertblocksetentryFastCommand = m_connection.CreateCommand(@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") VALUES (?,?,?)");
m_insertblocksetentryCommand = m_connection.CreateCommand(@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") SELECT ? AS A, ? AS B, ""ID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?");
m_insertblocklistHashesCommand = m_connection.CreateCommand(@"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (?, ?, ?)");
m_insertmetadatasetCommand = m_connection.CreateCommand(@"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (?); SELECT last_insert_rowid();");
m_selectfilelastmodifiedCommand = m_connection.CreateCommand(@"SELECT ""A"".""ID"", ""B"".""LastModified"" FROM (SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = ? AND ""Path"" = ?) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = ?");
m_selectfilelastmodifiedWithSizeCommand = m_connection.CreateCommand(@"SELECT ""C"".""ID"", ""C"".""LastModified"", ""D"".""Length"" FROM (SELECT ""A"".""ID"", ""B"".""LastModified"", ""A"".""BlocksetID"" FROM (SELECT ""ID"", ""BlocksetID"" FROM ""FileLookup"" WHERE ""PrefixID"" = ? AND ""Path"" = ?) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = ?) AS ""C"", ""Blockset"" AS ""D"" WHERE ""C"".""BlocksetID"" == ""D"".""ID"" ");
m_selectfilemetadatahashandsizeCommand = m_connection.CreateCommand(@"SELECT ""Blockset"".""Length"", ""Blockset"".""FullHash"" FROM ""Blockset"", ""Metadataset"", ""File"" WHERE ""File"".""ID"" = ? AND ""Blockset"".""ID"" = ""Metadataset"".""BlocksetID"" AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" ");
m_findblockCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @hash AND ""Size"" = @Size");
m_findblocksetCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Fullhash"" = @Fullhash AND ""Length"" = @Length");
m_findmetadatasetCommand = m_connection.CreateCommand(@"SELECT ""A"".""ID"" FROM ""Metadataset"" A, ""BlocksetEntry"" B, ""Block"" C WHERE ""A"".""BlocksetID"" = ""B"".""BlocksetID"" AND ""B"".""BlockID"" = ""C"".""ID"" AND ""C"".""Hash"" = @Hash AND ""C"".""Size"" = @Size");
m_findfilesetCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""FileLookup"" WHERE ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId AND ""Path"" = @Path AND ""PrefixID"" = @PrefixId");
m_insertblockCommand = m_connection.CreateCommand(@"INSERT INTO ""Block"" (""Hash"", ""VolumeID"", ""Size"") VALUES (@Hash, @VolumeId, @Size); SELECT last_insert_rowid();");
m_insertfileOperationCommand = m_connection.CreateCommand(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (@FilesetId, @FileId, @LastModified)");
m_insertfileCommand = m_connection.CreateCommand(@"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"",""BlocksetID"", ""MetadataID"") VALUES (@PrefixId, @Path, @BlocksetId, @MetadataId); SELECT last_insert_rowid();");
m_insertblocksetCommand = m_connection.CreateCommand(@"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (@Length, @Fullhash); SELECT last_insert_rowid();");
m_insertblocksetentryCommand = m_connection.CreateCommand(@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") SELECT @BlocksetId AS A, @Index AS B, ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size");
m_insertblocklistHashesCommand = m_connection.CreateCommand(@"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (@BlocksetId, @Index, @Hash)");
m_insertmetadatasetCommand = m_connection.CreateCommand(@"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (@BlocksetId); SELECT last_insert_rowid();");
m_selectfilelastmodifiedCommand = m_connection.CreateCommand(@"SELECT ""A"".""ID"", ""B"".""LastModified"" FROM (SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = @FilesetId");
m_selectfilelastmodifiedWithSizeCommand = m_connection.CreateCommand(@"SELECT ""C"".""ID"", ""C"".""LastModified"", ""D"".""Length"" FROM (SELECT ""A"".""ID"", ""B"".""LastModified"", ""A"".""BlocksetID"" FROM (SELECT ""ID"", ""BlocksetID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = @FilesetId) AS ""C"", ""Blockset"" AS ""D"" WHERE ""C"".""BlocksetID"" == ""D"".""ID"" ");
m_selectfilemetadatahashandsizeCommand = m_connection.CreateCommand(@"SELECT ""Blockset"".""Length"", ""Blockset"".""FullHash"" FROM ""Blockset"", ""Metadataset"", ""File"" WHERE ""File"".""ID"" = @FileId AND ""Blockset"".""ID"" = ""Metadataset"".""BlocksetID"" AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" ");
// Allow users to test on real-world data
// to get feedback on potential performance
@@ -137,11 +116,11 @@ namespace Duplicati.Library.Main.Database
findQuery =
@" SELECT ""FileLookup"".""ID"" AS ""FileID"", ""FilesetEntry"".""Lastmodified"", ""FileBlockset"".""Length"", ""MetaBlockset"".""Fullhash"" AS ""Metahash"", ""MetaBlockset"".""Length"" AS ""Metasize"" " +
@" FROM ""FileLookup"", ""FilesetEntry"", ""Fileset"", ""Blockset"" ""FileBlockset"", ""Metadataset"", ""Blockset"" ""MetaBlockset"" " +
@" WHERE ""FileLookup"".""PrefixID"" = ? AND ""FileLookup"".""Path"" = ? " +
@" WHERE ""FileLookup"".""PrefixID"" = @PrefixId AND ""FileLookup"".""Path"" = @Path " +
@" AND ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" " +
@" AND ""FileBlockset"".""ID"" = ""FileLookup"".""BlocksetID"" " +
@" AND ""Metadataset"".""ID"" = ""FileLookup"".""MetadataID"" AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"" " +
@" AND ? IS NOT NULL" +
@" AND @FilesetId IS NOT NULL" +
@" ORDER BY ""Fileset"".""Timestamp"" DESC " +
@" LIMIT 1 ";
break;
@@ -151,10 +130,10 @@ namespace Duplicati.Library.Main.Database
case 2:
var getLastFileEntryForPath =
@"SELECT ""A"".""ID"", ""B"".""LastModified"", ""A"".""BlocksetID"", ""A"".""MetadataID"" " +
@" FROM (SELECT ""ID"", ""BlocksetID"", ""MetadataID"" FROM ""FileLookup"" WHERE ""PrefixID"" = ? AND ""Path"" = ?) ""A"" " +
@" FROM (SELECT ""ID"", ""BlocksetID"", ""MetadataID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path) ""A"" " +
@" CROSS JOIN ""FilesetEntry"" ""B"" " +
@" WHERE ""A"".""ID"" = ""B"".""FileID"" " +
@" AND ""B"".""FilesetID"" = ? ";
@" AND ""B"".""FilesetID"" = @FilesetId ";
findQuery = FormatInvariant($@"SELECT ""C"".""ID"" AS ""FileID"", ""C"".""LastModified"", ""D"".""Length"", ""E"".""FullHash"" as ""Metahash"", ""E"".""Length"" AS ""Metasize""
FROM ({getLastFileEntryForPath}) AS ""C"", ""Blockset"" AS ""D"", ""Blockset"" AS ""E"", ""Metadataset"" ""F""
@@ -173,7 +152,7 @@ LIMIT 1");
@"INNER JOIN Metadataset ON (Metadataset.ID = FileLookup.MetadataID) " +
@"INNER JOIN Blockset AS MetaBlockset ON (MetaBlockset.ID = Metadataset.BlocksetID) " +
@" LEFT JOIN Blockset AS FileBlockset ON (FileBlockset.ID = FileLookup.BlocksetID) " +
@" WHERE FileLookup.PrefixID = ? AND FileLookup.Path = ? AND FilesetID = ? " +
@" WHERE FileLookup.PrefixID = @PrefixId AND FileLookup.Path = @Path AND FilesetID = @FilesetId " +
@" LIMIT 1 ";
break;
@@ -182,8 +161,8 @@ LIMIT 1");
findQuery =
@" SELECT ""FileLookup"".""ID"" AS ""FileID"", ""FilesetEntry"".""Lastmodified"", ""FileBlockset"".""Length"", ""MetaBlockset"".""Fullhash"" AS ""Metahash"", ""MetaBlockset"".""Length"" AS ""Metasize"" " +
@" FROM ""FileLookup"", ""FilesetEntry"", ""Fileset"", ""Blockset"" ""FileBlockset"", ""Metadataset"", ""Blockset"" ""MetaBlockset"" " +
@" WHERE ""FileLookup"".""PrefixID"" = ? AND ""FileLookup"".""Path"" = ? " +
@" AND ""Fileset"".""ID"" = ? " +
@" WHERE ""FileLookup"".""PrefixID"" = @PrefixId AND ""FileLookup"".""Path"" = @Path " +
@" AND ""Fileset"".""ID"" = @FilesetId " +
@" AND ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" " +
@" AND ""FileBlockset"".""ID"" = ""FileLookup"".""BlocksetID"" " +
@" AND ""Metadataset"".""ID"" = ""FileLookup"".""MetadataID"" AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"" " +
@@ -194,11 +173,10 @@ LIMIT 1");
m_findfileCommand = m_connection.CreateCommand(findQuery);
m_selectfileHashCommand = m_connection.CreateCommand(@"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"", ""FileLookup"" WHERE ""Blockset"".""ID"" = ""FileLookup"".""BlocksetID"" AND ""FileLookup"".""ID"" = ? ");
m_selectblocklistHashesCommand = m_connection.CreateCommand(@"SELECT ""Hash"" FROM ""BlocklistHash"" WHERE ""BlocksetID"" = ? ORDER BY ""Index"" ASC ");
m_selectfileHashCommand = m_connection.CreateCommand(@"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"", ""FileLookup"" WHERE ""Blockset"".""ID"" = ""FileLookup"".""BlocksetID"" AND ""FileLookup"".""ID"" = @FileId ");
m_getfirstfilesetwithblockinblockset = m_connection.CreateCommand(@"SELECT MIN(""FilesetEntry"".""FilesetID"") FROM ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" IN (
SELECT ""File"".""ID"" FROM ""File"" WHERE ""File"".""BlocksetID"" IN(
SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""BlocklistHash"".""Hash"" = ?))");
SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""BlocklistHash"".""Hash"" = @Hash))");
m_blocklistHashes = new HashSet<string>();
}
@@ -209,12 +187,12 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
/// <param name="key">The block key</param>
/// <param name="size">The size of the block</param>
/// <returns>True if the block should be added to the current output</returns>
public long FindBlockID(string key, long size, IDbTransaction transaction = null)
public long FindBlockID(string key, long size, IDbTransaction? transaction = null)
{
m_findblockCommand.Transaction = transaction;
m_findblockCommand.SetParameterValue(0, key);
m_findblockCommand.SetParameterValue(1, size);
return m_findblockCommand.ExecuteScalarInt64(m_logQueries, -1);
return m_findblockCommand.SetTransaction(transaction)
.SetParameterValue("@Hash", key)
.SetParameterValue("@Size", size)
.ExecuteScalarInt64(m_logQueries, -1);
}
/// <summary>
@@ -223,20 +201,18 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
/// <param name="key">The block key</param>
/// <param name="size">The size of the block</param>
/// <returns>True if the block should be added to the current output</returns>
public bool AddBlock(string key, long size, long volumeid, IDbTransaction transaction = null)
public bool AddBlock(string key, long size, long volumeid, IDbTransaction? transaction = null)
{
m_findblockCommand.Transaction = transaction;
m_findblockCommand.SetParameterValue(0, key);
m_findblockCommand.SetParameterValue(1, size);
var r = m_findblockCommand.ExecuteScalarInt64(m_logQueries, -1);
var r = FindBlockID(key, size, transaction);
if (r == -1L)
{
m_insertblockCommand.Transaction = transaction;
m_insertblockCommand.SetParameterValue(0, key);
m_insertblockCommand.SetParameterValue(1, volumeid);
m_insertblockCommand.SetParameterValue(2, size);
m_insertblockCommand.ExecuteScalarInt64(m_logQueries);
var ins = m_insertblockCommand.SetTransaction(transaction)
.SetParameterValue("@Hash", key)
.SetParameterValue("@VolumeId", volumeid)
.SetParameterValue("@Size", size)
.ExecuteNonQuery(m_logQueries);
if (ins != 1)
throw new Exception($"Failed to insert block {key} with size {size}, result count: {ins}");
return true;
}
else
@@ -255,58 +231,64 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
/// <param name="hashes">The list of hashes</param>
/// <param name="blocksetid">The id of the blockset, new or old</param>
/// <returns>True if the blockset was created, false otherwise</returns>
public bool AddBlockset(string filehash, long size, int blocksize, IEnumerable<string> hashes, IEnumerable<string> blocklistHashes, out long blocksetid, IDbTransaction transaction = null)
public bool AddBlockset(string filehash, long size, int blocksize, IEnumerable<string> hashes, IEnumerable<string> blocklistHashes, out long blocksetid, IDbTransaction? transaction = null)
{
m_findblocksetCommand.Transaction = transaction;
blocksetid = m_findblocksetCommand.ExecuteScalarInt64(m_logQueries, null, -1, filehash, size);
blocksetid = m_findblocksetCommand.SetTransaction(transaction)
.SetParameterValue("@Fullhash", filehash)
.SetParameterValue("@Length", size)
.ExecuteScalarInt64(m_logQueries, null, -1);
if (blocksetid != -1)
return false; //Found it
using (var tr = new TemporaryTransactionWrapper(m_connection, transaction))
{
m_insertblocksetCommand.Transaction = tr.Parent;
m_insertblocksetCommand.SetParameterValue(0, size);
m_insertblocksetCommand.SetParameterValue(1, filehash);
blocksetid = m_insertblocksetCommand.ExecuteScalarInt64(m_logQueries);
blocksetid = m_insertblocksetCommand.SetTransaction(tr.Parent)
.SetParameterValue("@Length", size)
.SetParameterValue("@Fullhash", filehash)
.ExecuteScalarInt64(m_logQueries);
long ix = 0;
if (blocklistHashes != null)
{
m_insertblocklistHashesCommand.SetParameterValue(0, blocksetid);
m_insertblocklistHashesCommand.Transaction = tr.Parent;
m_insertblocklistHashesCommand.SetTransaction(tr.Parent)
.SetParameterValue("@BlocksetId", blocksetid);
foreach (var bh in blocklistHashes)
{
m_insertblocklistHashesCommand.SetParameterValue(1, ix);
m_insertblocklistHashesCommand.SetParameterValue(2, bh);
m_insertblocklistHashesCommand.ExecuteNonQuery(m_logQueries);
var c = m_insertblocklistHashesCommand.SetParameterValue("@Index", ix)
.SetParameterValue("@Hash", bh)
.ExecuteNonQuery(m_logQueries);
if (c != 1)
throw new Exception($"Failed to insert blocklist hash {bh} for blockset {blocksetid}, result count: {c}");
ix++;
}
}
m_insertblocksetentryCommand.SetParameterValue(0, blocksetid);
m_insertblocksetentryCommand.Transaction = tr.Parent;
m_insertblocksetentryFastCommand.SetParameterValue(0, blocksetid);
m_insertblocksetentryFastCommand.Transaction = tr.Parent;
m_insertblocksetentryCommand.SetTransaction(tr.Parent)
.SetParameterValue("@BlocksetId", blocksetid);
ix = 0;
long remainsize = size;
foreach (var h in hashes)
{
var exsize = remainsize < blocksize ? remainsize : blocksize;
m_insertblocksetentryCommand.SetParameterValue(1, ix);
m_insertblocksetentryCommand.SetParameterValue(2, h);
m_insertblocksetentryCommand.SetParameterValue(3, exsize);
var c = m_insertblocksetentryCommand.ExecuteNonQuery(m_logQueries);
var c = m_insertblocksetentryCommand.SetParameterValue("@Index", ix)
.SetParameterValue("@Hash", h)
.SetParameterValue("@Size", exsize)
.ExecuteNonQuery(m_logQueries);
if (c != 1)
{
Logging.Log.WriteErrorMessage(LOGTAG, "CheckingErrorsForIssue1400", null, "Checking errors, related to #1400. Unexpected result count: {0}, expected {1}, hash: {2}, size: {3}, blocksetid: {4}, ix: {5}, fullhash: {6}, fullsize: {7}", c, 1, h, exsize, blocksetid, ix, filehash, size);
using (var cmd = m_connection.CreateCommand(tr.Parent))
{
var bid = cmd.ExecuteScalarInt64(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = ?", -1, h);
var bid = cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash")
.SetParameterValue("@Hash", h)
.ExecuteScalarInt64(-1);
if (bid == -1)
throw new Exception($"Could not find any blocks with the given hash: {h}");
foreach (var rd in cmd.ExecuteReaderEnumerable(@"SELECT ""Size"" FROM ""Block"" WHERE ""Hash"" = ?", h))
cmd.SetCommandAndParameters(@"SELECT ""Size"" FROM ""Block"" WHERE ""Hash"" = @Hash")
.SetParameterValue("@Hash", h);
foreach (var rd in cmd.ExecuteReaderEnumerable())
Logging.Log.WriteErrorMessage(LOGTAG, "FoundIssue1400Error", null, "Found block with ID {0} and hash {1} and size {2}", bid, h, rd.ConvertValueToInt64(0, -1));
}
@@ -331,12 +313,14 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
/// <param name="size">The size of the metadata.</param>
/// <param name="metadataid">The ID of the metadataset.</param>
/// <param name="transaction">An optional transaction.</param>
public bool GetMetadatasetID(string filehash, long size, out long metadataid, IDbTransaction transaction = null)
public bool GetMetadatasetID(string filehash, long size, out long metadataid, IDbTransaction? transaction = null)
{
if (size > 0)
{
m_findmetadatasetCommand.Transaction = transaction;
metadataid = m_findmetadatasetCommand.ExecuteScalarInt64(m_logQueries, null, -1, filehash, size);
metadataid = m_findmetadatasetCommand.SetTransaction(transaction)
.SetParameterValue("@Hash", filehash)
.SetParameterValue("@Size", size)
.ExecuteScalarInt64(m_logQueries, null, -1);
return metadataid != -1;
}
@@ -353,16 +337,16 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
/// <param name="blocksetid">The id of the blockset to add</param>
/// <param name="metadataid">The id of the metadata set</param>
/// <returns>True if the set was added to the database, false otherwise</returns>
public bool AddMetadataset(string filehash, long size, long blocksetid, out long metadataid, IDbTransaction transaction = null)
public bool AddMetadataset(string filehash, long size, long blocksetid, out long metadataid, IDbTransaction? transaction = null)
{
if (GetMetadatasetID(filehash, size, out metadataid, transaction))
return false;
using (var tr = new TemporaryTransactionWrapper(m_connection, transaction))
{
m_insertmetadatasetCommand.Transaction = tr.Parent;
m_insertmetadatasetCommand.SetParameterValue(0, blocksetid);
metadataid = m_insertmetadatasetCommand.ExecuteScalarInt64(m_logQueries);
metadataid = m_insertmetadatasetCommand.SetTransaction(tr.Parent)
.SetParameterValue("@BlocksetId", blocksetid)
.ExecuteScalarInt64(m_logQueries);
tr.Commit();
return true;
}
@@ -377,35 +361,30 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
/// <param name="blocksetID">The ID of the hashkey for the file</param>
/// <param name="metadataID">The ID for the metadata</param>
/// <param name="transaction">The transaction to use for insertion, or null for no transaction</param>
public void AddFile(long pathprefixid, string filename, DateTime lastmodified, long blocksetID, long metadataID, IDbTransaction transaction)
public void AddFile(long pathprefixid, string filename, DateTime lastmodified, long blocksetID, long metadataID, IDbTransaction? transaction)
{
var fileidobj = -1L;
m_findfilesetCommand.Transaction = transaction;
m_findfilesetCommand.SetParameterValue(0, blocksetID);
m_findfilesetCommand.SetParameterValue(1, metadataID);
m_findfilesetCommand.SetParameterValue(2, filename);
m_findfilesetCommand.SetParameterValue(3, pathprefixid);
fileidobj = m_findfilesetCommand.ExecuteScalarInt64(m_logQueries);
var fileidobj = m_findfilesetCommand.SetTransaction(transaction)
.SetParameterValue("@BlocksetId", blocksetID)
.SetParameterValue("@MetadataId", metadataID)
.SetParameterValue("@Path", filename)
.SetParameterValue("@PrefixId", pathprefixid)
.ExecuteScalarInt64(m_logQueries);
if (fileidobj == -1)
{
using (var tr = new TemporaryTransactionWrapper(m_connection, transaction))
{
m_insertfileCommand.Transaction = tr.Parent;
m_insertfileCommand.SetParameterValue(0, pathprefixid);
m_insertfileCommand.SetParameterValue(1, filename);
m_insertfileCommand.SetParameterValue(2, blocksetID);
m_insertfileCommand.SetParameterValue(3, metadataID);
fileidobj = m_insertfileCommand.ExecuteScalarInt64(m_logQueries);
fileidobj = m_insertfileCommand.SetTransaction(tr.Parent)
.SetParameterValue("@PrefixId", pathprefixid)
.SetParameterValue("@Path", filename)
.SetParameterValue("@BlocksetId", blocksetID)
.SetParameterValue("@MetadataId", metadataID)
.ExecuteScalarInt64(m_logQueries);
tr.Commit();
}
}
m_insertfileOperationCommand.Transaction = transaction;
m_insertfileOperationCommand.SetParameterValue(0, m_filesetId);
m_insertfileOperationCommand.SetParameterValue(1, fileidobj);
m_insertfileOperationCommand.SetParameterValue(2, lastmodified.ToUniversalTime().Ticks);
m_insertfileOperationCommand.ExecuteNonQuery(m_logQueries);
AddKnownFile(fileidobj, lastmodified, transaction);
}
/// <summary>
@@ -416,39 +395,45 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
/// <param name="blocksetID">The ID of the hashkey for the file</param>
/// <param name="metadataID">The ID for the metadata</param>
/// <param name="transaction">The transaction to use for insertion, or null for no transaction</param>
public void AddFile(string filename, DateTime lastmodified, long blocksetID, long metadataID, IDbTransaction transaction)
public void AddFile(string filename, DateTime lastmodified, long blocksetID, long metadataID, IDbTransaction? transaction)
{
var split = SplitIntoPrefixAndName(filename);
AddFile(GetOrCreatePathPrefix(split.Key, transaction), split.Value, lastmodified, blocksetID, metadataID, transaction);
}
public void AddUnmodifiedFile(long fileid, DateTime lastmodified, IDbTransaction transaction = null)
/// <summary>
/// Adds a known file to the fileset
/// </summary>
/// <param name="fileid">Id of the file</param>
/// <param name="lastmodified">The time the file was modified</param>
/// <param name="transaction">The transaction to use for insertion, or null for no transaction</param>
public void AddKnownFile(long fileid, DateTime lastmodified, IDbTransaction? transaction = null)
{
m_insertfileOperationCommand.Transaction = transaction;
m_insertfileOperationCommand.SetParameterValue(0, m_filesetId);
m_insertfileOperationCommand.SetParameterValue(1, fileid);
m_insertfileOperationCommand.SetParameterValue(2, lastmodified.ToUniversalTime().Ticks);
m_insertfileOperationCommand.ExecuteNonQuery(m_logQueries);
m_insertfileOperationCommand.SetTransaction(transaction)
.SetParameterValue("@FilesetId", m_filesetId)
.SetParameterValue("@FileId", fileid)
.SetParameterValue("@LastModified", lastmodified.ToUniversalTime().Ticks)
.ExecuteNonQuery(m_logQueries);
}
public void AddDirectoryEntry(string path, long metadataID, DateTime lastmodified, IDbTransaction transaction = null)
public void AddDirectoryEntry(string path, long metadataID, DateTime lastmodified, IDbTransaction? transaction = null)
{
AddFile(path, lastmodified, FOLDER_BLOCKSET_ID, metadataID, transaction);
}
public void AddSymlinkEntry(string path, long metadataID, DateTime lastmodified, IDbTransaction transaction = null)
public void AddSymlinkEntry(string path, long metadataID, DateTime lastmodified, IDbTransaction? transaction = null)
{
AddFile(path, lastmodified, SYMLINK_BLOCKSET_ID, metadataID, transaction);
}
public long GetFileLastModified(long prefixid, string path, long filesetid, bool includeLength, out DateTime oldModified, out long length, IDbTransaction transaction = null)
public long GetFileLastModified(long prefixid, string path, long filesetid, bool includeLength, out DateTime oldModified, out long length, IDbTransaction? transaction = null)
{
if (includeLength)
{
m_selectfilelastmodifiedWithSizeCommand.Transaction = transaction;
m_selectfilelastmodifiedWithSizeCommand.SetParameterValue(0, prefixid);
m_selectfilelastmodifiedWithSizeCommand.SetParameterValue(1, path);
m_selectfilelastmodifiedWithSizeCommand.SetParameterValue(2, filesetid);
m_selectfilelastmodifiedWithSizeCommand.SetTransaction(transaction)
.SetParameterValue("@PrefixId", prefixid)
.SetParameterValue("@Path", path)
.SetParameterValue("@FilesetId", filesetid);
using (var rd = m_selectfilelastmodifiedWithSizeCommand.ExecuteReader(m_logQueries, null))
if (rd.Read())
{
@@ -459,10 +444,10 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
}
else
{
m_selectfilelastmodifiedCommand.Transaction = transaction;
m_selectfilelastmodifiedCommand.SetParameterValue(0, prefixid);
m_selectfilelastmodifiedCommand.SetParameterValue(1, path);
m_selectfilelastmodifiedCommand.SetParameterValue(2, filesetid);
m_selectfilelastmodifiedCommand.SetTransaction(transaction)
.SetParameterValue("@PrefixId", prefixid)
.SetParameterValue("@Path", path)
.SetParameterValue("@FilesetId", filesetid);
using (var rd = m_selectfilelastmodifiedCommand.ExecuteReader(m_logQueries, null))
if (rd.Read())
{
@@ -478,12 +463,12 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
}
public long GetFileEntry(long prefixid, string path, long filesetid, out DateTime oldModified, out long lastFileSize, out string oldMetahash, out long oldMetasize, IDbTransaction transaction)
public long GetFileEntry(long prefixid, string path, long filesetid, out DateTime oldModified, out long lastFileSize, out string? oldMetahash, out long oldMetasize, IDbTransaction transaction)
{
m_findfileCommand.SetParameterValue(0, prefixid);
m_findfileCommand.SetParameterValue(1, path);
m_findfileCommand.SetParameterValue(2, filesetid);
m_findfileCommand.Transaction = transaction;
m_findfileCommand.SetTransaction(transaction)
.SetParameterValue("@PrefixId", prefixid)
.SetParameterValue("@Path", path)
.SetParameterValue("@FilesetId", filesetid);
using (var rd = m_findfileCommand.ExecuteReader())
if (rd.Read())
@@ -504,24 +489,24 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
}
}
public Tuple<long, string> GetMetadataHashAndSizeForFile(long fileid, IDbTransaction transaction)
public (long Size, string MetadataHash)? GetMetadataHashAndSizeForFile(long fileid, IDbTransaction transaction)
{
m_selectfilemetadatahashandsizeCommand.SetParameterValue(0, fileid);
m_selectfilemetadatahashandsizeCommand.Transaction = transaction;
m_selectfilemetadatahashandsizeCommand.SetTransaction(transaction)
.SetParameterValue("@FileId", fileid);
using (var rd = m_findfileCommand.ExecuteReader())
using (var rd = m_selectfilemetadatahashandsizeCommand.ExecuteReader())
if (rd.Read())
return new Tuple<long, string>(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1));
return (rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? throw new InvalidOperationException("Metadata hash is null"));
return null;
}
public string GetFileHash(long fileid, IDbTransaction transaction)
public string? GetFileHash(long fileid, IDbTransaction transaction)
{
m_selectfileHashCommand.SetParameterValue(0, fileid);
m_selectfileHashCommand.Transaction = transaction;
var r = m_selectfileHashCommand.ExecuteScalar(m_logQueries, null);
var r = m_selectfileHashCommand.SetTransaction(transaction)
.SetParameterValue("@FileId", fileid)
.ExecuteScalar(m_logQueries, null);
if (r == null || r == DBNull.Value)
return null;
@@ -535,13 +520,16 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
private long GetPreviousFilesetID(IDbCommand cmd)
{
return GetPreviousFilesetID(cmd, OperationTimestamp, m_filesetId);
return GetPreviousFilesetID(cmd, OperationTimestamp, m_filesetId, cmd.Transaction);
}
private long GetPreviousFilesetID(IDbCommand cmd, DateTime timestamp, long filesetid)
private long GetPreviousFilesetID(IDbCommand cmd, DateTime timestamp, long filesetid, IDbTransaction? transaction)
{
var lastFilesetId = cmd.ExecuteScalarInt64(@"SELECT ""ID"" FROM ""Fileset"" WHERE ""Timestamp"" < ? AND ""ID"" != ? ORDER BY ""Timestamp"" DESC ", -1, Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(timestamp), filesetid);
return lastFilesetId;
return cmd.SetTransaction(transaction)
.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Fileset"" WHERE ""Timestamp"" < @Timestamp AND ""ID"" != @FilesetId ORDER BY ""Timestamp"" DESC ")
.SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(timestamp))
.SetParameterValue("@FilesetId", filesetid)
.ExecuteScalarInt64(-1);
}
internal Tuple<long, long> GetLastBackupFileCountAndSize()
@@ -549,8 +537,17 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
using (var cmd = m_connection.CreateCommand())
{
var lastFilesetId = cmd.ExecuteScalarInt64(@"SELECT ""ID"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC LIMIT 1");
var count = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""FileLookup"" INNER JOIN ""FilesetEntry"" ON ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""FileLookup"".""BlocksetID"" NOT IN (?, ?)", -1, lastFilesetId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID);
var size = cmd.ExecuteScalarInt64(@"SELECT SUM(""Blockset"".""Length"") FROM ""FileLookup"", ""FilesetEntry"", ""Blockset"" WHERE ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""FileLookup"".""BlocksetID"" NOT IN (?, ?)", -1, lastFilesetId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID);
var count = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""FileLookup"" INNER JOIN ""FilesetEntry"" ON ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""FileLookup"".""BlocksetID"" NOT IN (@FolderBlocksetId, @SymlinkBlocksetId)")
.SetParameterValue("@FilesetId", lastFilesetId)
.SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID)
.SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID)
.ExecuteScalarInt64(-1);
var size = cmd.SetCommandAndParameters(@"SELECT SUM(""Blockset"".""Length"") FROM ""FileLookup"", ""FilesetEntry"", ""Blockset"" WHERE ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""FileLookup"".""BlocksetID"" NOT IN (@FolderBlocksetId, @SymlinkBlocksetId)")
.SetParameterValue("@FilesetId", lastFilesetId)
.SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID)
.SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID)
.ExecuteScalarInt64(-1);
return new Tuple<long, long>(count, size);
}
@@ -562,32 +559,68 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
{
// TODO: Optimize these queries to not use the "File" view
var lastFilesetId = GetPreviousFilesetID(cmd);
results.AddedFolders = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" = ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)", 0, m_filesetId, FOLDER_BLOCKSET_ID, lastFilesetId);
results.AddedSymlinks = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" = ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)", 0, m_filesetId, SYMLINK_BLOCKSET_ID, lastFilesetId);
results.AddedFolders = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @CurrentFilesetId AND ""File"".""BlocksetID"" = @BlocksetId AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @LastFilsetId)")
.SetParameterValue("@CurrentFilesetId", m_filesetId)
.SetParameterValue("@BlocksetId", FOLDER_BLOCKSET_ID)
.SetParameterValue("@LastFilsetId", lastFilesetId)
.ExecuteScalarInt64(0);
results.AddedSymlinks = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @CurrentFilesetId AND ""File"".""BlocksetID"" = @BlocksetId AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @LastFilsetId)")
.SetParameterValue("@CurrentFilesetId", m_filesetId)
.SetParameterValue("@BlocksetId", SYMLINK_BLOCKSET_ID)
.SetParameterValue("@LastFilsetId", lastFilesetId)
.ExecuteScalarInt64(0);
results.DeletedFolders = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" = ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)", 0, lastFilesetId, FOLDER_BLOCKSET_ID, m_filesetId);
results.DeletedSymlinks = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" = ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)", 0, lastFilesetId, SYMLINK_BLOCKSET_ID, m_filesetId);
results.DeletedFolders = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @LastFilsetId AND ""File"".""BlocksetID"" = @BlocksetId AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @CurrentFilesetId)")
.SetParameterValue("@LastFilsetId", lastFilesetId)
.SetParameterValue("@BlocksetId", FOLDER_BLOCKSET_ID)
.SetParameterValue("@CurrentFilesetId", m_filesetId)
.ExecuteScalarInt64(0);
results.DeletedSymlinks = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @LastFilsetId AND ""File"".""BlocksetID"" = @BlocksetId AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @CurrentFilesetId)")
.SetParameterValue("@LastFilsetId", lastFilesetId)
.SetParameterValue("@BlocksetId", SYMLINK_BLOCKSET_ID)
.SetParameterValue("@CurrentFilesetId", m_filesetId)
.ExecuteScalarInt64(0);
var subqueryNonFiles = @"SELECT ""File"".""Path"", ""Blockset"".""Fullhash"" FROM ""File"", ""FilesetEntry"", ""Metadataset"", ""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" AND ""File"".""BlocksetID"" = ? AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FilesetID"" = ? ";
results.ModifiedFolders = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM (" + subqueryNonFiles + @") A, (" + subqueryNonFiles + @") B WHERE ""A"".""Path"" = ""B"".""Path"" AND ""A"".""Fullhash"" != ""B"".""Fullhash"" ", 0, lastFilesetId, FOLDER_BLOCKSET_ID, m_filesetId, FOLDER_BLOCKSET_ID);
results.ModifiedSymlinks = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM (" + subqueryNonFiles + @") A, (" + subqueryNonFiles + @") B WHERE ""A"".""Path"" = ""B"".""Path"" AND ""A"".""Fullhash"" != ""B"".""Fullhash"" ", 0, lastFilesetId, SYMLINK_BLOCKSET_ID, m_filesetId, SYMLINK_BLOCKSET_ID);
var subqueryNonFiles = @"SELECT ""File"".""Path"", ""Blockset"".""Fullhash"" FROM ""File"", ""FilesetEntry"", ""Metadataset"", ""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" AND ""File"".""BlocksetID"" = @BlocksetId AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FilesetID"" = ";
var subQueryNonFilesPrevious = subqueryNonFiles + "@LastFilesetId";
var subQueryNonFilesCurrent = subqueryNonFiles + "@CurrentFilesetId";
results.ModifiedFolders = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM (" + subQueryNonFilesPrevious + @") A, (" + subQueryNonFilesCurrent + @") B WHERE ""A"".""Path"" = ""B"".""Path"" AND ""A"".""Fullhash"" != ""B"".""Fullhash"" ")
.SetParameterValue("@LastFilesetId", lastFilesetId)
.SetParameterValue("@CurrentFilesetId", m_filesetId)
.SetParameterValue("@BlocksetId", FOLDER_BLOCKSET_ID)
.ExecuteScalarInt64(0);
results.ModifiedSymlinks = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM (" + subQueryNonFilesPrevious + @") A, (" + subQueryNonFilesCurrent + @") B WHERE ""A"".""Path"" = ""B"".""Path"" AND ""A"".""Fullhash"" != ""B"".""Fullhash"" ")
.SetParameterValue("@LastFilesetId", lastFilesetId)
.SetParameterValue("@CurrentFilesetId", m_filesetId)
.SetParameterValue("@BlocksetId", SYMLINK_BLOCKSET_ID)
.ExecuteScalarInt64(0);
var tmpName1 = "TmpFileList-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
var tmpName2 = "TmpFileList-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
try
{
var subqueryFiles = @"SELECT ""File"".""Path"" AS ""Path"", ""A"".""Fullhash"" AS ""Filehash"", ""B"".""Fullhash"" AS ""Metahash"" FROM ""File"", ""FilesetEntry"", ""Blockset"" A, ""Blockset"" B, ""Metadataset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""A"".""ID"" = ""File"".""BlocksetID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""B"".""ID"" ";
var subqueryFiles = @"SELECT ""File"".""Path"" AS ""Path"", ""A"".""Fullhash"" AS ""Filehash"", ""B"".""Fullhash"" AS ""Metahash"" FROM ""File"", ""FilesetEntry"", ""Blockset"" A, ""Blockset"" B, ""Metadataset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""A"".""ID"" = ""File"".""BlocksetID"" AND ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""File"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""B"".""ID"" ";
cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tmpName1}"" AS {subqueryFiles}"), lastFilesetId);
cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tmpName2}"" AS {subqueryFiles}"), m_filesetId);
cmd.SetCommandAndParameters(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tmpName1}"" AS {subqueryFiles}"))
.SetParameterValue("@FilesetId", lastFilesetId)
.ExecuteNonQuery();
cmd.SetCommandAndParameters(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tmpName2}"" AS {subqueryFiles}"))
.SetParameterValue("@FilesetId", m_filesetId)
.ExecuteNonQuery();
cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""nn_tmpName1"" ON ""{tmpName1}"" (""Path"")"));
cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""nn_tmpName2"" ON ""{tmpName2}"" (""Path"")"));
results.AddedFiles = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" != ? AND ""File"".""BlocksetID"" != ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""{tmpName1}"")"), 0, m_filesetId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID);
results.DeletedFiles = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""{tmpName1}"" WHERE ""{tmpName1}"".""Path"" NOT IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)"), 0, m_filesetId);
results.AddedFiles = cmd.SetCommandAndParameters(FormatInvariant($@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""File"".""BlocksetID"" != @FolderBlocksetId AND ""File"".""BlocksetID"" != @SymlinkBlocksetId AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""{tmpName1}"")"))
.SetParameterValue("@FilesetId", m_filesetId)
.SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID)
.SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID)
.ExecuteScalarInt64(0);
results.DeletedFiles = cmd.SetCommandAndParameters(FormatInvariant($@"SELECT COUNT(*) FROM ""{tmpName1}"" WHERE ""{tmpName1}"".""Path"" NOT IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId)"))
.SetParameterValue("@FilesetId", m_filesetId)
.ExecuteScalarInt64(0);
results.ModifiedFiles = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""{tmpName1}"" A, ""{tmpName2}"" B WHERE ""A"".""Path"" = ""B"".""Path"" AND (""A"".""Filehash"" != ""B"".""Filehash"" OR ""A"".""Metahash"" != ""B"".""Metahash"")"), 0);
}
finally
{
@@ -605,7 +638,7 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
/// </summary>
/// <param name="transaction">Transaction</param>
/// <param name="deleted">List of deleted paths, or null</param>
public void AppendFilesFromPreviousSet(IDbTransaction transaction, IEnumerable<string> deleted = null)
public void AppendFilesFromPreviousSet(IDbTransaction transaction, IEnumerable<string>? deleted = null)
{
AppendFilesFromPreviousSet(transaction, deleted, m_filesetId, -1, OperationTimestamp);
}
@@ -619,29 +652,28 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
/// <param name="filesetid">Current file-set ID</param>
/// <param name="prevId">Source file-set ID</param>
/// <param name="timestamp">If <c>filesetid</c> == -1, used to locate previous file-set</param>
public void AppendFilesFromPreviousSet(IDbTransaction transaction, IEnumerable<string> deleted, long filesetid, long prevId, DateTime timestamp)
public void AppendFilesFromPreviousSet(IDbTransaction transaction, IEnumerable<string>? deleted, long filesetid, long prevId, DateTime timestamp)
{
using (var cmd = m_connection.CreateCommand())
using (var cmdDelete = m_connection.CreateCommand())
using (var tr = new TemporaryTransactionWrapper(m_connection, transaction))
{
long lastFilesetId = prevId < 0 ? GetPreviousFilesetID(cmd, timestamp, filesetid) : prevId;
long lastFilesetId = prevId < 0 ? GetPreviousFilesetID(cmd, timestamp, filesetid, tr.Parent) : prevId;
cmd.Transaction = tr.Parent;
cmd.ExecuteNonQuery(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") SELECT ? AS ""FilesetID"", ""FileID"", ""Lastmodified"" FROM (SELECT DISTINCT ""FilesetID"", ""FileID"", ""Lastmodified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ? AND ""FileID"" NOT IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ?)) ", filesetid, lastFilesetId, filesetid);
cmd.SetTransaction(tr.Parent)
.SetCommandAndParameters(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") SELECT @CurrentFilesetId AS ""FilesetID"", ""FileID"", ""Lastmodified"" FROM (SELECT DISTINCT ""FilesetID"", ""FileID"", ""Lastmodified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @PreviousFilesetId AND ""FileID"" NOT IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @CurrentFilesetId)) ")
.SetParameterValue("@CurrentFilesetId", filesetid)
.SetParameterValue("@PreviousFilesetId", lastFilesetId)
.ExecuteNonQuery(m_logQueries);
if (deleted != null)
{
cmdDelete.Transaction = tr.Parent;
cmdDelete.CommandText = @"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = ? AND ""FileID"" IN (SELECT ""ID"" FROM ""File"" WHERE ""Path"" = ?) ";
cmdDelete.AddParameters(2);
cmdDelete.SetParameterValue(0, filesetid);
foreach (string s in deleted)
{
cmdDelete.SetParameterValue(1, s);
cmdDelete.ExecuteNonQuery();
}
using var tmplist = new TemporaryDbValueList(m_connection, tr.Parent, deleted);
cmdDelete.SetTransaction(tr.Parent)
.SetCommandAndParameters(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId AND ""FileID"" IN (SELECT ""ID"" FROM ""File"" WHERE ""Path"" IN (@Paths)) ")
.SetParameterValue("@FilesetId", filesetid)
.ExpandInClauseParameter("@Paths", tmplist)
.ExecuteNonQuery(m_logQueries);
}
tr.Commit();
@@ -683,18 +715,20 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
using (var cmdDelete = m_connection.CreateCommand())
using (var tr = new TemporaryTransactionWrapper(m_connection, transaction))
{
long lastFilesetId = prevFileSetId < 0 ? GetPreviousFilesetID(cmd, timestamp, fileSetId) : prevFileSetId;
long lastFilesetId = prevFileSetId < 0 ? GetPreviousFilesetID(cmd, timestamp, fileSetId, tr.Parent) : prevFileSetId;
// copy entries from previous file set into a temporary table, except those file IDs already added by the current backup
var tempFileSetTable = "FilesetEntry-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
cmd.Transaction = tr.Parent;
cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tempFileSetTable}"" AS SELECT ""FileID"", ""Lastmodified"" FROM (SELECT DISTINCT ""FilesetID"", ""FileID"", ""Lastmodified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ? AND ""FileID"" NOT IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ?))"), lastFilesetId, fileSetId);
cmd.SetCommandAndParameters(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tempFileSetTable}"" AS SELECT ""FileID"", ""Lastmodified"" FROM (SELECT DISTINCT ""FilesetID"", ""FileID"", ""Lastmodified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @PreviousFilesetId AND ""FileID"" NOT IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @CurrentFilesetId))"))
.SetParameterValue("@PreviousFilesetId", lastFilesetId)
.SetParameterValue("@CurrentFilesetId", fileSetId)
.ExecuteNonQuery();
// now we need to remove, from the above, any entries that were enumerated by the
// UNC-driven backup
cmdDelete.Transaction = tr.Parent;
cmdDelete.CommandText = FormatInvariant($@"DELETE FROM ""{tempFileSetTable}"" WHERE ""FileID"" = ?");
cmdDelete.AddParameters(1);
cmdDelete.SetTransaction(tr.Parent)
.SetCommandAndParameters(FormatInvariant($@"DELETE FROM ""{tempFileSetTable}"" WHERE ""FileID"" = @FileId"));
// enumerate files from new temporary file set, and remove any entries handled by UNC
cmd.Transaction = tr.Parent;
@@ -704,19 +738,19 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
LEFT JOIN ""File"" AS f ON fs.""FileID"" = f.""ID""
LEFT JOIN ""Blockset"" AS bs ON f.""BlocksetID"" = bs.""ID"";")))
{
var path = row.GetString(0);
var size = row.GetInt64(3);
var path = row.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for path");
var size = row.ConvertValueToInt64(3);
if (exclusionPredicate(path, size))
{
cmdDelete.SetParameterValue(0, row.GetInt64(1));
cmdDelete.ExecuteNonQuery();
}
cmdDelete.SetParameterValue("@FileId", row.ConvertValueToInt64(1))
.ExecuteNonQuery();
}
// now copy the temporary table into the FileSetEntry table
cmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"")
SELECT ?, ""FileID"", ""Lastmodified"" FROM ""{tempFileSetTable}"""), fileSetId);
cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"")
SELECT @FilesetId, ""FileID"", ""Lastmodified"" FROM ""{tempFileSetTable}"""))
.SetParameterValue("@FilesetId", fileSetId)
.ExecuteNonQuery();
tr.Commit();
}
@@ -728,12 +762,12 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
/// <param name="volumeid">The ID of the fileset volume to update</param>
/// <param name="timestamp">The timestamp of the operation to create</param>
/// <param name="transaction">An optional external transaction</param>
public override long CreateFileset(long volumeid, DateTime timestamp, IDbTransaction transaction = null)
public override long CreateFileset(long volumeid, DateTime timestamp, IDbTransaction? transaction = null)
{
return m_filesetId = base.CreateFileset(volumeid, timestamp, transaction);
}
public IEnumerable<string> GetTemporaryFilelistVolumeNames(bool latestOnly, IDbTransaction transaction = null)
public IEnumerable<string> GetTemporaryFilelistVolumeNames(bool latestOnly, IDbTransaction? transaction = null)
{
var incompleteFilesetIDs = GetIncompleteFilesets(transaction).OrderBy(x => x.Value).Select(x => x.Key).ToArray();
@@ -752,10 +786,14 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
public IEnumerable<string> GetMissingIndexFiles(IDbTransaction transaction)
{
using (var cmd = m_connection.CreateCommand(transaction))
using (var rd = cmd.ExecuteReader(@"SELECT ""Name"" FROM ""RemoteVolume"" WHERE ""Type"" = ? AND NOT ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"") AND ""State"" IN (?,?)", RemoteVolumeType.Blocks.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()))
using var cmd = m_connection.CreateCommand(transaction)
.SetCommandAndParameters(@"SELECT ""Name"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND NOT ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"") AND ""State"" IN (@States)")
.SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString())
.ExpandInClauseParameter("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]);
using (var rd = cmd.ExecuteReader())
while (rd.Read())
yield return rd.GetValue(0).ToString();
yield return rd.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for volume name");
}
public void MoveBlockToVolume(string blockkey, long size, long sourcevolumeid, long targetvolumeid, IDbTransaction transaction)
@@ -763,7 +801,12 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
using (var cmd = m_connection.CreateCommand())
{
cmd.Transaction = transaction;
var c = cmd.ExecuteNonQuery(@"UPDATE ""Block"" SET ""VolumeID"" = ? WHERE ""Hash"" = ? AND ""Size"" = ? AND ""VolumeID"" = ? ", targetvolumeid, blockkey, size, sourcevolumeid);
var c = cmd.SetCommandAndParameters(@"UPDATE ""Block"" SET ""VolumeID"" = @NewVolumeId WHERE ""Hash"" = @Hash AND ""Size"" = @Size AND ""VolumeID"" = @PreviousVolumeId ")
.SetParameterValue("@NewVolumeId", targetvolumeid)
.SetParameterValue("@Hash", blockkey)
.SetParameterValue("@Size", size)
.SetParameterValue("@PreviousVolumeId", sourcevolumeid)
.ExecuteNonQuery();
if (c != 1)
throw new Exception($"Failed to move block {blockkey}:{size} from volume {sourcevolumeid}, count: {c}");
}
@@ -775,7 +818,9 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
using (var cmd = m_connection.CreateCommand(transaction))
{
var c = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Block"" WHERE ""VolumeID"" = ? ", -1, volumeid);
var c = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""Block"" WHERE ""VolumeID"" = @VolumeId ")
.SetParameterValue("@VolumeId", volumeid)
.ExecuteScalarInt64(-1);
if (c != 0)
throw new Exception($"Failed to safe-delete volume {name}, blocks: {c}");
@@ -789,14 +834,15 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
using (var cmd = m_connection.CreateCommand(transaction))
{
// Grab the strings and return as array to avoid concurrent access to the IEnumerable
return cmd.ExecuteReaderEnumerable(
@"SELECT DISTINCT ""Block"".""Hash"" FROM ""Block"" WHERE ""Block"".""VolumeID"" = ? AND ""Block"".""Hash"" IN (SELECT ""Hash"" FROM ""BlocklistHash"")", volumeid)
.Select(x => x.ConvertValueToString(0))
cmd.SetCommandAndParameters(@"SELECT DISTINCT ""Block"".""Hash"" FROM ""Block"" WHERE ""Block"".""VolumeID"" = @VolumeId AND ""Block"".""Hash"" IN (SELECT ""Hash"" FROM ""BlocklistHash"")")
.SetParameterValue("@VolumeId", volumeid);
return cmd.ExecuteReaderEnumerable()
.Select(x => x.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for blocklist hash"))
.ToArray();
}
}
public string GetFirstPath()
public string? GetFirstPath()
{
using (var cmd = m_connection.CreateCommand())
{
@@ -817,11 +863,9 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
{
var data = new List<Interface.USNJournalDataEntry>();
using (var cmd = m_connection.CreateCommand())
using (var rd =
cmd.ExecuteReader(
@"SELECT ""VolumeName"", ""JournalID"", ""NextUSN"", ""ConfigHash"" FROM ""ChangeJournalData"" WHERE ""FilesetID"" = ?",
fileSetId))
using var cmd = m_connection.CreateCommand(@"SELECT ""VolumeName"", ""JournalID"", ""NextUSN"", ""ConfigHash"" FROM ""ChangeJournalData"" WHERE ""FilesetID"" = @FilesetId")
.SetParameterValue("@FilesetId", fileSetId);
using (var rd = cmd.ExecuteReader())
{
while (rd.Read())
{
@@ -844,7 +888,7 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
/// </summary>
/// <param name="data">Data to add</param>
/// <param name="transaction">An optional external transaction</param>
public void CreateChangeJournalData(IEnumerable<Interface.USNJournalDataEntry> data, IDbTransaction transaction = null)
public void CreateChangeJournalData(IEnumerable<Interface.USNJournalDataEntry> data, IDbTransaction? transaction = null)
{
using (var tr = new TemporaryTransactionWrapper(m_connection, transaction))
{
@@ -853,9 +897,14 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
using (var cmd = m_connection.CreateCommand())
{
cmd.Transaction = tr.Parent;
var c = cmd.ExecuteNonQuery(
@"INSERT INTO ""ChangeJournalData"" (""FilesetID"", ""VolumeName"", ""JournalID"", ""NextUSN"", ""ConfigHash"") VALUES (?, ?, ?, ?, ?);",
m_filesetId, entry.Volume, entry.JournalId, entry.NextUsn, entry.ConfigHash);
var c = cmd.SetCommandAndParameters(
@"INSERT INTO ""ChangeJournalData"" (""FilesetID"", ""VolumeName"", ""JournalID"", ""NextUSN"", ""ConfigHash"") VALUES (@FilesetId, @VolumeName, @JournalId, @NextUsn, @ConfigHash);")
.SetParameterValue("@FilesetId", m_filesetId)
.SetParameterValue("@VolumeName", entry.Volume)
.SetParameterValue("@JournalId", entry.JournalId)
.SetParameterValue("@NextUsn", entry.NextUsn)
.SetParameterValue("@ConfigHash", entry.ConfigHash)
.ExecuteNonQuery();
if (c != 1)
throw new Exception("Unable to add change journal entry");
@@ -872,7 +921,7 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
/// <param name="data">Data to add</param>
/// <param name="fileSetId">Existing file set to update</param>
/// <param name="transaction">An optional external transaction</param>
public void UpdateChangeJournalData(IEnumerable<Interface.USNJournalDataEntry> data, long fileSetId, IDbTransaction transaction = null)
public void UpdateChangeJournalData(IEnumerable<Interface.USNJournalDataEntry> data, long fileSetId, IDbTransaction? transaction = null)
{
using (var tr = new TemporaryTransactionWrapper(m_connection, transaction))
{
@@ -880,10 +929,14 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
{
using (var cmd = m_connection.CreateCommand())
{
cmd.Transaction = tr.Parent;
cmd.ExecuteNonQuery(
@"UPDATE ""ChangeJournalData"" SET ""NextUSN"" = ? WHERE ""FilesetID"" = ? AND ""VolumeName"" = ? AND ""JournalID"" = ?;",
entry.NextUsn, fileSetId, entry.Volume, entry.JournalId);
cmd.SetTransaction(tr.Parent)
.SetCommandAndParameters(
@"UPDATE ""ChangeJournalData"" SET ""NextUSN"" = @NextUsn WHERE ""FilesetID"" = @FilesetId AND ""VolumeName"" = @VolumeName AND ""JournalID"" = @JournalId;")
.SetParameterValue("@NextUsn", entry.NextUsn)
.SetParameterValue("@FilesetId", fileSetId)
.SetParameterValue("@VolumeName", entry.Volume)
.SetParameterValue("@JournalId", entry.JournalId)
.ExecuteNonQuery();
}
}
@@ -899,9 +952,9 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist
/// <returns>True if the hash is known, false otherwise</returns>
public bool IsBlocklistHashKnown(string hash, IDbTransaction transaction)
{
m_getfirstfilesetwithblockinblockset.Transaction = transaction;
m_getfirstfilesetwithblockinblockset.SetParameterValue(0, hash);
var res = m_getfirstfilesetwithblockinblockset.ExecuteScalarInt64();
var res = m_getfirstfilesetwithblockinblockset.SetTransaction(transaction)
.SetParameterValue("@Hash", hash)
.ExecuteScalarInt64();
if (res != -1 && res != m_filesetId)
return true;
else
Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs
+2 -2
View File
@@ -100,7 +100,7 @@ namespace Duplicati.Library.Main.Operation.Backup
return RunOnMain(() => m_database.AddSymlinkEntry(filename, metadataid, lastModified, GetTransaction()));
}
public Task<Tuple<long, string>> GetMetadataHashAndSizeForFileAsync(long fileid)
public Task<(long Size, string MetadataHash)?> GetMetadataHashAndSizeForFileAsync(long fileid)
{
return RunOnMain(() => m_database.GetMetadataHashAndSizeForFile(fileid, GetTransaction()));
}
@@ -154,7 +154,7 @@ namespace Duplicati.Library.Main.Operation.Backup
public Task AddUnmodifiedAsync(long fileid, DateTime lastModified)
{
return RunOnMain(() => m_database.AddUnmodifiedFile(fileid, lastModified, GetTransaction()));
return RunOnMain(() => m_database.AddKnownFile(fileid, lastModified, GetTransaction()));
}
Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs
+2 -2
View File
@@ -124,8 +124,8 @@ namespace Duplicati.Library.Main.Operation.Backup
var tp = await database.GetMetadataHashAndSizeForFileAsync(e.OldId);
if (tp != null)
{
e.OldMetaSize = tp.Item1;
e.OldMetaHash = tp.Item2;
e.OldMetaSize = tp.Value.Size;
e.OldMetaHash = tp.Value.MetadataHash;
}
}