1640 lines
70 KiB
C#
1640 lines
70 KiB
C#
// Copyright (C) 2025, The Duplicati Team
|
|
// https://duplicati.com, hello@duplicati.com
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a
|
|
// copy of this software and associated documentation files (the "Software"),
|
|
// to deal in the Software without restriction, including without limitation
|
|
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
// and/or sell copies of the Software, and to permit persons to whom the
|
|
// Software is furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in
|
|
// all copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
|
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
// DEALINGS IN THE SOFTWARE.
|
|
|
|
#nullable enable
|
|
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using System.Threading.Tasks;
|
|
using Duplicati.Library.Utility;
|
|
using Microsoft.Data.Sqlite;
|
|
|
|
namespace Duplicati.Library.Main.Database
|
|
{
|
|
|
|
internal class LocalBackupDatabase : LocalDatabase
|
|
{
|
|
/// <summary>
|
|
/// The tag used for logging
|
|
/// </summary>
|
|
private static readonly string LOGTAG = Logging.Log.LogTagFromType<LocalBackupDatabase>();
|
|
|
|
private SqliteCommand m_findblockCommand = null!;
|
|
private SqliteCommand m_findblocksetCommand = null!;
|
|
private SqliteCommand m_findfilesetCommand = null!;
|
|
private SqliteCommand m_findmetadatasetCommand = null!;
|
|
|
|
private SqliteCommand m_insertblockCommand = null!;
|
|
|
|
private SqliteCommand m_insertfileCommand = null!;
|
|
|
|
private SqliteCommand m_insertblocksetCommand = null!;
|
|
private SqliteCommand m_insertblocksetentryCommand = null!;
|
|
private SqliteCommand m_insertblocklistHashesCommand = null!;
|
|
|
|
private SqliteCommand m_insertmetadatasetCommand = null!;
|
|
|
|
private SqliteCommand m_findfileCommand = null!;
|
|
private SqliteCommand m_selectfilelastmodifiedCommand = null!;
|
|
private SqliteCommand m_selectfilelastmodifiedWithSizeCommand = null!;
|
|
private SqliteCommand m_selectfileHashCommand = null!;
|
|
|
|
private SqliteCommand m_insertfileOperationCommand = null!;
|
|
private SqliteCommand m_selectfilemetadatahashandsizeCommand = null!;
|
|
private SqliteCommand m_getfirstfilesetwithblockinblockset = null!;
|
|
|
|
private HashSet<string> m_blocklistHashes = [];
|
|
|
|
/// <summary>
|
|
/// The temporary table with deleted blocks that can be re-used; null if not table is used
|
|
/// </summary>
|
|
private string? m_tempDeletedBlockTable;
|
|
/// <summary>
|
|
/// The in-mmeory lookup for deleted blocks; null if in-memory lookup is not used
|
|
/// </summary>
|
|
private Dictionary<string, Dictionary<long, long>>? m_deletedBlockLookup;
|
|
/// <summary>
|
|
/// The command used to move deleted blocks to the main block table; null if not used
|
|
/// </summary>
|
|
private SqliteCommand? m_moveblockfromdeletedCommand;
|
|
/// <summary>
|
|
/// The command used to find blocks in the deleted blocks table; null if not used
|
|
/// </summary>
|
|
private SqliteCommand? m_findindeletedCommand;
|
|
|
|
private long m_filesetId;
|
|
|
|
private bool m_logQueries;
|
|
|
|
public static async Task<LocalBackupDatabase> CreateAsync(string path, Options options, LocalBackupDatabase? dbnew = null)
|
|
{
|
|
dbnew ??= new LocalBackupDatabase();
|
|
|
|
dbnew = (LocalBackupDatabase)await CreateLocalDatabaseAsync(path, "Backup", false, options.SqlitePageCache, dbnew);
|
|
dbnew.ShouldCloseConnection = true;
|
|
|
|
return await CreateAsync(dbnew, options);
|
|
}
|
|
|
|
public static async Task<LocalBackupDatabase> CreateAsync(LocalDatabase dbparent, Options options, LocalBackupDatabase? dbnew = null)
|
|
{
|
|
dbnew ??= new LocalBackupDatabase();
|
|
|
|
dbnew = (LocalBackupDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew);
|
|
|
|
dbnew.m_logQueries = options.ProfileAllDatabaseQueries;
|
|
|
|
dbnew.m_findblockCommand = await dbnew.Connection.CreateCommandAsync(@"
|
|
SELECT ""ID""
|
|
FROM ""Block""
|
|
WHERE
|
|
""Hash"" = @Hash
|
|
AND ""Size"" = @Size
|
|
");
|
|
|
|
dbnew.m_findblocksetCommand = await dbnew.Connection.CreateCommandAsync(@"
|
|
SELECT ""ID""
|
|
FROM ""Blockset""
|
|
WHERE
|
|
""Fullhash"" = @Fullhash
|
|
AND ""Length"" = @Length
|
|
");
|
|
|
|
dbnew.m_findmetadatasetCommand = await dbnew.Connection.CreateCommandAsync(@"
|
|
SELECT ""A"".""ID""
|
|
FROM
|
|
""Metadataset"" A,
|
|
""BlocksetEntry"" B,
|
|
""Block"" C
|
|
WHERE
|
|
""A"".""BlocksetID"" = ""B"".""BlocksetID""
|
|
AND ""B"".""BlockID"" = ""C"".""ID""
|
|
AND ""C"".""Hash"" = @Hash
|
|
AND ""C"".""Size"" = @Size
|
|
");
|
|
|
|
dbnew.m_findfilesetCommand = await dbnew.Connection.CreateCommandAsync(@"
|
|
SELECT ""ID""
|
|
FROM ""FileLookup""
|
|
WHERE
|
|
""BlocksetID"" = @BlocksetId
|
|
AND ""MetadataID"" = @MetadataId
|
|
AND ""Path"" = @Path
|
|
AND ""PrefixID"" = @PrefixId
|
|
");
|
|
|
|
dbnew.m_insertblockCommand = await dbnew.Connection.CreateCommandAsync(@"
|
|
INSERT INTO ""Block"" (
|
|
""Hash"",
|
|
""VolumeID"",
|
|
""Size""
|
|
)
|
|
VALUES (
|
|
@Hash,
|
|
@VolumeId,
|
|
@Size
|
|
);
|
|
SELECT last_insert_rowid();
|
|
");
|
|
|
|
dbnew.m_insertfileOperationCommand = await dbnew.Connection.CreateCommandAsync(@"
|
|
INSERT INTO ""FilesetEntry"" (
|
|
""FilesetID"",
|
|
""FileID"",
|
|
""Lastmodified""
|
|
)
|
|
VALUES (
|
|
@FilesetId,
|
|
@FileId,
|
|
@LastModified
|
|
)
|
|
");
|
|
|
|
dbnew.m_insertfileCommand = await dbnew.Connection.CreateCommandAsync(@"
|
|
INSERT INTO ""FileLookup"" (
|
|
""PrefixID"",
|
|
""Path"",
|
|
""BlocksetID"",
|
|
""MetadataID""
|
|
)
|
|
VALUES (
|
|
@PrefixId,
|
|
@Path,
|
|
@BlocksetId,
|
|
@MetadataId
|
|
);
|
|
SELECT last_insert_rowid();
|
|
");
|
|
|
|
dbnew.m_insertblocksetCommand = await dbnew.Connection.CreateCommandAsync(@"
|
|
INSERT INTO ""Blockset"" (
|
|
""Length"",
|
|
""FullHash""
|
|
)
|
|
VALUES (
|
|
@Length,
|
|
@Fullhash
|
|
);
|
|
SELECT last_insert_rowid();");
|
|
|
|
dbnew.m_insertblocksetentryCommand = await dbnew.Connection.CreateCommandAsync(@"
|
|
INSERT INTO ""BlocksetEntry"" (
|
|
""BlocksetID"",
|
|
""Index"",
|
|
""BlockID""
|
|
)
|
|
SELECT
|
|
@BlocksetId AS A,
|
|
@Index AS B,
|
|
""ID""
|
|
FROM ""Block""
|
|
WHERE ""Hash"" = @Hash
|
|
AND ""Size"" = @Size
|
|
");
|
|
|
|
dbnew.m_insertblocklistHashesCommand = await dbnew.Connection.CreateCommandAsync(@"
|
|
INSERT INTO ""BlocklistHash"" (
|
|
""BlocksetID"",
|
|
""Index"",
|
|
""Hash""
|
|
)
|
|
VALUES (
|
|
@BlocksetId,
|
|
@Index,
|
|
@Hash
|
|
)
|
|
");
|
|
|
|
dbnew.m_insertmetadatasetCommand = await dbnew.Connection.CreateCommandAsync(@"
|
|
INSERT INTO ""Metadataset"" (""BlocksetID"")
|
|
VALUES (@BlocksetId);
|
|
SELECT last_insert_rowid();
|
|
");
|
|
|
|
dbnew.m_selectfilelastmodifiedCommand = await dbnew.Connection.CreateCommandAsync(@"
|
|
SELECT
|
|
""A"".""ID"",
|
|
""B"".""LastModified""
|
|
FROM (
|
|
SELECT ""ID""
|
|
FROM ""FileLookup""
|
|
WHERE ""PrefixID"" = @PrefixId
|
|
AND ""Path"" = @Path
|
|
) ""A""
|
|
CROSS JOIN ""FilesetEntry"" ""B""
|
|
WHERE
|
|
""A"".""ID"" = ""B"".""FileID""
|
|
AND ""B"".""FilesetID"" = @FilesetId
|
|
");
|
|
|
|
dbnew.m_selectfilelastmodifiedWithSizeCommand = await dbnew.Connection.CreateCommandAsync(@"
|
|
SELECT
|
|
""C"".""ID"",
|
|
""C"".""LastModified"",
|
|
""D"".""Length""
|
|
FROM
|
|
(
|
|
SELECT
|
|
""A"".""ID"",
|
|
""B"".""LastModified"",
|
|
""A"".""BlocksetID""
|
|
FROM (
|
|
SELECT
|
|
""ID"",
|
|
""BlocksetID""
|
|
FROM ""FileLookup""
|
|
WHERE
|
|
""PrefixID"" = @PrefixId
|
|
AND ""Path"" = @Path
|
|
) ""A""
|
|
CROSS JOIN ""FilesetEntry"" ""B""
|
|
WHERE
|
|
""A"".""ID"" = ""B"".""FileID""
|
|
AND ""B"".""FilesetID"" = @FilesetId
|
|
) AS ""C"",
|
|
""Blockset"" AS ""D""
|
|
WHERE ""C"".""BlocksetID"" == ""D"".""ID""
|
|
");
|
|
|
|
dbnew.m_selectfilemetadatahashandsizeCommand = await dbnew.Connection.CreateCommandAsync(@"
|
|
SELECT
|
|
""Blockset"".""Length"",
|
|
""Blockset"".""FullHash""
|
|
FROM
|
|
""Blockset"",
|
|
""Metadataset"",
|
|
""File""
|
|
WHERE
|
|
""File"".""ID"" = @FileId
|
|
AND ""Blockset"".""ID"" = ""Metadataset"".""BlocksetID""
|
|
AND ""Metadataset"".""ID"" = ""File"".""MetadataID""
|
|
");
|
|
|
|
// Experimental toggling of the deleted block cache
|
|
// If the value is less than zero, the lookup is disabled
|
|
// meaning that deleted blocks are never reused (same as 2.1.0.5 and earlier)
|
|
// A value of zero disables the in-memory cache, always using a temporary table
|
|
// Any other value is the size of the in-memory cache
|
|
// If the number of deleted blocks exceed the cache size, a temporary table is used
|
|
var deletedBlockCacheSize = Environment.GetEnvironmentVariable("DUPLICATI_DELETEDBLOCKCACHESIZE");
|
|
if (!long.TryParse(deletedBlockCacheSize, out var deletedBlockCacheSizeLong))
|
|
deletedBlockCacheSizeLong = 10000;
|
|
|
|
if (deletedBlockCacheSizeLong >= 0)
|
|
{
|
|
using (var cmd = dbnew.Connection.CreateCommand())
|
|
{
|
|
dbnew.m_tempDeletedBlockTable = "DeletedBlock-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
|
|
await cmd.SetCommandAndParameters($@"
|
|
CREATE TEMPORARY TABLE ""{dbnew.m_tempDeletedBlockTable}"" AS
|
|
SELECT
|
|
MAX(""ID"") AS ""ID"",
|
|
""Hash"",
|
|
""Size""
|
|
FROM ""DeletedBlock""
|
|
WHERE ""VolumeID"" IN (
|
|
SELECT ""ID""
|
|
FROM ""RemoteVolume""
|
|
WHERE ""State"" NOT IN (@States)
|
|
)
|
|
GROUP BY
|
|
""Hash"",
|
|
""Size""
|
|
")
|
|
.ExpandInClauseParameterAsync("@States", [
|
|
RemoteVolumeState.Deleted,
|
|
RemoteVolumeState.Deleting
|
|
])
|
|
.ExecuteNonQueryAsync();
|
|
|
|
var deletedBlocks = await cmd.ExecuteScalarInt64Async(@$"
|
|
SELECT COUNT(*)
|
|
FROM ""{dbnew.m_tempDeletedBlockTable}""
|
|
", 0);
|
|
|
|
// There are no deleted blocks, so we can drop the table
|
|
if (deletedBlocks == 0)
|
|
{
|
|
await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{dbnew.m_tempDeletedBlockTable}""");
|
|
dbnew.m_tempDeletedBlockTable = null;
|
|
|
|
}
|
|
// The deleted blocks are small enough to fit in memory
|
|
else if (deletedBlocks <= deletedBlockCacheSizeLong)
|
|
{
|
|
dbnew.m_deletedBlockLookup = new Dictionary<string, Dictionary<long, long>>();
|
|
cmd.SetCommandAndParameters(@$"
|
|
SELECT
|
|
""ID"",
|
|
""Hash"",
|
|
""Size""
|
|
FROM ""{dbnew.m_tempDeletedBlockTable}""
|
|
");
|
|
|
|
using (var reader = await cmd.ExecuteReaderAsync())
|
|
while (await reader.ReadAsync())
|
|
{
|
|
var id = reader.ConvertValueToInt64(0);
|
|
var hash = reader.ConvertValueToString(1) ?? throw new Exception("Hash is null");
|
|
var size = reader.ConvertValueToInt64(2);
|
|
|
|
if (!dbnew.m_deletedBlockLookup.TryGetValue(hash, out var sizes))
|
|
dbnew.m_deletedBlockLookup[hash] = sizes = new Dictionary<long, long>();
|
|
sizes[size] = id;
|
|
}
|
|
|
|
await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{dbnew.m_tempDeletedBlockTable}""");
|
|
dbnew.m_tempDeletedBlockTable = null;
|
|
}
|
|
// The deleted blocks are too large to fit in memory, so we use a temporary table
|
|
else
|
|
{
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
CREATE UNIQUE INDEX ""unique_{dbnew.m_tempDeletedBlockTable}""
|
|
ON ""{dbnew.m_tempDeletedBlockTable}"" (
|
|
""Hash"",
|
|
""Size""
|
|
)
|
|
");
|
|
|
|
dbnew.m_findindeletedCommand = await dbnew.Connection.CreateCommandAsync($@"
|
|
SELECT ""ID""
|
|
FROM ""{dbnew.m_tempDeletedBlockTable}""
|
|
WHERE
|
|
""Hash"" = @Hash
|
|
AND ""Size"" = @Size
|
|
");
|
|
|
|
dbnew.m_moveblockfromdeletedCommand = await dbnew.Connection.CreateCommandAsync(@$"
|
|
INSERT INTO ""Block"" (
|
|
""Hash"",
|
|
""Size"",
|
|
""VolumeID""
|
|
)
|
|
SELECT
|
|
""Hash"",
|
|
""Size"",
|
|
""VolumeID""
|
|
FROM ""DeletedBlock""
|
|
WHERE ""ID"" = @DeletedBlockId LIMIT 1;
|
|
|
|
DELETE FROM ""DeletedBlock""
|
|
WHERE ""ID"" = @DeletedBlockId;
|
|
|
|
DELETE FROM {dbnew.m_tempDeletedBlockTable}
|
|
WHERE ""ID"" = @DeletedBlockId;
|
|
|
|
SELECT last_insert_rowid()
|
|
");
|
|
|
|
}
|
|
|
|
if (deletedBlocks > 0)
|
|
{
|
|
dbnew.m_moveblockfromdeletedCommand = await dbnew.m_connection.CreateCommandAsync(@$"
|
|
INSERT INTO ""Block"" (
|
|
""Hash"",
|
|
""Size"",
|
|
""VolumeID""
|
|
)
|
|
SELECT
|
|
""Hash"",
|
|
""Size"",
|
|
""VolumeID""
|
|
FROM ""DeletedBlock""
|
|
WHERE ""ID"" = @DeletedBlockId LIMIT 1;
|
|
|
|
DELETE FROM ""DeletedBlock""
|
|
WHERE ""ID"" = @DeletedBlockId;
|
|
|
|
SELECT last_insert_rowid()
|
|
");
|
|
}
|
|
|
|
}
|
|
}
|
|
|
|
// Allow users to test on real-world data
|
|
// to get feedback on potential performance
|
|
int.TryParse(Environment.GetEnvironmentVariable("TEST_QUERY_VERSION"), out var testqueryversion);
|
|
|
|
if (testqueryversion != 0)
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "TestFileQuery", null, "Using performance test query version {0} as the TEST_QUERY_VERSION environment variable is set", testqueryversion);
|
|
|
|
// The original query (v==1) finds the most recent entry of the file in question,
|
|
// but it requires some large joins to extract the required information.
|
|
// To speed it up, we use a slightly simpler approach that only looks at the
|
|
// previous fileset, and uses information here.
|
|
// If there is a case where a file is sometimes there and sometimes not
|
|
// (i.e. filter file, remove filter) we will not find the file.
|
|
// We currently use this faster version,
|
|
// but allow users to switch back via an environment variable
|
|
// such that we can get performance feedback
|
|
|
|
string findQuery;
|
|
switch (testqueryversion)
|
|
{
|
|
// The query used in Duplicati until 2.0.3.9
|
|
case 1:
|
|
findQuery = @"
|
|
SELECT
|
|
""FileLookup"".""ID"" AS ""FileID"",
|
|
""FilesetEntry"".""Lastmodified"",
|
|
""FileBlockset"".""Length"",
|
|
""MetaBlockset"".""Fullhash"" AS ""Metahash"",
|
|
""MetaBlockset"".""Length"" AS ""Metasize""
|
|
FROM
|
|
""FileLookup"",
|
|
""FilesetEntry"",
|
|
""Fileset"",
|
|
""Blockset"" ""FileBlockset"",
|
|
""Metadataset"",
|
|
""Blockset"" ""MetaBlockset""
|
|
WHERE
|
|
""FileLookup"".""PrefixID"" = @PrefixId
|
|
AND ""FileLookup"".""Path"" = @Path
|
|
AND ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID""
|
|
AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID""
|
|
AND ""FileBlockset"".""ID"" = ""FileLookup"".""BlocksetID""
|
|
AND ""Metadataset"".""ID"" = ""FileLookup"".""MetadataID""
|
|
AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID""
|
|
AND @FilesetId IS NOT NULL
|
|
ORDER BY ""Fileset"".""Timestamp"" DESC
|
|
LIMIT 1
|
|
";
|
|
break;
|
|
|
|
// The fastest reported query in Duplicati 2.0.3.10, but with "LIMIT 1" added
|
|
default:
|
|
case 2:
|
|
var getLastFileEntryForPath = @"
|
|
SELECT
|
|
""A"".""ID"",
|
|
""B"".""LastModified"",
|
|
""A"".""BlocksetID"",
|
|
""A"".""MetadataID""
|
|
FROM (
|
|
SELECT
|
|
""ID"",
|
|
""BlocksetID"",
|
|
""MetadataID""
|
|
FROM ""FileLookup""
|
|
WHERE
|
|
""PrefixID"" = @PrefixId
|
|
AND ""Path"" = @Path
|
|
) ""A""
|
|
CROSS JOIN ""FilesetEntry"" ""B""
|
|
WHERE
|
|
""A"".""ID"" = ""B"".""FileID""
|
|
AND ""B"".""FilesetID"" = @FilesetId
|
|
";
|
|
|
|
findQuery = $@"
|
|
SELECT
|
|
""C"".""ID"" AS ""FileID"",
|
|
""C"".""LastModified"",
|
|
""D"".""Length"",
|
|
""E"".""FullHash"" as ""Metahash"",
|
|
""E"".""Length"" AS ""Metasize""
|
|
FROM
|
|
({getLastFileEntryForPath}) AS ""C"",
|
|
""Blockset"" AS ""D"",
|
|
""Blockset"" AS ""E"",
|
|
""Metadataset"" ""F""
|
|
WHERE
|
|
""C"".""BlocksetID"" == ""D"".""ID""
|
|
AND ""C"".""MetadataID"" == ""F"".""ID""
|
|
AND ""F"".""BlocksetID"" = ""E"".""ID""
|
|
LIMIT 1
|
|
";
|
|
break;
|
|
|
|
// Potentially faster query: https://forum.duplicati.com/t/release-2-0-3-10-canary-2018-08-30/4497/25
|
|
case 3:
|
|
findQuery = @"
|
|
SELECT
|
|
FileLookup.ID as FileID,
|
|
FilesetEntry.Lastmodified,
|
|
FileBlockset.Length,
|
|
MetaBlockset.FullHash AS Metahash,
|
|
MetaBlockset.Length as Metasize
|
|
FROM FilesetEntry
|
|
INNER JOIN Fileset
|
|
ON (FileSet.ID = FilesetEntry.FilesetID)
|
|
INNER JOIN FileLookup
|
|
ON (FileLookup.ID = FilesetEntry.FileID)
|
|
INNER JOIN Metadataset
|
|
ON (Metadataset.ID = FileLookup.MetadataID)
|
|
INNER JOIN Blockset AS MetaBlockset
|
|
ON (MetaBlockset.ID = Metadataset.BlocksetID)
|
|
LEFT JOIN Blockset AS FileBlockset
|
|
ON (FileBlockset.ID = FileLookup.BlocksetID)
|
|
WHERE
|
|
FileLookup.PrefixID = @PrefixId
|
|
AND FileLookup.Path = @Path
|
|
AND FilesetID = @FilesetId
|
|
LIMIT 1
|
|
";
|
|
break;
|
|
|
|
// The slow query used in Duplicati 2.0.3.10, but with "LIMIT 1" added
|
|
case 4:
|
|
findQuery = @"
|
|
SELECT
|
|
""FileLookup"".""ID"" AS ""FileID"",
|
|
""FilesetEntry"".""Lastmodified"",
|
|
""FileBlockset"".""Length"",
|
|
""MetaBlockset"".""Fullhash"" AS ""Metahash"",
|
|
""MetaBlockset"".""Length"" AS ""Metasize""
|
|
FROM
|
|
""FileLookup"",
|
|
""FilesetEntry"",
|
|
""Fileset"",
|
|
""Blockset"" ""FileBlockset"",
|
|
""Metadataset"",
|
|
""Blockset"" ""MetaBlockset""
|
|
WHERE
|
|
""FileLookup"".""PrefixID"" = @PrefixId
|
|
AND ""FileLookup"".""Path"" = @Path
|
|
AND ""Fileset"".""ID"" = @FilesetId
|
|
AND ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID""
|
|
AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID""
|
|
AND ""FileBlockset"".""ID"" = ""FileLookup"".""BlocksetID""
|
|
AND ""Metadataset"".""ID"" = ""FileLookup"".""MetadataID""
|
|
AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID""
|
|
LIMIT 1
|
|
";
|
|
break;
|
|
|
|
}
|
|
|
|
dbnew.m_findfileCommand = dbnew.m_connection.CreateCommand(findQuery);
|
|
|
|
dbnew.m_selectfileHashCommand = dbnew.m_connection.CreateCommand(@"
|
|
SELECT ""Blockset"".""Fullhash""
|
|
FROM ""Blockset"", ""FileLookup""
|
|
WHERE
|
|
""Blockset"".""ID"" = ""FileLookup"".""BlocksetID""
|
|
AND ""FileLookup"".""ID"" = @FileId
|
|
");
|
|
|
|
dbnew.m_getfirstfilesetwithblockinblockset = dbnew.m_connection.CreateCommand(@"
|
|
SELECT MIN(""FilesetEntry"".""FilesetID"")
|
|
FROM ""FilesetEntry""
|
|
WHERE ""FilesetEntry"".""FileID"" IN (
|
|
SELECT ""File"".""ID""
|
|
FROM ""File""
|
|
WHERE ""File"".""BlocksetID"" IN(
|
|
SELECT ""BlocklistHash"".""BlocksetID""
|
|
FROM ""BlocklistHash""
|
|
WHERE ""BlocklistHash"".""Hash"" = @Hash
|
|
)
|
|
)
|
|
");
|
|
|
|
dbnew.m_blocklistHashes = new HashSet<string>();
|
|
|
|
return dbnew;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Probes to see if a block already exists
|
|
/// </summary>
|
|
/// <param name="key">The block key</param>
|
|
/// <param name="size">The size of the block</param>
|
|
/// <returns>True if the block should be added to the current output</returns>
|
|
public async Task<long> FindBlockID(string key, long size)
|
|
{
|
|
return await m_findblockCommand
|
|
.SetTransaction(m_rtr)
|
|
.SetParameterValue("@Hash", key)
|
|
.SetParameterValue("@Size", size)
|
|
.ExecuteScalarInt64Async(m_logQueries, -1);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Adds a block to the local database, returning a value indicating if the value presents a new block
|
|
/// </summary>
|
|
/// <param name="key">The block key</param>
|
|
/// <param name="size">The size of the block</param>
|
|
/// <returns>True if the block should be added to the current output</returns>
|
|
public async Task<bool> AddBlock(string key, long size, long volumeid)
|
|
{
|
|
var r = await FindBlockID(key, size);
|
|
if (r == -1L)
|
|
{
|
|
if (m_moveblockfromdeletedCommand != null)
|
|
{
|
|
if (m_deletedBlockLookup != null)
|
|
{
|
|
if (m_deletedBlockLookup.TryGetValue(key, out var sizes))
|
|
if (sizes.TryGetValue(size, out var id))
|
|
{
|
|
await m_moveblockfromdeletedCommand
|
|
.SetTransaction(m_rtr)
|
|
.SetParameterValue("@DeletedBlockId", id)
|
|
.ExecuteNonQueryAsync(m_logQueries);
|
|
|
|
sizes.Remove(size);
|
|
if (sizes.Count == 0)
|
|
m_deletedBlockLookup.Remove(key);
|
|
return false;
|
|
}
|
|
}
|
|
else if (m_findindeletedCommand != null)
|
|
{
|
|
// No transaction on the temporary table
|
|
var id = await m_findindeletedCommand
|
|
.SetParameterValue("@Hash", key)
|
|
.SetParameterValue("@Size", size)
|
|
.ExecuteScalarInt64Async(m_logQueries, -1);
|
|
|
|
if (id != -1)
|
|
{
|
|
var c = await m_moveblockfromdeletedCommand
|
|
.SetTransaction(m_rtr)
|
|
.SetParameterValue("@DeletedBlockId", id)
|
|
.ExecuteNonQueryAsync(m_logQueries);
|
|
|
|
if (c != 2)
|
|
throw new Exception($"Failed to move block {key} with size {size}, result count: {c}");
|
|
|
|
// We do not clean up the temporary table, as the regular block lookup should now find it
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
|
|
var ins = await m_insertblockCommand
|
|
.SetTransaction(m_rtr)
|
|
.SetParameterValue("@Hash", key)
|
|
.SetParameterValue("@VolumeId", volumeid)
|
|
.SetParameterValue("@Size", size)
|
|
.ExecuteNonQueryAsync(m_logQueries);
|
|
if (ins != 1)
|
|
throw new Exception($"Failed to insert block {key} with size {size}, result count: {ins}");
|
|
|
|
return true;
|
|
}
|
|
else
|
|
{
|
|
//Update lookup cache if required
|
|
return false;
|
|
}
|
|
}
|
|
|
|
|
|
/// <summary>
|
|
/// Adds a blockset to the database, returns a value indicating if the blockset is new
|
|
/// </summary>
|
|
/// <param name="filehash">The hash of the blockset</param>
|
|
/// <param name="size">The size of the blockset</param>
|
|
/// <param name="hashes">The list of hashes</param>
|
|
/// <param name="blocksetid">The id of the blockset, new or old</param>
|
|
/// <returns>True if the blockset was created, false otherwise</returns>
|
|
public async Task<(bool, long)> AddBlockset(string filehash, long size, int blocksize, IEnumerable<string> hashes, IEnumerable<string> blocklistHashes)
|
|
{
|
|
long blocksetid = await m_findblocksetCommand
|
|
.SetTransaction(m_rtr)
|
|
.SetParameterValue("@Fullhash", filehash)
|
|
.SetParameterValue("@Length", size)
|
|
.ExecuteScalarInt64Async(m_logQueries, null, -1);
|
|
|
|
if (blocksetid != -1)
|
|
return (false, blocksetid); //Found it
|
|
|
|
blocksetid = await m_insertblocksetCommand
|
|
.SetTransaction(m_rtr)
|
|
.SetParameterValue("@Length", size)
|
|
.SetParameterValue("@Fullhash", filehash)
|
|
.ExecuteScalarInt64Async(m_logQueries);
|
|
|
|
long ix = 0;
|
|
if (blocklistHashes != null)
|
|
{
|
|
m_insertblocklistHashesCommand
|
|
.SetTransaction(m_rtr)
|
|
.SetParameterValue("@BlocksetId", blocksetid);
|
|
|
|
foreach (var bh in blocklistHashes)
|
|
{
|
|
var c = await m_insertblocklistHashesCommand
|
|
.SetParameterValue("@Index", ix)
|
|
.SetParameterValue("@Hash", bh)
|
|
.ExecuteNonQueryAsync(m_logQueries);
|
|
if (c != 1)
|
|
throw new Exception($"Failed to insert blocklist hash {bh} for blockset {blocksetid}, result count: {c}");
|
|
ix++;
|
|
}
|
|
}
|
|
|
|
m_insertblocksetentryCommand
|
|
.SetTransaction(m_rtr)
|
|
.SetParameterValue("@BlocksetId", blocksetid);
|
|
|
|
ix = 0;
|
|
long remainsize = size;
|
|
foreach (var h in hashes)
|
|
{
|
|
var exsize = remainsize < blocksize ? remainsize : blocksize;
|
|
var c = await m_insertblocksetentryCommand
|
|
.SetParameterValue("@Index", ix)
|
|
.SetParameterValue("@Hash", h)
|
|
.SetParameterValue("@Size", exsize)
|
|
.ExecuteNonQueryAsync(m_logQueries);
|
|
if (c != 1)
|
|
{
|
|
Logging.Log.WriteErrorMessage(LOGTAG, "CheckingErrorsForIssue1400", null, "Checking errors, related to #1400. Unexpected result count: {0}, expected {1}, hash: {2}, size: {3}, blocksetid: {4}, ix: {5}, fullhash: {6}, fullsize: {7}", c, 1, h, exsize, blocksetid, ix, filehash, size);
|
|
using (var cmd = m_connection.CreateCommand(m_rtr))
|
|
{
|
|
var bid = await cmd.SetCommandAndParameters(@"
|
|
SELECT ""ID""
|
|
FROM ""Block""
|
|
WHERE ""Hash"" = @Hash
|
|
")
|
|
.SetParameterValue("@Hash", h)
|
|
.ExecuteScalarInt64Async(-1);
|
|
|
|
if (bid == -1)
|
|
throw new Exception($"Could not find any blocks with the given hash: {h}");
|
|
|
|
cmd.SetCommandAndParameters(@"
|
|
SELECT ""Size""
|
|
FROM ""Block""
|
|
WHERE ""Hash"" = @Hash
|
|
")
|
|
.SetParameterValue("@Hash", h);
|
|
|
|
await foreach (var rd in cmd.ExecuteReaderEnumerableAsync())
|
|
Logging.Log.WriteErrorMessage(LOGTAG, "FoundIssue1400Error", null, "Found block with ID {0} and hash {1} and size {2}", bid, h, rd.ConvertValueToInt64(0, -1));
|
|
}
|
|
|
|
throw new Exception($"Unexpected result count: {c}, expected {1}, check log for more messages");
|
|
}
|
|
|
|
ix++;
|
|
remainsize -= blocksize;
|
|
}
|
|
|
|
await m_rtr.CommitAsync();
|
|
|
|
return (true, blocksetid);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the metadataset ID from the filehash
|
|
/// </summary>
|
|
/// <returns><c>true</c>, if metadataset found, false if does not exist.</returns>
|
|
/// <param name="filehash">The metadata hash.</param>
|
|
/// <param name="size">The size of the metadata.</param>
|
|
/// <param name="metadataid">The ID of the metadataset.</param>
|
|
/// <param name="transaction">An optional transaction.</param>
|
|
public async Task<(bool, long)> GetMetadatasetID(string filehash, long size)
|
|
{
|
|
long metadataid;
|
|
|
|
if (size > 0)
|
|
{
|
|
metadataid = await m_findmetadatasetCommand
|
|
.SetTransaction(m_rtr)
|
|
.SetParameterValue("@Hash", filehash)
|
|
.SetParameterValue("@Size", size)
|
|
.ExecuteScalarInt64Async(m_logQueries, null, -1);
|
|
return (metadataid != -1, metadataid);
|
|
}
|
|
|
|
metadataid = -2;
|
|
return (false, metadataid);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Adds a metadata set to the database, and returns a value indicating if the record was new
|
|
/// </summary>
|
|
/// <param name="filehash">The metadata hash</param>
|
|
/// <param name="size">The size of the metadata</param>
|
|
/// <param name="transaction">The transaction to execute under</param>
|
|
/// <param name="blocksetid">The id of the blockset to add</param>
|
|
/// <param name="metadataid">The id of the metadata set</param>
|
|
/// <returns>True if the set was added to the database, false otherwise</returns>
|
|
public async Task<(bool, long)> AddMetadataset(string filehash, long size, long blocksetid)
|
|
{
|
|
var (metadatafound, metadataid) = await GetMetadatasetID(filehash, size);
|
|
if (metadatafound)
|
|
return (false, metadataid);
|
|
|
|
metadataid = await m_insertmetadatasetCommand
|
|
.SetTransaction(m_rtr)
|
|
.SetParameterValue("@BlocksetId", blocksetid)
|
|
.ExecuteScalarInt64Async(m_logQueries);
|
|
|
|
m_rtr.Commit();
|
|
|
|
return (true, metadataid);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Adds a file record to the database
|
|
/// </summary>
|
|
/// <param name="pathprefixid">The path prefix ID</param>
|
|
/// <param name="filename">The path to the file</param>
|
|
/// <param name="lastmodified">The time the file was modified</param>
|
|
/// <param name="blocksetID">The ID of the hashkey for the file</param>
|
|
/// <param name="metadataID">The ID for the metadata</param>
|
|
/// <param name="transaction">The transaction to use for insertion, or null for no transaction</param>
|
|
public async Task AddFile(long pathprefixid, string filename, DateTime lastmodified, long blocksetID, long metadataID)
|
|
{
|
|
var fileidobj = await m_findfilesetCommand
|
|
.SetTransaction(m_rtr)
|
|
.SetParameterValue("@BlocksetId", blocksetID)
|
|
.SetParameterValue("@MetadataId", metadataID)
|
|
.SetParameterValue("@Path", filename)
|
|
.SetParameterValue("@PrefixId", pathprefixid)
|
|
.ExecuteScalarInt64Async(m_logQueries);
|
|
|
|
if (fileidobj == -1)
|
|
{
|
|
fileidobj = await m_insertfileCommand.SetTransaction(m_rtr)
|
|
.SetParameterValue("@PrefixId", pathprefixid)
|
|
.SetParameterValue("@Path", filename)
|
|
.SetParameterValue("@BlocksetId", blocksetID)
|
|
.SetParameterValue("@MetadataId", metadataID)
|
|
.ExecuteScalarInt64Async(m_logQueries);
|
|
|
|
await m_rtr.CommitAsync();
|
|
}
|
|
|
|
await AddKnownFile(fileidobj, lastmodified);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Adds a file record to the database
|
|
/// </summary>
|
|
/// <param name="filename">The path to the file</param>
|
|
/// <param name="lastmodified">The time the file was modified</param>
|
|
/// <param name="blocksetID">The ID of the hashkey for the file</param>
|
|
/// <param name="metadataID">The ID for the metadata</param>
|
|
/// <param name="transaction">The transaction to use for insertion, or null for no transaction</param>
|
|
public async Task AddFile(string filename, DateTime lastmodified, long blocksetID, long metadataID)
|
|
{
|
|
var split = SplitIntoPrefixAndName(filename);
|
|
|
|
await AddFile(
|
|
await GetOrCreatePathPrefix(split.Key),
|
|
split.Value,
|
|
lastmodified,
|
|
blocksetID,
|
|
metadataID
|
|
);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Adds a known file to the fileset
|
|
/// </summary>
|
|
/// <param name="fileid">Id of the file</param>
|
|
/// <param name="lastmodified">The time the file was modified</param>
|
|
/// <param name="transaction">The transaction to use for insertion, or null for no transaction</param>
|
|
public async Task AddKnownFile(long fileid, DateTime lastmodified)
|
|
{
|
|
await m_insertfileOperationCommand
|
|
.SetTransaction(m_rtr)
|
|
.SetParameterValue("@FilesetId", m_filesetId)
|
|
.SetParameterValue("@FileId", fileid)
|
|
.SetParameterValue("@LastModified", lastmodified.ToUniversalTime().Ticks)
|
|
.ExecuteNonQueryAsync(m_logQueries);
|
|
}
|
|
|
|
public async Task AddDirectoryEntry(string path, long metadataID, DateTime lastmodified)
|
|
{
|
|
await AddFile(path, lastmodified, FOLDER_BLOCKSET_ID, metadataID);
|
|
}
|
|
|
|
public async Task AddSymlinkEntry(string path, long metadataID, DateTime lastmodified)
|
|
{
|
|
await AddFile(path, lastmodified, SYMLINK_BLOCKSET_ID, metadataID);
|
|
}
|
|
|
|
public async Task<(long, DateTime, long)> GetFileLastModified(long prefixid, string path, long filesetid, bool includeLength)
|
|
{
|
|
DateTime oldModified;
|
|
long length;
|
|
if (includeLength)
|
|
{
|
|
m_selectfilelastmodifiedWithSizeCommand
|
|
.SetTransaction(m_rtr)
|
|
.SetParameterValue("@PrefixId", prefixid)
|
|
.SetParameterValue("@Path", path)
|
|
.SetParameterValue("@FilesetId", filesetid);
|
|
|
|
using (var rd = await m_selectfilelastmodifiedWithSizeCommand.ExecuteReaderAsync(m_logQueries, null))
|
|
if (await rd.ReadAsync())
|
|
{
|
|
oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc);
|
|
length = rd.ConvertValueToInt64(2);
|
|
return (rd.ConvertValueToInt64(0), oldModified, length);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
m_selectfilelastmodifiedCommand.SetTransaction(m_rtr)
|
|
.SetParameterValue("@PrefixId", prefixid)
|
|
.SetParameterValue("@Path", path)
|
|
.SetParameterValue("@FilesetId", filesetid);
|
|
|
|
using (var rd = await m_selectfilelastmodifiedCommand.ExecuteReaderAsync(m_logQueries, null))
|
|
if (await rd.ReadAsync())
|
|
{
|
|
length = -1;
|
|
oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc);
|
|
return (rd.ConvertValueToInt64(0), oldModified, length);
|
|
}
|
|
}
|
|
|
|
oldModified = new DateTime(0, DateTimeKind.Utc);
|
|
length = -1;
|
|
return (-1, oldModified, length);
|
|
}
|
|
|
|
|
|
public async Task<(long, DateTime, long, string?, long)> GetFileEntry(long prefixid, string path, long filesetid)
|
|
{
|
|
DateTime oldModified;
|
|
long lastFileSize;
|
|
string? oldMetahash;
|
|
long oldMetasize;
|
|
|
|
m_findfileCommand
|
|
.SetTransaction(m_rtr)
|
|
.SetParameterValue("@PrefixId", prefixid)
|
|
.SetParameterValue("@Path", path)
|
|
.SetParameterValue("@FilesetId", filesetid);
|
|
|
|
using (var rd = await m_findfileCommand.ExecuteReaderAsync())
|
|
if (await rd.ReadAsync())
|
|
{
|
|
oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc);
|
|
lastFileSize = rd.ConvertValueToInt64(2);
|
|
oldMetahash = rd.ConvertValueToString(3);
|
|
oldMetasize = rd.ConvertValueToInt64(4);
|
|
return (
|
|
rd.ConvertValueToInt64(0),
|
|
oldModified,
|
|
lastFileSize,
|
|
oldMetahash,
|
|
oldMetasize
|
|
);
|
|
}
|
|
else
|
|
{
|
|
oldModified = new DateTime(0, DateTimeKind.Utc);
|
|
lastFileSize = -1;
|
|
oldMetahash = null;
|
|
oldMetasize = -1;
|
|
return (
|
|
-1,
|
|
oldModified,
|
|
lastFileSize,
|
|
oldMetahash,
|
|
oldMetasize
|
|
);
|
|
}
|
|
}
|
|
|
|
public async Task<(long Size, string MetadataHash)?> GetMetadataHashAndSizeForFile(long fileid)
|
|
{
|
|
m_selectfilemetadatahashandsizeCommand
|
|
.SetTransaction(m_rtr)
|
|
.SetParameterValue("@FileId", fileid);
|
|
|
|
using (var rd = await m_selectfilemetadatahashandsizeCommand.ExecuteReaderAsync())
|
|
if (await rd.ReadAsync())
|
|
return (
|
|
rd.ConvertValueToInt64(0),
|
|
rd.ConvertValueToString(1) ?? throw new InvalidOperationException("Metadata hash is null")
|
|
);
|
|
|
|
return null;
|
|
}
|
|
|
|
|
|
public async Task<string?> GetFileHash(long fileid)
|
|
{
|
|
var r = await m_selectfileHashCommand
|
|
.SetTransaction(m_rtr)
|
|
.SetParameterValue("@FileId", fileid)
|
|
.ExecuteScalarAsync(m_logQueries, null);
|
|
|
|
if (r == null || r == DBNull.Value)
|
|
return null;
|
|
|
|
return r.ToString();
|
|
}
|
|
|
|
public override void Dispose()
|
|
{
|
|
DisposeAsync().Await();
|
|
}
|
|
|
|
public override async Task DisposeAsync()
|
|
{
|
|
if (!string.IsNullOrWhiteSpace(m_tempDeletedBlockTable))
|
|
try
|
|
{
|
|
using (var cmd = m_connection.CreateCommand())
|
|
await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{m_tempDeletedBlockTable}""");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "DropTempTableFailed", ex, "Failed to drop temporary table {0}: {1}", m_tempDeletedBlockTable, ex.Message);
|
|
}
|
|
|
|
base.Dispose();
|
|
}
|
|
|
|
public async Task<long> GetLastWrittenDBlockVolumeSize()
|
|
{
|
|
using (var cmd = m_connection.CreateCommand(m_rtr))
|
|
return await cmd.SetCommandAndParameters(@"
|
|
SELECT ""Size""
|
|
FROM ""RemoteVolume""
|
|
WHERE
|
|
""State"" = @State
|
|
AND ""Type"" = @Type
|
|
ORDER BY ""ID"" DESC
|
|
LIMIT 1
|
|
")
|
|
.SetParameterValue("@State", RemoteVolumeState.Uploaded.ToString())
|
|
.SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString())
|
|
.ExecuteScalarInt64Async(-1);
|
|
}
|
|
|
|
private async Task<long> GetPreviousFilesetID(SqliteCommand cmd)
|
|
{
|
|
return await GetPreviousFilesetID(cmd, OperationTimestamp, m_filesetId);
|
|
}
|
|
|
|
private async Task<long> GetPreviousFilesetID(SqliteCommand cmd, DateTime timestamp, long filesetid)
|
|
{
|
|
return await cmd
|
|
.SetTransaction(m_rtr)
|
|
.SetCommandAndParameters(@"
|
|
SELECT ""ID""
|
|
FROM ""Fileset""
|
|
WHERE
|
|
""Timestamp"" < @Timestamp
|
|
AND ""ID"" != @FilesetId
|
|
ORDER BY ""Timestamp"" DESC
|
|
")
|
|
.SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(timestamp))
|
|
.SetParameterValue("@FilesetId", filesetid)
|
|
.ExecuteScalarInt64Async(-1);
|
|
}
|
|
|
|
internal async Task<Tuple<long, long>> GetLastBackupFileCountAndSize()
|
|
{
|
|
using (var cmd = m_connection.CreateCommand(m_rtr))
|
|
{
|
|
var lastFilesetId = await cmd.ExecuteScalarInt64Async(@"
|
|
SELECT ""ID""
|
|
FROM ""Fileset""
|
|
ORDER BY ""Timestamp"" DESC
|
|
LIMIT 1
|
|
");
|
|
|
|
var count = await cmd.SetCommandAndParameters(@"
|
|
SELECT COUNT(*)
|
|
FROM ""FileLookup""
|
|
INNER JOIN ""FilesetEntry""
|
|
ON ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID""
|
|
WHERE
|
|
""FilesetEntry"".""FilesetID"" = @FilesetId
|
|
AND ""FileLookup"".""BlocksetID"" NOT IN (
|
|
@FolderBlocksetId,
|
|
@SymlinkBlocksetId
|
|
)
|
|
")
|
|
.SetParameterValue("@FilesetId", lastFilesetId)
|
|
.SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID)
|
|
.SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID)
|
|
.ExecuteScalarInt64Async(-1);
|
|
|
|
var size = await cmd.SetCommandAndParameters(@"
|
|
SELECT SUM(""Blockset"".""Length"")
|
|
FROM
|
|
""FileLookup"",
|
|
""FilesetEntry"",
|
|
""Blockset""
|
|
WHERE
|
|
""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND
|
|
""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" AND
|
|
""FilesetEntry"".""FilesetID"" = @FilesetId AND
|
|
""FileLookup"".""BlocksetID"" NOT IN (
|
|
@FolderBlocksetId,
|
|
@SymlinkBlocksetId
|
|
)
|
|
")
|
|
.SetParameterValue("@FilesetId", lastFilesetId)
|
|
.SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID)
|
|
.SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID)
|
|
.ExecuteScalarInt64Async(-1);
|
|
|
|
return new Tuple<long, long>(count, size);
|
|
}
|
|
}
|
|
|
|
internal async Task UpdateChangeStatistics(BackupResults results)
|
|
{
|
|
using (var cmd = m_connection.CreateCommand(m_rtr))
|
|
{
|
|
var prevFileSetId = await GetPreviousFilesetID(cmd);
|
|
await ChangeStatistics.UpdateChangeStatistics(cmd, results, m_filesetId, prevFileSetId);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Populates FilesetEntry table with files from previous fileset, which aren't
|
|
/// yet part of the new fileset, and which aren't on the (optional) list of <c>deleted</c> paths.
|
|
/// </summary>
|
|
/// <param name="transaction">Transaction</param>
|
|
/// <param name="deleted">List of deleted paths, or null</param>
|
|
public async Task AppendFilesFromPreviousSet(IEnumerable<string>? deleted = null)
|
|
{
|
|
await AppendFilesFromPreviousSet(deleted, m_filesetId, -1, OperationTimestamp);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Populates FilesetEntry table with files from previous fileset, which aren't
|
|
/// yet part of the new fileset, and which aren't on the (optional) list of <c>deleted</c> paths.
|
|
/// </summary>
|
|
/// <param name="transaction">Transaction</param>
|
|
/// <param name="deleted">List of deleted paths, or null</param>
|
|
/// <param name="filesetid">Current file-set ID</param>
|
|
/// <param name="prevId">Source file-set ID</param>
|
|
/// <param name="timestamp">If <c>filesetid</c> == -1, used to locate previous file-set</param>
|
|
public async Task AppendFilesFromPreviousSet(IEnumerable<string>? deleted, long filesetid, long prevId, DateTime timestamp)
|
|
{
|
|
using (var cmd = m_connection.CreateCommand())
|
|
using (var cmdDelete = m_connection.CreateCommand())
|
|
{
|
|
long lastFilesetId = prevId < 0 ? await GetPreviousFilesetID(cmd, timestamp, filesetid) : prevId;
|
|
|
|
await cmd.SetTransaction(m_rtr)
|
|
.SetCommandAndParameters(@"
|
|
INSERT INTO ""FilesetEntry"" (
|
|
""FilesetID"",
|
|
""FileID"",
|
|
""Lastmodified""
|
|
)
|
|
SELECT
|
|
@CurrentFilesetId AS ""FilesetID"",
|
|
""FileID"",
|
|
""Lastmodified""
|
|
FROM (
|
|
SELECT DISTINCT
|
|
""FilesetID"",
|
|
""FileID"",
|
|
""Lastmodified""
|
|
FROM ""FilesetEntry""
|
|
WHERE
|
|
""FilesetID"" = @PreviousFilesetId
|
|
AND ""FileID"" NOT IN (
|
|
SELECT ""FileID""
|
|
FROM ""FilesetEntry""
|
|
WHERE ""FilesetID"" = @CurrentFilesetId
|
|
)
|
|
)
|
|
")
|
|
.SetParameterValue("@CurrentFilesetId", filesetid)
|
|
.SetParameterValue("@PreviousFilesetId", lastFilesetId)
|
|
.ExecuteNonQueryAsync(m_logQueries);
|
|
|
|
if (deleted != null)
|
|
{
|
|
using var tmplist = await TemporaryDbValueList.CreateAsync(this, deleted);
|
|
await (await cmdDelete.SetTransaction(m_rtr)
|
|
.SetCommandAndParameters(@"
|
|
DELETE FROM ""FilesetEntry""
|
|
WHERE
|
|
""FilesetID"" = @FilesetId
|
|
AND ""FileID"" IN (
|
|
SELECT ""ID""
|
|
FROM ""File""
|
|
WHERE ""Path"" IN (@Paths)
|
|
)
|
|
")
|
|
.SetParameterValue("@FilesetId", filesetid)
|
|
.ExpandInClauseParameterAsync("@Paths", tmplist))
|
|
.ExecuteNonQueryAsync(m_logQueries);
|
|
}
|
|
|
|
await m_rtr.CommitAsync();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Populates FilesetEntry table with files from previous fileset, which aren't
|
|
/// yet part of the new fileset, and which aren't excluded by the (optional) exclusion
|
|
/// predicate.
|
|
/// </summary>
|
|
/// <param name="transaction">Transaction</param>
|
|
/// <param name="exclusionPredicate">Optional exclusion predicate (true = exclude file)</param>
|
|
public async Task AppendFilesFromPreviousSetWithPredicate(Func<string, long, bool> exclusionPredicate)
|
|
{
|
|
await AppendFilesFromPreviousSetWithPredicate(exclusionPredicate, m_filesetId, -1, OperationTimestamp);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Populates FilesetEntry table with files from previous fileset, which aren't
|
|
/// yet part of the new fileset, and which aren't excluded by the (optional) exclusion
|
|
/// predicate.
|
|
/// </summary>
|
|
/// <param name="transaction">Transaction</param>
|
|
/// <param name="exclusionPredicate">Optional exclusion predicate (true = exclude file)</param>
|
|
/// <param name="fileSetId">Current fileset ID</param>
|
|
/// <param name="prevFileSetId">Source fileset ID</param>
|
|
/// <param name="timestamp">If <c>prevFileSetId</c> == -1, used to locate previous fileset</param>
|
|
public async Task AppendFilesFromPreviousSetWithPredicate(Func<string, long, bool> exclusionPredicate, long fileSetId, long prevFileSetId, DateTime timestamp)
|
|
{
|
|
if (exclusionPredicate == null)
|
|
{
|
|
await AppendFilesFromPreviousSet(null, fileSetId, prevFileSetId, timestamp);
|
|
return;
|
|
}
|
|
|
|
using (var cmd = m_connection.CreateCommand())
|
|
using (var cmdDelete = m_connection.CreateCommand())
|
|
{
|
|
long lastFilesetId = prevFileSetId < 0 ? await GetPreviousFilesetID(cmd, timestamp, fileSetId) : prevFileSetId;
|
|
|
|
// copy entries from previous file set into a temporary table, except those file IDs already added by the current backup
|
|
var tempFileSetTable = "FilesetEntry-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
|
|
await cmd
|
|
.SetTransaction(m_rtr)
|
|
.SetCommandAndParameters($@"
|
|
CREATE TEMPORARY TABLE ""{tempFileSetTable}"" AS
|
|
SELECT
|
|
""FileID"",
|
|
""Lastmodified""
|
|
FROM (
|
|
SELECT DISTINCT
|
|
""FilesetID"",
|
|
""FileID"",
|
|
""Lastmodified""
|
|
FROM ""FilesetEntry""
|
|
WHERE
|
|
""FilesetID"" = @PreviousFilesetId
|
|
AND ""FileID"" NOT IN (
|
|
SELECT ""FileID""
|
|
FROM ""FilesetEntry""
|
|
WHERE ""FilesetID"" = @CurrentFilesetId
|
|
)
|
|
)
|
|
")
|
|
.SetParameterValue("@PreviousFilesetId", lastFilesetId)
|
|
.SetParameterValue("@CurrentFilesetId", fileSetId)
|
|
.ExecuteNonQueryAsync();
|
|
|
|
// now we need to remove, from the above, any entries that were enumerated by the
|
|
// UNC-driven backup
|
|
cmdDelete.SetTransaction(m_rtr)
|
|
.SetCommandAndParameters($@"
|
|
DELETE FROM ""{tempFileSetTable}""
|
|
WHERE ""FileID"" = @FileId
|
|
");
|
|
|
|
// enumerate files from new temporary file set, and remove any entries handled by UNC
|
|
cmd.SetCommandAndParameters($@"
|
|
SELECT
|
|
f.""Path"",
|
|
fs.""FileID"",
|
|
fs.""Lastmodified"",
|
|
COALESCE(bs.""Length"", -1)
|
|
FROM (
|
|
SELECT DISTINCT
|
|
""FileID"",
|
|
""Lastmodified""
|
|
FROM ""{tempFileSetTable}""
|
|
) AS fs
|
|
LEFT JOIN ""File"" AS f
|
|
ON fs.""FileID"" = f.""ID""
|
|
LEFT JOIN ""Blockset"" AS bs
|
|
ON f.""BlocksetID"" = bs.""ID"";
|
|
");
|
|
|
|
await foreach (var row in cmd.ExecuteReaderEnumerableAsync())
|
|
{
|
|
var path = row.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for path");
|
|
var size = row.ConvertValueToInt64(3);
|
|
|
|
if (exclusionPredicate(path, size))
|
|
await cmdDelete.SetParameterValue("@FileId", row.ConvertValueToInt64(1))
|
|
.ExecuteNonQueryAsync();
|
|
}
|
|
|
|
// now copy the temporary table into the FileSetEntry table
|
|
await cmd.SetCommandAndParameters($@"
|
|
INSERT INTO ""FilesetEntry"" (
|
|
""FilesetID"",
|
|
""FileID"",
|
|
""Lastmodified""
|
|
)
|
|
SELECT
|
|
@FilesetId,
|
|
""FileID"",
|
|
""Lastmodified""
|
|
FROM ""{tempFileSetTable}""
|
|
")
|
|
.SetParameterValue("@FilesetId", fileSetId)
|
|
.ExecuteNonQueryAsync();
|
|
|
|
await m_rtr.CommitAsync();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates a timestamped backup operation to correctly associate the fileset with the time it was created.
|
|
/// </summary>
|
|
/// <param name="volumeid">The ID of the fileset volume to update</param>
|
|
/// <param name="timestamp">The timestamp of the operation to create</param>
|
|
/// <param name="transaction">An optional external transaction</param>
|
|
public override async Task<long> CreateFileset(long volumeid, DateTime timestamp)
|
|
{
|
|
return m_filesetId = await base.CreateFileset(volumeid, timestamp);
|
|
}
|
|
|
|
public async Task<IEnumerable<string>> GetTemporaryFilelistVolumeNames(bool latestOnly)
|
|
{
|
|
var incompleteFilesetIDs = GetIncompleteFilesets().OrderBy(x => x.Value).Select(x => x.Key);
|
|
|
|
if (!await incompleteFilesetIDs.AnyAsync())
|
|
return [];
|
|
|
|
if (latestOnly)
|
|
incompleteFilesetIDs = new long[] { await incompleteFilesetIDs.LastAsync() }.ToAsyncEnumerable();
|
|
|
|
var volumeNames = new List<string>();
|
|
await foreach (var filesetID in incompleteFilesetIDs)
|
|
volumeNames.Add((await GetRemoteVolumeFromFilesetID(filesetID)).Name);
|
|
|
|
return volumeNames;
|
|
}
|
|
|
|
public async IAsyncEnumerable<string> GetMissingIndexFiles()
|
|
{
|
|
using var cmd = m_connection.CreateCommand(m_rtr)
|
|
.SetCommandAndParameters(@"
|
|
SELECT ""Name""
|
|
FROM ""RemoteVolume""
|
|
WHERE
|
|
""Type"" = @Type
|
|
AND NOT ""ID"" IN (
|
|
SELECT ""BlockVolumeID""
|
|
FROM ""IndexBlockLink""
|
|
)
|
|
AND ""State"" IN (@States)
|
|
")
|
|
.SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString())
|
|
.ExpandInClauseParameterAsync("@States", [
|
|
RemoteVolumeState.Uploaded.ToString(),
|
|
RemoteVolumeState.Verified.ToString()
|
|
]);
|
|
|
|
using (var rd = await cmd.ExecuteReaderAsync())
|
|
while (await rd.ReadAsync())
|
|
yield return rd.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for volume name");
|
|
}
|
|
|
|
public async Task MoveBlockToVolume(string blockkey, long size, long sourcevolumeid, long targetvolumeid)
|
|
{
|
|
using (var cmd = m_connection.CreateCommand(m_rtr))
|
|
{
|
|
var c = await cmd.SetCommandAndParameters(@"
|
|
UPDATE ""Block""
|
|
SET ""VolumeID"" = @NewVolumeId
|
|
WHERE
|
|
""Hash"" = @Hash
|
|
AND ""Size"" = @Size
|
|
AND ""VolumeID"" = @PreviousVolumeId
|
|
")
|
|
.SetParameterValue("@NewVolumeId", targetvolumeid)
|
|
.SetParameterValue("@Hash", blockkey)
|
|
.SetParameterValue("@Size", size)
|
|
.SetParameterValue("@PreviousVolumeId", sourcevolumeid)
|
|
.ExecuteNonQueryAsync();
|
|
|
|
if (c != 1)
|
|
throw new Exception($"Failed to move block {blockkey}:{size} from volume {sourcevolumeid}, count: {c}");
|
|
}
|
|
}
|
|
|
|
public async Task SafeDeleteRemoteVolume(string name)
|
|
{
|
|
var volumeid = await GetRemoteVolumeID(name);
|
|
|
|
using (var cmd = m_connection.CreateCommand(m_rtr))
|
|
{
|
|
var c = await cmd.SetCommandAndParameters(@"
|
|
SELECT COUNT(*)
|
|
FROM ""Block""
|
|
WHERE ""VolumeID"" = @VolumeId
|
|
")
|
|
.SetParameterValue("@VolumeId", volumeid)
|
|
.ExecuteScalarInt64Async(-1);
|
|
|
|
if (c != 0)
|
|
throw new Exception($"Failed to safe-delete volume {name}, blocks: {c}");
|
|
|
|
await RemoveRemoteVolume(name);
|
|
}
|
|
}
|
|
|
|
public async Task<string[]> GetBlocklistHashes(string name)
|
|
{
|
|
var volumeid = GetRemoteVolumeID(name);
|
|
using (var cmd = m_connection.CreateCommand(m_rtr))
|
|
{
|
|
// Grab the strings and return as array to avoid concurrent access to the IEnumerable
|
|
cmd.SetCommandAndParameters(@"
|
|
SELECT DISTINCT ""Block"".""Hash""
|
|
FROM ""Block""
|
|
WHERE
|
|
""Block"".""VolumeID"" = @VolumeId
|
|
AND ""Block"".""Hash"" IN (
|
|
SELECT ""Hash""
|
|
FROM ""BlocklistHash""
|
|
)
|
|
")
|
|
.SetParameterValue("@VolumeId", volumeid);
|
|
|
|
return await cmd.ExecuteReaderEnumerableAsync()
|
|
.Select(x => x.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for blocklist hash"))
|
|
.ToArrayAsync();
|
|
}
|
|
}
|
|
|
|
public async Task<string?> GetFirstPath()
|
|
{
|
|
using (var cmd = m_connection.CreateCommand(m_rtr))
|
|
{
|
|
var v0 = await cmd.ExecuteScalarAsync(@"
|
|
SELECT ""Path""
|
|
FROM ""File""
|
|
ORDER BY LENGTH(""Path"") DESC
|
|
LIMIT 1
|
|
");
|
|
|
|
if (v0 == null || v0 == DBNull.Value)
|
|
return null;
|
|
|
|
return v0.ToString();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Retrieves change journal data for file set
|
|
/// </summary>
|
|
/// <param name="fileSetId">Fileset-ID</param>
|
|
public async IAsyncEnumerable<Interface.USNJournalDataEntry> GetChangeJournalData(long fileSetId)
|
|
{
|
|
var data = new List<Interface.USNJournalDataEntry>();
|
|
|
|
using var cmd = m_connection.CreateCommand(@"
|
|
SELECT
|
|
""VolumeName"",
|
|
""JournalID"",
|
|
""NextUSN"",
|
|
""ConfigHash""
|
|
FROM ""ChangeJournalData""
|
|
WHERE ""FilesetID"" = @FilesetId
|
|
")
|
|
.SetTransaction(m_rtr)
|
|
.SetParameterValue("@FilesetId", fileSetId);
|
|
|
|
using (var rd = await cmd.ExecuteReaderAsync())
|
|
{
|
|
while (await rd.ReadAsync())
|
|
{
|
|
yield return new Interface.USNJournalDataEntry
|
|
{
|
|
Volume = rd.ConvertValueToString(0),
|
|
JournalId = rd.ConvertValueToInt64(1),
|
|
NextUsn = rd.ConvertValueToInt64(2),
|
|
ConfigHash = rd.ConvertValueToString(3)
|
|
};
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Adds NTFS change journal data for file set and volume
|
|
/// </summary>
|
|
/// <param name="data">Data to add</param>
|
|
/// <param name="transaction">An optional external transaction</param>
|
|
public async Task CreateChangeJournalData(IEnumerable<Interface.USNJournalDataEntry> data)
|
|
{
|
|
foreach (var entry in data)
|
|
{
|
|
using (var cmd = m_connection.CreateCommand(m_rtr))
|
|
{
|
|
var c = await cmd.SetCommandAndParameters(@"
|
|
INSERT INTO ""ChangeJournalData"" (
|
|
""FilesetID"",
|
|
""VolumeName"",
|
|
""JournalID"",
|
|
""NextUSN"",
|
|
""ConfigHash""
|
|
)
|
|
VALUES (
|
|
@FilesetId,
|
|
@VolumeName,
|
|
@JournalId,
|
|
@NextUsn,
|
|
@ConfigHash
|
|
);
|
|
")
|
|
.SetParameterValue("@FilesetId", m_filesetId)
|
|
.SetParameterValue("@VolumeName", entry.Volume)
|
|
.SetParameterValue("@JournalId", entry.JournalId)
|
|
.SetParameterValue("@NextUsn", entry.NextUsn)
|
|
.SetParameterValue("@ConfigHash", entry.ConfigHash)
|
|
.ExecuteNonQueryAsync();
|
|
|
|
if (c != 1)
|
|
throw new Exception("Unable to add change journal entry");
|
|
}
|
|
}
|
|
|
|
await m_rtr.CommitAsync();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Adds NTFS change journal data for file set and volume
|
|
/// </summary>
|
|
/// <param name="data">Data to add</param>
|
|
/// <param name="fileSetId">Existing file set to update</param>
|
|
/// <param name="transaction">An optional external transaction</param>
|
|
public async Task UpdateChangeJournalData(IEnumerable<Interface.USNJournalDataEntry> data, long fileSetId)
|
|
{
|
|
foreach (var entry in data)
|
|
{
|
|
using (var cmd = m_connection.CreateCommand())
|
|
{
|
|
await cmd.SetCommandAndParameters(@"
|
|
UPDATE ""ChangeJournalData""
|
|
SET ""NextUSN"" = @NextUsn
|
|
WHERE
|
|
""FilesetID"" = @FilesetId
|
|
AND ""VolumeName"" = @VolumeName
|
|
AND ""JournalID"" = @JournalId;
|
|
")
|
|
.SetTransaction(m_rtr)
|
|
.SetParameterValue("@NextUsn", entry.NextUsn)
|
|
.SetParameterValue("@FilesetId", fileSetId)
|
|
.SetParameterValue("@VolumeName", entry.Volume)
|
|
.SetParameterValue("@JournalId", entry.JournalId)
|
|
.ExecuteNonQueryAsync();
|
|
}
|
|
}
|
|
|
|
await m_rtr.CommitAsync();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Checks if a blocklist hash is known
|
|
/// </summary>
|
|
/// <param name="hash">The hash to check</param>
|
|
/// <param name="transaction">An optional external transaction</param>
|
|
/// <returns>True if the hash is known, false otherwise</returns>
|
|
public async Task<bool> IsBlocklistHashKnown(string hash)
|
|
{
|
|
var res = await m_getfirstfilesetwithblockinblockset
|
|
.SetTransaction(m_rtr)
|
|
.SetParameterValue("@Hash", hash)
|
|
.ExecuteScalarInt64Async();
|
|
|
|
if (res != -1 && res != m_filesetId)
|
|
return true;
|
|
else
|
|
return !m_blocklistHashes.Add(hash);
|
|
}
|
|
}
|
|
}
|