haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
edf1d93f564a002fda212cb3a57dd7cd384e3a2f
duplicati/Duplicati/Library/Main/Database/LocalDatabase.cs
T

2571 lines
109 KiB
C#
Raw Blame History

// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Collections.Generic;
using System.Data;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading.Tasks;
using Duplicati.Library.Modules.Builtin.ResultSerialization;
using Duplicati.Library.Utility;
using Duplicati.Library.Interface;
using Microsoft.Data.Sqlite;
// Expose internal classes to UnitTests, so that Database classes can be tested
[assembly: InternalsVisibleTo("Duplicati.UnitTest")]
namespace Duplicati.Library.Main.Database
{
internal class LocalDatabase : IDisposable
{
/// <summary>
/// The tag used for logging
/// </summary>
private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalDatabase));
/// <summary>
/// The chunk size for batch operations
/// </summary>
/// <remarks>SQLite has a limit of 999 parameters in a single statement</remarks>
public const int CHUNK_SIZE = 128;
// All of the required fields have been set to null! to ignore the compiler warning, as they will be initialized in the Create* factory functions.
protected SqliteConnection m_connection = null!;
protected long m_operationid = -1;
protected long m_pagecachesize;
private bool m_hasExecutedVacuum;
// TODO use this rather than passing down transactions.
// TODO dispose should check this, and report if it was not disposed prior to calling dispose. Maybe the rtr itself should check this during dispose? And just throw a warning during runtime.
protected ReusableTransaction m_rtr;
public ReusableTransaction Transaction { get { return m_rtr; } }
private SqliteCommand m_updateremotevolumeCommand = null!;
private SqliteCommand m_selectremotevolumesCommand = null!;
private SqliteCommand m_selectremotevolumeCommand = null!;
private SqliteCommand m_removeremotevolumeCommand = null!;
private SqliteCommand m_removedeletedremotevolumeCommand = null!;
private SqliteCommand m_selectremotevolumeIdCommand = null!;
private SqliteCommand m_createremotevolumeCommand = null!;
private SqliteCommand m_selectduplicateRemoteVolumesCommand = null!;
private SqliteCommand m_insertlogCommand = null!;
private SqliteCommand m_insertremotelogCommand = null!;
private SqliteCommand m_insertIndexBlockLink = null!;
private SqliteCommand m_findpathprefixCommand = null!;
private SqliteCommand m_insertpathprefixCommand = null!;
public const long FOLDER_BLOCKSET_ID = -100;
public const long SYMLINK_BLOCKSET_ID = -200;
public DateTime OperationTimestamp { get; private set; }
internal SqliteConnection Connection { get { return m_connection; } }
public bool IsDisposed { get; private set; }
public bool ShouldCloseConnection { get; set; }
// Constructor is private to force use of CreateLocalDatabaseAsync
protected LocalDatabase() { }
// Arguments are not used, but are required to match the constructor signatures
[Obsolete("Calling the constructor will throw an exception. Use the CreateLocalDatabaseAsync or CreateLocalDatabase functions instead")]
public LocalDatabase(object? _ignore1 = null, object? _ignore2 = null, object? _ignore3 = null, object? _ignore4 = null)
{
throw new NotImplementedException("Use the CreateLocalDatabaseAsync or CreateLocalDatabase functions instead");
}
protected static SqliteConnection CreateConnection(string path, long pagecachesize)
{
return CreateConnectionAsync(path, pagecachesize).Await();
}
protected static async Task<SqliteConnection> CreateConnectionAsync(string path, long pagecachesize)
{
path = Path.GetFullPath(path);
if (!Directory.Exists(Path.GetDirectoryName(path)))
Directory.CreateDirectory(Path.GetDirectoryName(path) ?? throw new DirectoryNotFoundException("Path was a root folder."));
var c = await SQLiteHelper.SQLiteLoader.LoadConnectionAsync(path, pagecachesize);
try
{
SQLiteHelper.DatabaseUpgrader.UpgradeDatabase(c, path, typeof(DatabaseSchemaMarker));
}
catch
{
//Don't leak database connections when something goes wrong
await c.DisposeAsync();
throw;
}
return c;
}
// TODO Create these once the signature is stable.
// public static LocalDatabase CreateLocalDatabase(string path, string operation, bool shouldclose, long pagecachesize)
// {
// return CreateLocalDatabaseAsync(path, operation, shouldclose, pagecachesize).Await();
// }
// public static LocalDatabase CreateLocalDatabase(LocalDatabase db)
// {
// return CreateLocalDatabaseAsync(db).Await();
// }
// public static LocalDatabase CreateLocalDatabase(SqliteConnection connection, string operation)
// {
// return CreateLocalDatabaseAsync(connection, operation).Await();
// }
// private static LocalDatabase CreateLocalDatabase(SqliteConnection connection)
// {
// return CreateLocalDatabaseAsync(connection).Await();
// }
public static async Task<LocalDatabase> CreateLocalDatabaseAsync(string path, string operation, bool shouldclose, long pagecachesize, LocalDatabase? db = null)
{
db ??= new LocalDatabase();
var connection = await CreateConnectionAsync(path, pagecachesize);
db = await CreateLocalDatabaseAsync(connection, operation, db);
db.ShouldCloseConnection = shouldclose;
db.m_pagecachesize = pagecachesize;
return db;
}
public static async Task<LocalDatabase> CreateLocalDatabaseAsync(LocalDatabase dbparent, LocalDatabase? dbnew = null)
{
dbnew ??= new LocalDatabase();
dbnew = await CreateLocalDatabaseAsync(dbparent.m_connection, dbnew);
dbnew.OperationTimestamp = dbparent.OperationTimestamp;
dbnew.m_connection = dbparent.m_connection;
dbnew.m_operationid = dbparent.m_operationid;
dbnew.m_pagecachesize = dbparent.m_pagecachesize;
return dbnew;
}
public static async Task<LocalDatabase> CreateLocalDatabaseAsync(SqliteConnection connection, string operation, LocalDatabase? dbnew = null)
{
dbnew ??= new LocalDatabase();
dbnew = await CreateLocalDatabaseAsync(connection, dbnew);
dbnew.OperationTimestamp = DateTime.UtcNow;
if (dbnew.m_connection.State != ConnectionState.Open)
await dbnew.m_connection.OpenAsync();
using var cmd = dbnew.m_connection.CreateCommand();
using var transaction = dbnew.m_connection.BeginTransaction();
cmd.Transaction = transaction;
if (operation != null)
{
dbnew.m_operationid = await cmd.SetCommandAndParameters(@"
INSERT INTO ""Operation"" (
""Description"",
""Timestamp""
)
VALUES (
@Description,
@Timestamp
);
SELECT last_insert_rowid();
")
.SetParameterValue("@Description", operation)
.SetParameterValue("@Timestamp", dbnew.OperationTimestamp)
.ExecuteScalarInt64Async(-1);
}
else
{
// Get last operation
using var rd = await cmd.ExecuteReaderAsync(@"
SELECT
""ID"",
""Timestamp""
FROM ""Operation""
ORDER BY ""Timestamp""
DESC LIMIT 1
");
if (!await rd.ReadAsync())
throw new Exception("LocalDatabase does not contain a previous operation.");
dbnew.m_operationid = rd.ConvertValueToInt64(0);
dbnew.OperationTimestamp = ParseFromEpochSeconds(rd.ConvertValueToInt64(1));
}
return dbnew;
}
private static async Task<LocalDatabase> CreateLocalDatabaseAsync(SqliteConnection connection, LocalDatabase? dbnew = null)
{
dbnew ??= new LocalDatabase();
dbnew.m_connection = connection;
var selectremotevolumes_sql = @"
SELECT
""ID"",
""Name"",
""Type"",
""Size"",
""Hash"",
""State"",
""DeleteGraceTime"",
""ArchiveTime""
FROM ""Remotevolume""
";
dbnew.m_insertlogCommand = await connection.CreateCommandAsync(@"
INSERT INTO ""LogData"" (
""OperationID"",
""Timestamp"",
""Type"",
""Message"",
""Exception""
)
VALUES (
@OperationID,
@Timestamp,
@Type,
@Message,
@Exception
)
");
dbnew.m_insertremotelogCommand = await connection.CreateCommandAsync(@"
INSERT INTO ""RemoteOperation"" (
""OperationID"",
""Timestamp"",
""Operation"",
""Path"",
""Data""
)
VALUES (
@OperationID,
@Timestamp,
@Operation,
@Path,
@Data
)
");
dbnew.m_updateremotevolumeCommand = await connection.CreateCommandAsync(@"
UPDATE ""Remotevolume""
SET
""OperationID"" = @OperationID,
""State"" = @State,
""Hash"" = @Hash,
""Size"" = @Size
WHERE ""Name"" = @Name
");
dbnew.m_selectremotevolumesCommand = await connection.CreateCommandAsync(selectremotevolumes_sql);
dbnew.m_selectremotevolumeCommand = await connection.CreateCommandAsync(@$"
{selectremotevolumes_sql}
WHERE ""Name"" = @Name
");
dbnew.m_selectduplicateRemoteVolumesCommand = await connection.CreateCommandAsync($@"
SELECT DISTINCT ""Name"", ""State""
FROM ""Remotevolume""
WHERE
""Name"" IN (
SELECT ""Name""
FROM ""Remotevolume""
WHERE ""State"" IN ('{RemoteVolumeState.Deleted}', '{RemoteVolumeState.Deleting}')
)
AND NOT ""State"" IN ('{RemoteVolumeState.Deleted}', '{RemoteVolumeState.Deleting}')
");
dbnew.m_removeremotevolumeCommand = await connection.CreateCommandAsync(@"
DELETE FROM ""Remotevolume""
WHERE
""Name"" = @Name
AND (
""DeleteGraceTime"" < @Now
OR ""State"" != @State
)
");
// >12 is to handle removal of old records that were in ticks
dbnew.m_removedeletedremotevolumeCommand = await connection.CreateCommandAsync($@"
DELETE FROM ""Remotevolume""
WHERE
""State"" == '{RemoteVolumeState.Deleted}'
AND (
""DeleteGraceTime"" < @Now
OR LENGTH(""DeleteGraceTime"") > 12
)
");
dbnew.m_selectremotevolumeIdCommand = await connection.CreateCommandAsync(@"
SELECT ""ID""
FROM ""Remotevolume""
WHERE ""Name"" = @Name
");
dbnew.m_createremotevolumeCommand = await connection.CreateCommandAsync(@"
INSERT INTO ""Remotevolume"" (
""OperationID"",
""Name"",
""Type"",
""State"",
""Size"",
""VerificationCount"",
""DeleteGraceTime"",
""ArchiveTime""
)
VALUES (
@OperationID,
@Name,
@Type,
@State,
@Size,
@VerificationCount,
@DeleteGraceTime,
@ArchiveTime
);
SELECT last_insert_rowid();
");
dbnew.m_insertIndexBlockLink = await connection.CreateCommandAsync(@"
INSERT INTO ""IndexBlockLink"" (
""IndexVolumeID"",
""BlockVolumeID""
)
VALUES (
@IndexVolumeId,
@BlockVolumeId
)
");
dbnew.m_findpathprefixCommand = await connection.CreateCommandAsync(@"
SELECT ""ID""
FROM ""PathPrefix""
WHERE ""Prefix"" = @Prefix
");
dbnew.m_insertpathprefixCommand = await connection.CreateCommandAsync(@"
INSERT INTO ""PathPrefix"" (""Prefix"")
VALUES (@Prefix);
SELECT last_insert_rowid();
");
return dbnew;
}
/// <summary>
/// TODO Remove from here and call the utility function directly - or maybe rename this so that the utility function makes more sense?
/// Creates a DateTime instance by adding the specified number of seconds to the EPOCH value
/// </summary>
public static DateTime ParseFromEpochSeconds(long seconds)
{
return Library.Utility.Utility.EPOCH.AddSeconds(seconds);
}
public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash)
{
await UpdateRemoteVolume(name, state, size, hash, false);
}
public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, bool suppressCleanup)
{
await UpdateRemoteVolume(name, state, size, hash, suppressCleanup, new TimeSpan(0), null);
}
public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, bool suppressCleanup, TimeSpan deleteGraceTime, bool? setArchived)
{
var c = await m_updateremotevolumeCommand.SetTransaction(m_rtr)
.SetParameterValue("@OperationID", m_operationid)
.SetParameterValue("@State", state.ToString())
.SetParameterValue("@Hash", hash)
.SetParameterValue("@Size", size)
.SetParameterValue("@Name", name)
.ExecuteNonQueryAsync();
if (c != 1)
{
throw new Exception($"Unexpected number of remote volumes detected: {c}!");
}
if (deleteGraceTime.Ticks > 0)
{
using var cmd = await m_connection.CreateCommandAsync(@"
UPDATE ""RemoteVolume""
SET ""DeleteGraceTime"" = @DeleteGraceTime
WHERE ""Name"" = @Name
");
c = await cmd.SetTransaction(m_rtr)
.SetParameterValue("@DeleteGraceTime", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow + deleteGraceTime))
.SetParameterValue("@Name", name)
.ExecuteNonQueryAsync();
if (c != 1)
throw new Exception($"Unexpected number of updates when recording remote volume updates: {c}!");
}
if (setArchived.HasValue)
{
using var cmd = await m_connection.CreateCommandAsync(@"
UPDATE ""RemoteVolume""
SET ""ArchiveTime"" = @ArchiveTime
WHERE ""Name"" = @Name
");
c = await cmd.SetTransaction(m_rtr)
.SetParameterValue("@ArchiveTime", setArchived.Value ? Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow) : 0)
.SetParameterValue("@Name", name)
.ExecuteNonQueryAsync();
if (c != 1)
throw new Exception($"Unexpected number of updates when recording remote volume archive-time updates: {c}!");
}
if (!suppressCleanup && state == RemoteVolumeState.Deleted)
{
await RemoveRemoteVolume(name);
}
}
public async IAsyncEnumerable<KeyValuePair<long, DateTime>> FilesetTimes()
{
using var cmd = await m_connection.CreateCommandAsync(@"
SELECT ""ID"", ""Timestamp""
FROM ""Fileset""
ORDER BY ""Timestamp"" DESC
");
using var rd = await cmd.ExecuteReaderAsync();
while (await rd.ReadAsync())
yield return new KeyValuePair<long, DateTime>(rd.ConvertValueToInt64(0), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime());
}
public async Task<(string Query, Dictionary<string, object?> Values)> GetFilelistWhereClause(DateTime time, long[] versions, IEnumerable<KeyValuePair<long, DateTime>>? filesetslist = null, bool singleTimeMatch = false)
{
KeyValuePair<long, DateTime>[] filesets;
if (filesetslist != null)
filesets = [.. filesetslist];
else
filesets = await FilesetTimes().ToArrayAsync();
var query = new StringBuilder();
var args = new Dictionary<string, object?>();
if (time.Ticks > 0 || (versions != null && versions.Length > 0))
{
var hasTime = false;
if (time.Ticks > 0)
{
if (time.Kind == DateTimeKind.Unspecified)
throw new Exception("Invalid DateTime given, must be either local or UTC");
query.Append(singleTimeMatch ? @" ""Timestamp"" = @Timestamp" : @" ""Timestamp"" <= @Timestamp");
// Make sure the resolution is the same (i.e. no milliseconds)
args.Add("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(time));
hasTime = true;
}
if (versions != null && versions.Length > 0)
{
var qs = new StringBuilder();
foreach (var v in versions)
{
if (v >= 0 && v < filesets.Length)
{
var argName = "@Fileset" + v;
args.Add(argName, filesets[v].Key);
qs.Append(argName);
qs.Append(',');
}
else
Logging.Log.WriteWarningMessage(LOGTAG, "SkipInvalidVersion", null, "Skipping invalid version: {0}", v);
}
if (qs.Length > 0)
{
if (hasTime)
query.Append(" OR ");
query.Append(@" ""ID"" IN (" + qs.ToString(0, qs.Length - 1) + ")");
}
}
if (query.Length > 0)
{
query.Insert(0, " WHERE ");
}
}
return (query.ToString(), args);
}
public async Task<long> GetRemoteVolumeID(string file)
{
return await m_selectremotevolumeIdCommand
.SetTransaction(m_rtr)
.SetParameterValue("@Name", file)
.ExecuteScalarInt64Async(-1);
}
public async IAsyncEnumerable<KeyValuePair<string, long>> GetRemoteVolumeIDs(IEnumerable<string> files)
{
using var cmd = await m_connection.CreateCommandAsync(@"
SELECT
""Name"",
""ID""
FROM ""RemoteVolume""
WHERE ""Name"" IN (@Name)
");
cmd.SetTransaction(m_rtr);
using var tmptable = await TemporaryDbValueList.CreateAsync(this, files);
await cmd.ExpandInClauseParameterAsync("@Name", tmptable);
using var rd = await cmd.ExecuteReaderAsync();
while (await rd.ReadAsync())
yield return new KeyValuePair<string, long>(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1));
}
public async Task<RemoteVolumeEntry> GetRemoteVolume(string file)
{
m_selectremotevolumeCommand
.SetTransaction(m_rtr)
.SetParameterValue("@Name", file);
using (var rd = await m_selectremotevolumeCommand.ExecuteReaderAsync())
if (await rd.ReadAsync())
return new RemoteVolumeEntry(
rd.ConvertValueToInt64(0),
rd.ConvertValueToString(1),
rd.ConvertValueToString(4),
rd.ConvertValueToInt64(3, -1),
(RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), rd.ConvertValueToString(2) ?? ""),
(RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.ConvertValueToString(5) ?? ""),
ParseFromEpochSeconds(rd.ConvertValueToInt64(6, 0)),
ParseFromEpochSeconds(rd.ConvertValueToInt64(7, 0))
);
return RemoteVolumeEntry.Empty;
}
public async IAsyncEnumerable<KeyValuePair<string, RemoteVolumeState>> DuplicateRemoteVolumes()
{
m_selectduplicateRemoteVolumesCommand.SetTransaction(m_rtr);
await foreach (var rd in m_selectduplicateRemoteVolumesCommand.ExecuteReaderEnumerableAsync())
{
yield return new KeyValuePair<string, RemoteVolumeState>(
rd.ConvertValueToString(0) ?? throw new Exception("Name was null"),
(RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.ConvertValueToString(1) ?? "")
);
}
}
public async IAsyncEnumerable<RemoteVolumeEntry> GetRemoteVolumes()
{
m_selectremotevolumesCommand.SetTransaction(m_rtr);
using var rd = await m_selectremotevolumesCommand.ExecuteReaderAsync();
while (await rd.ReadAsync())
{
yield return new RemoteVolumeEntry(
rd.ConvertValueToInt64(0),
rd.ConvertValueToString(1),
rd.ConvertValueToString(4),
rd.ConvertValueToInt64(3, -1),
(RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), rd.ConvertValueToString(2) ?? ""),
(RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.ConvertValueToString(5) ?? ""),
ParseFromEpochSeconds(rd.ConvertValueToInt64(6, 0)),
ParseFromEpochSeconds(rd.ConvertValueToInt64(7, 0))
);
}
}
/// <summary>
/// Log an operation performed on the remote backend
/// </summary>
/// <param name="operation">The operation performed</param>
/// <param name="path">The path involved</param>
/// <param name="data">Any data relating to the operation</param>
public async Task LogRemoteOperation(string operation, string path, string? data)
{
await m_insertremotelogCommand
.SetTransaction(m_rtr)
.SetParameterValue("@OperationID", m_operationid)
.SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow))
.SetParameterValue("@Operation", operation)
.SetParameterValue("@Path", path)
.SetParameterValue("@Data", data)
.ExecuteNonQueryAsync();
}
/// <summary>
/// Log a debug message
/// </summary>
/// <param name="type">The message type</param>
/// <param name="message">The message</param>
/// <param name="exception">An optional exception</param>
public async Task LogMessage(string type, string message, Exception? exception)
{
await m_insertlogCommand
.SetTransaction(m_rtr)
.SetParameterValue("@OperationID", m_operationid)
.SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow))
.SetParameterValue("@Type", type)
.SetParameterValue("@Message", message)
.SetParameterValue("@Exception", exception?.ToString())
.ExecuteNonQueryAsync();
}
public async Task UnlinkRemoteVolume(string name, RemoteVolumeState state)
{
using var cmd = await m_connection.CreateCommandAsync(@"
DELETE FROM ""RemoteVolume""
WHERE
""Name"" = @Name
AND ""State"" = @State
");
var c = await cmd.SetTransaction(m_rtr)
.SetParameterValue("@Name", name)
.SetParameterValue("@State", state.ToString())
.ExecuteNonQueryAsync();
if (c != 1)
throw new Exception($"Unexpected number of remote volumes deleted: {c}, expected {1}");
await m_rtr.CommitAsync();
}
public async Task RemoveRemoteVolume(string name)
{
await RemoveRemoteVolumes([name]);
}
public async Task RemoveRemoteVolumes(IEnumerable<string> names)
{
if (names == null || !names.Any()) return;
using var deletecmd = m_connection.CreateCommand(m_rtr);
string temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
var volidstable = "DelVolSetIds-" + temptransguid;
var blocksetidstable = "DelBlockSetIds-" + temptransguid;
var filesetidstable = "DelFilesetIds-" + temptransguid;
// Create and fill a temp table with the volids to delete. We avoid using too many parameters that way.
await deletecmd.ExecuteNonQueryAsync($@"
CREATE TEMP TABLE ""{volidstable}"" (
""ID"" INTEGER PRIMARY KEY
)
");
await deletecmd.SetCommandAndParameters($@"
INSERT OR IGNORE INTO ""{volidstable}""
SELECT ""ID""
FROM ""RemoteVolume""
WHERE ""Name"" IN (@VolumeNames)
")
.ExpandInClauseParameter("@VolumeNames", [.. names])
.ExecuteNonQueryAsync();
var volIdsSubQuery = $@"SELECT ""ID"" FROM ""{volidstable}"" ";
deletecmd.Parameters.Clear();
var bsIdsSubQuery = @$"
SELECT DISTINCT ""BlocksetEntry"".""BlocksetID""
FROM ""BlocksetEntry"", ""Block""
WHERE
""BlocksetEntry"".""BlockID"" = ""Block"".""ID""
AND ""Block"".""VolumeID"" IN ({volIdsSubQuery})
UNION ALL
SELECT DISTINCT ""BlocksetID""
FROM ""BlocklistHash""
WHERE ""Hash"" IN (
SELECT ""Hash""
FROM ""Block""
WHERE ""VolumeID"" IN ({volIdsSubQuery})
)
";
// Create a temporary table to cache subquery result, as it might take long (SQLite does not cache at all).
await deletecmd.ExecuteNonQueryAsync($@"
CREATE TEMP TABLE ""{blocksetidstable}"" (
""ID"" INTEGER PRIMARY KEY
)
");
await deletecmd.ExecuteNonQueryAsync($@"
INSERT OR IGNORE INTO ""{blocksetidstable}"" (""ID"")
{bsIdsSubQuery}
");
bsIdsSubQuery = $@"
SELECT DISTINCT ""ID""
FROM ""{blocksetidstable}""
";
deletecmd.Parameters.Clear();
// Create a temp table to associate metadata that is being deleted to a fileset
var metadataFilesetQuery = $@"
SELECT Metadataset.ID, FilesetEntry.FilesetD
FROM Metadataset
INNER JOIN FileLookup
ON FileLookup.MetadataID = Metadataset.ID
INNER JOIN FilesetEntry
ON FilesetEntry.FileID = FileLookup.ID
WHERE Metadataset.BlocksetID IN ({bsIdsSubQuery})
OR Metadataset.ID IN (
SELECT MetadataID
FROM FileLookup
WHERE BlocksetID IN ({bsIdsSubQuery})
)
";
var metadataFilesetTable = $"DelMetadataFilesetIds-{temptransguid}";
await deletecmd.ExecuteNonQueryAsync($@"
CREATE TEMP TABLE ""{metadataFilesetTable}"" (
MetadataID INTEGER PRIMARY KEY,
FilesetID INTEGER
)
");
await deletecmd.ExecuteNonQueryAsync($@"
INSERT OR IGNORE INTO ""{metadataFilesetTable}"" (
MetadataID,
FilesetID
)
{metadataFilesetQuery}
");
// Delete FilesetEntry rows that had their metadata deleted
await deletecmd.ExecuteNonQueryAsync($@"
DELETE FROM FilesetEntry
WHERE
FilesetEntry.FilesetID IN (
SELECT DISTINCT FilesetID FROM ""{metadataFilesetTable}""
)
AND FilesetEntry.FileID IN (
SELECT FilesetEntry.FileID
FROM FilesetEntry
INNER JOIN FileLookup
ON FileLookup.ID = FilesetEntry.FileID
WHERE FileLookup.MetadataID IN (
SELECT MetadataID FROM ""{metadataFilesetTable}""
)
)
");
// Delete FilesetEntry rows that had their blocks deleted
await deletecmd.ExecuteNonQueryAsync($@"
DELETE FROM FilesetEntry
WHERE FilesetEntry.FileID IN (
SELECT ID
FROM FileLookup
WHERE FileLookup.BlocksetID IN ({bsIdsSubQuery})
)
");
await deletecmd.ExecuteNonQueryAsync($@"
DELETE FROM FileLookup
WHERE FileLookup.MetadataID IN (
SELECT MetadataID
FROM ""{metadataFilesetTable}""
)
");
await deletecmd.ExecuteNonQueryAsync($@"
DELETE FROM ""Metadataset""
WHERE ""BlocksetID"" IN ({bsIdsSubQuery})
");
await deletecmd.ExecuteNonQueryAsync($@"
DELETE FROM ""FileLookup""
WHERE ""BlocksetID"" IN ({bsIdsSubQuery})
");
await deletecmd.ExecuteNonQueryAsync($@"
DELETE FROM ""Blockset""
WHERE ""ID"" IN ({bsIdsSubQuery})
");
await deletecmd.ExecuteNonQueryAsync($@"
DELETE FROM ""BlocksetEntry""
WHERE ""BlocksetID"" IN ({bsIdsSubQuery})
");
await deletecmd.ExecuteNonQueryAsync($@"
DELETE FROM ""BlocklistHash""
WHERE ""BlocklistHash"".""BlocksetID"" IN ({bsIdsSubQuery})
");
// If the volume is a block or index volume, this will update the crosslink table, otherwise nothing will happen
await deletecmd.ExecuteNonQueryAsync($@"
DELETE FROM ""IndexBlockLink""
WHERE ""BlockVolumeID"" IN ({volIdsSubQuery})
OR ""IndexVolumeID"" IN ({volIdsSubQuery})
");
await deletecmd.ExecuteNonQueryAsync($@"
DELETE FROM ""Block""
WHERE ""VolumeID"" IN ({volIdsSubQuery})
");
await deletecmd.ExecuteNonQueryAsync($@"
DELETE FROM ""DeletedBlock""
WHERE ""VolumeID"" IN ({volIdsSubQuery})
");
await deletecmd.ExecuteNonQueryAsync($@"
DELETE FROM ""ChangeJournalData""
WHERE ""FilesetID"" IN (
SELECT ""ID""
FROM ""Fileset""
WHERE ""VolumeID"" IN ({volIdsSubQuery})
)
");
await deletecmd.ExecuteNonQueryAsync($@"
DELETE FROM FilesetEntry
WHERE FilesetID IN (
SELECT ID
FROM Fileset
WHERE VolumeID IN ({volIdsSubQuery})
)
");
await deletecmd.ExecuteNonQueryAsync($@"
CREATE TABLE ""{filesetidstable}"" (
""ID"" INTEGER PRIMARY KEY
)
");
await deletecmd.ExecuteNonQueryAsync($@"
INSERT OR IGNORE INTO ""{filesetidstable}""
SELECT ""ID""
FROM ""Fileset""
WHERE ""VolumeID"" IN ({volIdsSubQuery})
");
// Delete from Fileset if FilesetEntry rows were deleted by related metadata and there are no references in FilesetEntry anymore
await deletecmd.ExecuteNonQueryAsync($@"
INSERT OR IGNORE INTO ""{filesetidstable}""
SELECT ""ID""
FROM ""Fileset""
WHERE
""Fileset"".""ID"" IN (
SELECT DISTINCT ""FilesetID""
FROM ""{metadataFilesetTable}""
)
AND ""Fileset"".""ID"" NOT IN (
SELECT DISTINCT ""FilesetID""
FROM FilesetEntry
)
");
// Since we are deleting the fileset, we also need to mark the remote volume as deleting so it will be cleaned up later
await deletecmd.SetCommandAndParameters($@"
UPDATE ""RemoteVolume""
SET ""State"" = @NewState
WHERE
""ID"" IN (
SELECT DISTINCT ""VolumeID""
FROM ""Fileset""
WHERE ""Fileset"".""ID"" IN (
SELECT ""ID""
FROM ""{filesetidstable}""
)
)
AND ""State"" IN (@AllowedStates)
")
.SetParameterValue("@NewState", RemoteVolumeState.Deleting.ToString())
.ExpandInClauseParameter("@AllowedStates", [
RemoteVolumeState.Uploading.ToString(),
RemoteVolumeState.Uploaded.ToString(),
RemoteVolumeState.Verified.ToString(),
RemoteVolumeState.Temporary.ToString()
])
.ExecuteNonQueryAsync();
await deletecmd.ExecuteNonQueryAsync($@"
DELETE FROM ""Fileset""
WHERE ""ID"" IN (
SELECT ""ID""
FROM ""{filesetidstable}""
)
");
// Clean up temp tables for subqueries. We truncate content and then try to delete.
// Drop in try-block, as it fails in nested transactions (SQLite problem)
// SQLite.SQLiteException (0x80004005): database table is locked
await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{blocksetidstable}"" ");
await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{volidstable}"" ");
await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{metadataFilesetTable}"" ");
await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{filesetidstable}"" ");
try
{
deletecmd.CommandTimeout = 2;
await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{blocksetidstable}"" ");
await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{volidstable}"" ");
await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{metadataFilesetTable}"" ");
await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filesetidstable}"" ");
}
catch { /* Ignore, will be deleted on close anyway. */ }
m_removeremotevolumeCommand
.SetTransaction(m_rtr)
.SetParameterValue("@Now", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow))
.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString());
foreach (var name in names)
{
await m_removeremotevolumeCommand
.SetTransaction(m_rtr)
.SetParameterValue("@Name", name)
.ExecuteNonQueryAsync();
}
// Validate before commiting changes
var nonAttachedFiles = await deletecmd.ExecuteScalarInt64Async(@"
SELECT COUNT(*)
FROM ""FilesetEntry""
WHERE ""FileID"" NOT IN (
SELECT ""ID"" FROM ""FileLookup""
)
");
if (nonAttachedFiles > 0)
throw new ConstraintException($"Detected {nonAttachedFiles} file(s) in FilesetEntry without corresponding FileLookup entry");
await m_rtr.CommitAsync();
}
public async Task Vacuum()
{
m_hasExecutedVacuum = true;
using var cmd = m_connection.CreateCommand();
await cmd.ExecuteNonQueryAsync("VACUUM");
}
public async Task<long> RegisterRemoteVolume(string name, RemoteVolumeType type, long size, RemoteVolumeState state)
{
return await RegisterRemoteVolume(name, type, state, size, new TimeSpan(0));
}
public async Task<long> RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state)
{
return await RegisterRemoteVolume(name, type, state, new TimeSpan(0));
}
public async Task<long> RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, TimeSpan deleteGraceTime)
{
return await RegisterRemoteVolume(name, type, state, -1, deleteGraceTime);
}
public async Task<long> RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, long size, TimeSpan deleteGraceTime)
{
var r = await m_createremotevolumeCommand
.SetTransaction(m_rtr)
.SetParameterValue("@OperationId", m_operationid)
.SetParameterValue("@Name", name)
.SetParameterValue("@Type", type.ToString())
.SetParameterValue("@State", state.ToString())
.SetParameterValue("@Size", size)
.SetParameterValue("@VerificationCount", 0)
.SetParameterValue("@DeleteGraceTime", deleteGraceTime.Ticks <= 0 ? 0 : (DateTime.UtcNow + deleteGraceTime).Ticks)
.SetParameterValue("@ArchiveTime", 0)
.ExecuteScalarInt64Async();
await m_rtr.CommitAsync();
return r;
}
public async Task<IEnumerable<long>> GetFilesetIDs(DateTime restoretime, long[] versions, bool singleTimeMatch = false)
{
if (restoretime.Kind == DateTimeKind.Unspecified)
throw new Exception("Invalid DateTime given, must be either local or UTC");
(var wherequery, var values) = await GetFilelistWhereClause(restoretime, versions, singleTimeMatch: singleTimeMatch);
var res = new List<long>();
using var cmd = m_connection.CreateCommand();
cmd.SetCommandAndParameters($@"
SELECT ""ID""
FROM ""Fileset""
{wherequery}
ORDER BY ""Timestamp"" DESC
")
.SetParameterValues(values);
using (var rd = await cmd.ExecuteReaderAsync())
while (await rd.ReadAsync())
res.Add(rd.ConvertValueToInt64(0));
if (res.Count == 0)
{
cmd.SetCommandAndParameters(@"
SELECT ""ID""
FROM ""Fileset""
ORDER BY ""Timestamp"" DESC
");
using (var rd = await cmd.ExecuteReaderAsync())
while (await rd.ReadAsync())
res.Add(rd.ConvertValueToInt64(0));
if (res.Count == 0)
throw new Duplicati.Library.Interface.UserInformationException("No backup at the specified date", "NoBackupAtDate");
else
Logging.Log.WriteWarningMessage(LOGTAG, "RestoreTimeNoMatch", null, "Restore time or version did not match any existing backups, selecting newest backup");
}
return res;
}
public async Task<IEnumerable<long>> FindMatchingFilesets(DateTime restoretime, long[] versions)
{
if (restoretime.Kind == DateTimeKind.Unspecified)
throw new Exception("Invalid DateTime given, must be either local or UTC");
var (wherequery, args) = await GetFilelistWhereClause(restoretime, versions, singleTimeMatch: true);
var res = new List<long>();
using var cmd = m_connection.CreateCommand();
cmd.SetCommandAndParameters(@$"
SELECT ""ID""
FROM ""Fileset""
{wherequery}
ORDER BY ""Timestamp"" DESC
")
.SetParameterValues(args);
using (var rd = await cmd.ExecuteReaderAsync())
while (await rd.ReadAsync())
res.Add(rd.ConvertValueToInt64(0));
return res;
}
public async Task<bool> IsFilesetFullBackup(DateTime filesetTime)
{
using var cmd = m_connection.CreateCommand();
cmd.SetCommandAndParameters($@"
SELECT ""IsFullBackup""
FROM ""Fileset""
WHERE ""Timestamp"" = @Timestamp
")
.SetTransaction(m_rtr)
.SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(filesetTime));
using var rd = await cmd.ExecuteReaderAsync();
if (!await rd.ReadAsync())
return false;
var isFullBackup = rd.GetInt32(0);
return isFullBackup == BackupType.FULL_BACKUP;
}
private async IAsyncEnumerable<KeyValuePair<string, string>> GetDbOptionList()
{
using var cmd = m_connection.CreateCommand(@"
SELECT
""Key"",
""Value""
FROM ""Configuration""
")
.SetTransaction(m_rtr);
using var rd = await cmd.ExecuteReaderAsync();
while (await rd.ReadAsync())
yield return new KeyValuePair<string, string>(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "");
}
public async Task<IDictionary<string, string>> GetDbOptions()
{
var res = await GetDbOptionList()
.ToDictionaryAsync(x => x.Key, x => x.Value);
await m_rtr.CommitAsync();
return res;
}
/// <summary>
/// Updates a database option
/// </summary>
/// <param name="key">The key to update</param>
/// <param name="value">The value to set</param>
private async Task UpdateDbOption(string key, bool value)
{
var opts = await GetDbOptions();
if (value)
opts[key] = "true";
else
opts.Remove(key);
await SetDbOptions(opts);
await m_rtr.CommitAsync();
}
/// <summary>
/// Flag indicating if a repair is in progress
/// </summary>
public async Task<bool> RepairInProgress(bool? value = null)
{
if (value is bool v)
{
await UpdateDbOption("repair-in-progress", v);
return v;
}
var opts = await GetDbOptions();
return opts.ContainsKey("repair-in-progress");
}
/// <summary>
/// Flag indicating if a repair is in progress
/// </summary>
public async Task<bool> PartiallyRecreated(bool? value = null)
{
if (value is bool v)
{
await UpdateDbOption("partially-recreated", v);
return v;
}
var opts = await GetDbOptions();
return opts.ContainsKey("partially-recreated");
}
/// <summary>
/// Flag indicating if the database can contain partial uploads
/// </summary>
public async Task<bool> TerminatedWithActiveUploads(bool? value = null)
{
if (value is bool v)
{
await UpdateDbOption("terminated-with-active-uploads", v);
return v;
}
var opts = await GetDbOptions();
return opts.ContainsKey("terminated-with-active-uploads");
}
/// <summary>
/// Sets the database options
/// </summary>
/// <param name="options">The options to set</param>
/// <param name="transaction">An optional transaction</param>
public async Task SetDbOptions(IDictionary<string, string> options)
{
using var cmd = m_connection.CreateCommand();
await cmd.ExecuteNonQueryAsync(@"
DELETE FROM ""Configuration""
");
foreach (var kp in options)
{
await cmd.SetCommandAndParameters(@"
INSERT INTO ""Configuration"" (
""Key"",
""Value""
)
VALUES (
@Key,
@Value
)
")
.SetParameterValue("@Key", kp.Key)
.SetParameterValue("@Value", kp.Value)
.ExecuteNonQueryAsync();
}
await m_rtr.CommitAsync();
}
public async Task<long> GetBlocksLargerThan(long fhblocksize)
{
using var cmd = await m_connection.CreateCommandAsync(@"
SELECT COUNT(*)
FROM ""Block""
WHERE ""Size"" > @Size
");
return await cmd.SetParameterValue("@Size", fhblocksize)
.ExecuteScalarInt64Async(-1);
}
/// <summary>
/// Verifies the consistency of the database
/// </summary>
/// <param name="blocksize">The block size in bytes</param>
/// <param name="hashsize">The hash size in byts</param>
/// <param name="verifyfilelists">Also verify filelists (can be slow)</param>
/// <param name="transaction">The transaction to run in</param>
public async Task VerifyConsistency(long blocksize, long hashsize, bool verifyfilelists)
=> await VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, false);
/// <summary>
/// Verifies the consistency of the database prior to repair
/// </summary>
/// <param name="blocksize">The block size in bytes</param>
/// <param name="hashsize">The hash size in byts</param>
/// <param name="verifyfilelists">Also verify filelists (can be slow)</param>
/// <param name="transaction">The transaction to run in</param>
public async Task VerifyConsistencyForRepair(long blocksize, long hashsize, bool verifyfilelists)
=> await VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, true);
/// <summary>
/// Verifies the consistency of the database
/// </summary>
/// <param name="blocksize">The block size in bytes</param>
/// <param name="hashsize">The hash size in byts</param>
/// <param name="verifyfilelists">Also verify filelists (can be slow)</param>
/// <param name="laxVerifyForRepair">Disable verify for errors that will be fixed by repair</param>
/// <param name="transaction">The transaction to run in</param>
private async Task VerifyConsistencyInner(long blocksize, long hashsize, bool verifyfilelists, bool laxVerifyForRepair)
{
using var cmd = m_connection.CreateCommand()
.SetTransaction(m_rtr);
// Calculate the lengths for each blockset
var combinedLengths = @"
SELECT
""A"".""ID"" AS ""BlocksetID"",
IFNULL(""B"".""CalcLen"", 0) AS ""CalcLen"",
""A"".""Length""
FROM ""Blockset"" A
LEFT OUTER JOIN (
SELECT
""BlocksetEntry"".""BlocksetID"",
SUM(""Block"".""Size"") AS ""CalcLen""
FROM ""BlocksetEntry""
LEFT OUTER JOIN ""Block""
ON ""Block"".""ID"" = ""BlocksetEntry"".""BlockID""
GROUP BY ""BlocksetEntry"".""BlocksetID""
) B
ON ""A"".""ID"" = ""B"".""BlocksetID""
";
// For each blockset with wrong lengths, fetch the file path
var reportDetails = @$"
SELECT
""CalcLen"",
""Length"", ""A"".""BlocksetID"",
""File"".""Path""
FROM ({combinedLengths}) A, ""File""
WHERE
""A"".""BlocksetID"" = ""File"".""BlocksetID""
AND ""A"".""CalcLen"" != ""A"".""Length""
";
using (var rd = await cmd.ExecuteReaderAsync(reportDetails))
if (await rd.ReadAsync())
{
var sb = new StringBuilder();
sb.AppendLine("Found inconsistency in the following files while validating database: ");
var c = 0;
do
{
if (c < 5)
sb.AppendFormat("{0}, actual size {1}, dbsize {2}, blocksetid: {3}{4}", rd.GetValue(3), rd.GetValue(1), rd.GetValue(0), rd.GetValue(2), Environment.NewLine);
c++;
} while (await rd.ReadAsync());
c -= 5;
if (c > 0)
sb.AppendFormat("... and {0} more", c);
sb.Append(". Run repair to fix it.");
throw new DatabaseInconsistencyException(sb.ToString());
}
var real_count = await cmd.ExecuteScalarInt64Async(@"
SELECT Count(*)
FROM ""BlocklistHash""
", 0);
var unique_count = await cmd.ExecuteScalarInt64Async(@"
SELECT Count(*)
FROM (
SELECT DISTINCT
""BlocksetID"",
""Index""
FROM ""BlocklistHash""
)", 0);
if (real_count != unique_count)
throw new DatabaseInconsistencyException($"Found {real_count} blocklist hashes, but there should be {unique_count}. Run repair to fix it.");
var itemswithnoblocklisthash = await cmd.ExecuteScalarInt64Async($@"
SELECT COUNT(*)
FROM (
SELECT *
FROM (
SELECT
""N"".""BlocksetID"",
((""N"".""BlockCount"" + {blocksize / hashsize} - 1) / {blocksize / hashsize}) AS ""BlocklistHashCountExpected"",
CASE
WHEN ""G"".""BlocklistHashCount"" IS NULL
THEN 0
ELSE ""G"".""BlocklistHashCount""
END AS ""BlocklistHashCountActual""
FROM (
SELECT
""BlocksetID"",
COUNT(*) AS ""BlockCount""
FROM ""BlocksetEntry""
GROUP BY ""BlocksetID""
) ""N""
LEFT OUTER JOIN (
SELECT
""BlocksetID"",
COUNT(*) AS ""BlocklistHashCount""
FROM ""BlocklistHash""
GROUP BY ""BlocksetID""
) ""G""
ON ""N"".""BlocksetID"" = ""G"".""BlocksetID""
WHERE ""N"".""BlockCount"" > 1
)
WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual""
)
", 0);
if (itemswithnoblocklisthash != 0)
throw new DatabaseInconsistencyException($"Found {itemswithnoblocklisthash} file(s) with missing blocklist hashes");
cmd.SetCommandAndParameters(@"
SELECT COUNT(*)
FROM ""Blockset""
WHERE
""Length"" > 0
AND ""ID"" NOT IN (
SELECT ""BlocksetId""
FROM ""BlocksetEntry""
)
");
if (await cmd.ExecuteScalarInt64Async() != 0)
throw new DatabaseInconsistencyException("Detected non-empty blocksets with no associated blocks!");
cmd.SetCommandAndParameters(@"
SELECT COUNT(*)
FROM ""FileLookup""
WHERE
""BlocksetID"" != @FolderBlocksetId
AND ""BlocksetID"" != @SymlinkBlocksetId
AND NOT ""BlocksetID"" IN (
SELECT ""ID""
FROM ""Blockset""
)
");
cmd.SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID);
cmd.SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID);
if (await cmd.ExecuteScalarInt64Async(0) != 0)
throw new DatabaseInconsistencyException("Detected files associated with non-existing blocksets!");
if (!laxVerifyForRepair)
{
cmd.SetCommandAndParameters(@"
SELECT COUNT(*)
FROM ""Fileset""
WHERE ""VolumeID"" NOT IN (
SELECT ""ID""
FROM ""RemoteVolume""
WHERE
""Type"" = @Type
AND ""State"" != @State
)
");
cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString());
cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString());
var filesetsMissingVolumes = await cmd.ExecuteScalarInt64Async(0);
if (filesetsMissingVolumes != 0)
{
if (filesetsMissingVolumes == 1)
{
cmd.SetCommandAndParameters(@"
SELECT
""ID"",
""Timestamp"",
""VolumeID""
FROM ""Fileset""
WHERE ""VolumeID"" NOT IN (
SELECT ""ID""
FROM ""RemoteVolume""
WHERE
""Type"" = @Type
AND ""State"" != @State
)
");
cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString());
cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString());
using var reader = await cmd.ExecuteReaderAsync();
if (await reader.ReadAsync())
throw new DatabaseInconsistencyException($"Detected 1 fileset with missing volume: FilesetId = {reader.ConvertValueToInt64(0)}, Time = ({ParseFromEpochSeconds(reader.ConvertValueToInt64(1))}), unmatched VolumeID {reader.ConvertValueToInt64(2)}");
}
throw new DatabaseInconsistencyException($"Detected {filesetsMissingVolumes} filesets with missing volumes");
}
cmd.SetCommandAndParameters(@"
SELECT COUNT(*)
FROM ""RemoteVolume""
WHERE
""Type"" = @Type
AND ""State"" != @State
AND ""ID"" NOT IN (
SELECT ""VolumeID""
FROM ""Fileset""
)
");
cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString());
cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString());
var volumesMissingFilests = await cmd.ExecuteScalarInt64Async(0);
if (volumesMissingFilests != 0)
{
if (volumesMissingFilests == 1)
{
cmd.SetCommandAndParameters(@"
SELECT
""ID"",
""Name"",
""State""
FROM ""RemoteVolume""
WHERE
""Type"" = @Type
AND ""State"" != @State
AND ""ID"" NOT IN (
SELECT ""VolumeID""
FROM ""Fileset""
)
");
cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString());
cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString());
using var reader = await cmd.ExecuteReaderAsync();
if (await reader.ReadAsync())
throw new DatabaseInconsistencyException($"Detected 1 volume with missing filesets: VolumeId = {reader.ConvertValueToInt64(0)}, Name = {reader.ConvertValueToString(1)}, State = {reader.ConvertValueToString(2)}");
}
throw new DatabaseInconsistencyException($"Detected {volumesMissingFilests} volumes with missing filesets");
}
}
var nonAttachedFiles = await cmd.ExecuteScalarInt64Async(@"
SELECT COUNT(*)
FROM ""FilesetEntry""
WHERE ""FileID"" NOT IN (
SELECT ""ID""
FROM ""FileLookup""
)
");
if (nonAttachedFiles != 0)
{
// Attempt to create a better error message by finding the first 10 fileset ids with the issue
using var filesetIdReader = await cmd.ExecuteReaderAsync(@"
SELECT DISTINCT(FilesetID)
FROM ""FilesetEntry""
WHERE ""FileID"" NOT IN (
SELECT ""ID""
FROM ""FileLookup""
)
LIMIT 11
");
var filesetIds = new HashSet<long>();
var overflow = false;
while (await filesetIdReader.ReadAsync())
{
if (filesetIds.Count >= 10)
{
overflow = true;
break;
}
filesetIds.Add(filesetIdReader.ConvertValueToInt64(0));
}
var pairs = FilesetTimes()
.Select((x, i) => new { FilesetId = x.Key, Version = i, Time = x.Value })
.Where(x => filesetIds.Contains(x.FilesetId))
.Select(x => $"Fileset {x.Version}: {x.Time} (id = {x.FilesetId})");
// Fall back to a generic error message if we can't find the fileset ids
if (!await pairs.AnyAsync())
throw new DatabaseInconsistencyException($"Detected {nonAttachedFiles} file(s) in FilesetEntry without corresponding FileLookup entry");
if (overflow)
pairs = pairs.Append("... and more");
throw new DatabaseInconsistencyException($"Detected {nonAttachedFiles} file(s) in FilesetEntry without corresponding FileLookup entry in the following filesets:{Environment.NewLine}{string.Join(Environment.NewLine, pairs)}");
}
if (verifyfilelists)
{
var anyError = new List<string>();
using (var cmd2 = m_connection.CreateCommand())
{
cmd2.SetTransaction(m_rtr);
cmd.SetCommandAndParameters(@"
SELECT ""ID""
FROM ""Fileset""
");
await foreach (var filesetid in cmd.ExecuteReaderEnumerableAsync().Select(x => x.ConvertValueToInt64(0, -1)))
{
var expandedlist = await cmd2.SetCommandAndParameters($@"
SELECT COUNT(*)
FROM (
SELECT DISTINCT ""Path""
FROM ({LIST_FILESETS})
UNION
SELECT DISTINCT ""Path""
FROM ({LIST_FOLDERS_AND_SYMLINKS})
)
")
.SetParameterValue("@FilesetId", filesetid)
.SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID)
.SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID)
.ExecuteScalarInt64Async(0);
//var storedfilelist = cmd2.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""FileLookup"" WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FileLookup"".""BlocksetID"" != @FolderBlocksetId AND ""FileLookup"".""BlocksetID"" != @SymlinkBlocksetId"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID);
var storedlist = await cmd2.SetCommandAndParameters(@"
SELECT COUNT(*)
FROM ""FilesetEntry""
WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId
")
.SetParameterValue("@FilesetId", filesetid)
.ExecuteScalarInt64Async(0);
if (expandedlist != storedlist)
{
var filesetname = filesetid.ToString();
var fileset = await FilesetTimes().Zip(AsyncEnumerable.Range(0, await FilesetTimes().CountAsync()), (a, b) => new Tuple<long, long, DateTime>(b, a.Key, a.Value)).FirstOrDefaultAsync(x => x.Item2 == filesetid);
if (fileset != null)
filesetname = $"version {fileset.Item1}: {fileset.Item3} (database id: {fileset.Item2})";
anyError.Add($"Unexpected difference in fileset {filesetname}, found {expandedlist} entries, but expected {storedlist}");
}
}
}
if (anyError.Count != 0)
{
throw new DatabaseInconsistencyException(string.Join("\n\r", anyError), "FilesetDifferences");
}
}
}
public interface IBlock
{
string Hash { get; }
long Size { get; }
}
internal class Block(string hash, long size) : IBlock
{
public string Hash { get; private set; } = hash;
public long Size { get; private set; } = size;
}
public async IAsyncEnumerable<IBlock> GetBlocks(long volumeid)
{
using var cmd = await m_connection.CreateCommandAsync(@"
SELECT DISTINCT
""Hash"",
""Size""
FROM ""Block""
WHERE ""VolumeID"" = @VolumeId
");
cmd.SetTransaction(m_rtr)
.SetParameterValue("@VolumeId", volumeid);
using var rd = await cmd.ExecuteReaderAsync();
while (await rd.ReadAsync())
yield return new Block(rd.ConvertValueToString(0) ?? throw new Exception("Hash is null"), rd.ConvertValueToInt64(1));
}
// TODO: Replace this with an enumerable method
private class BlocklistHashEnumerable : IEnumerable<string>
{
private class BlocklistHashEnumerator : IEnumerator<string>
{
private readonly SqliteDataReader m_reader;
private readonly BlocklistHashEnumerable m_parent;
private string? m_path = null;
private bool m_first = true;
private string? m_current = null;
public BlocklistHashEnumerator(BlocklistHashEnumerable parent, SqliteDataReader reader)
{
m_reader = reader;
m_parent = parent;
}
public string Current { get { return m_current!; } }
public void Dispose()
{
}
object System.Collections.IEnumerator.Current { get { return Current; } }
public bool MoveNext()
{
m_first = false;
if (m_path == null)
{
m_path = m_reader.ConvertValueToString(0);
m_current = m_reader.ConvertValueToString(6);
return true;
}
else
{
if (m_current == null)
return false;
if (!m_reader.ReadAsync().Await())
{
m_current = null;
m_parent.MoreData = false;
return false;
}
var np = m_reader.ConvertValueToString(0);
if (m_path != np)
{
m_current = null;
return false;
}
m_current = m_reader.ConvertValueToString(6);
return true;
}
}
public void Reset()
{
if (!m_first)
throw new Exception("Iterator reset not supported");
m_first = false;
}
}
private readonly SqliteDataReader m_reader;
public BlocklistHashEnumerable(SqliteDataReader reader)
{
m_reader = reader;
MoreData = true;
}
public bool MoreData { get; protected set; }
public IEnumerator<string> GetEnumerator()
{
return new BlocklistHashEnumerator(this, m_reader);
}
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
{
return GetEnumerator();
}
}
public const string LIST_FILESETS = @"
SELECT
""L"".""Path"",
""L"".""Lastmodified"",
""L"".""Filelength"",
""L"".""Filehash"",
""L"".""Metahash"",
""L"".""Metalength"",
""L"".""BlocklistHash"",
""L"".""FirstBlockHash"",
""L"".""FirstBlockSize"",
""L"".""FirstMetaBlockHash"",
""L"".""FirstMetaBlockSize"",
""M"".""Hash"" AS ""MetaBlocklistHash""
FROM (
SELECT
""J"".""Path"",
""J"".""Lastmodified"",
""J"".""Filelength"",
""J"".""Filehash"",
""J"".""Metahash"",
""J"".""Metalength"",
""K"".""Hash"" AS ""BlocklistHash"",
""J"".""FirstBlockHash"",
""J"".""FirstBlockSize"",
""J"".""FirstMetaBlockHash"",
""J"".""FirstMetaBlockSize"",
""J"".""MetablocksetID""
FROM (
SELECT
""A"".""Path"" AS ""Path"",
""D"".""Lastmodified"" AS ""Lastmodified"",
""B"".""Length"" AS ""Filelength"",
""B"".""FullHash"" AS ""Filehash"",
""E"".""FullHash"" AS ""Metahash"",
""E"".""Length"" AS ""Metalength"",
""A"".""BlocksetID"" AS ""BlocksetID"",
""F"".""Hash"" AS ""FirstBlockHash"",
""F"".""Size"" AS ""FirstBlockSize"",
""H"".""Hash"" AS ""FirstMetaBlockHash"",
""H"".""Size"" AS ""FirstMetaBlockSize"",
""C"".""BlocksetID"" AS ""MetablocksetID""
FROM ""File"" A
LEFT JOIN ""Blockset"" B
ON ""A"".""BlocksetID"" = ""B"".""ID""
LEFT JOIN ""Metadataset"" C
ON ""A"".""MetadataID"" = ""C"".""ID""
LEFT JOIN ""FilesetEntry"" D
ON ""A"".""ID"" = ""D"".""FileID""
LEFT JOIN ""Blockset"" E
ON ""E"".""ID"" = ""C"".""BlocksetID""
LEFT JOIN ""BlocksetEntry"" G
ON ""B"".""ID"" = ""G"".""BlocksetID""
LEFT JOIN ""Block"" F
ON ""G"".""BlockID"" = ""F"".""ID""
LEFT JOIN ""BlocksetEntry"" I
ON ""E"".""ID"" = ""I"".""BlocksetID""
LEFT JOIN ""Block"" H
ON ""I"".""BlockID"" = ""H"".""ID""
WHERE
""A"".""BlocksetId"" >= 0
AND ""D"".""FilesetID"" = @FilesetId
AND (
""I"".""Index"" = 0
OR ""I"".""Index"" IS NULL
)
AND (
""G"".""Index"" = 0
OR ""G"".""Index"" IS NULL
)
) J
LEFT OUTER JOIN ""BlocklistHash"" K
ON ""K"".""BlocksetID"" = ""J"".""BlocksetID""
ORDER BY
""J"".""Path"",
""K"".""Index""
) L
LEFT OUTER JOIN ""BlocklistHash"" M
ON ""M"".""BlocksetID"" = ""L"".""MetablocksetID""
";
public const string LIST_FOLDERS_AND_SYMLINKS = @"
SELECT
""G"".""BlocksetID"",
""G"".""ID"",
""G"".""Path"",
""G"".""Length"",
""G"".""FullHash"",
""G"".""Lastmodified"",
""G"".""FirstMetaBlockHash"",
""H"".""Hash"" AS ""MetablocklistHash""
FROM (
SELECT
""B"".""BlocksetID"",
""B"".""ID"",
""B"".""Path"",
""D"".""Length"",
""D"".""FullHash"",
""A"".""Lastmodified"",
""F"".""Hash"" AS ""FirstMetaBlockHash"",
""C"".""BlocksetID"" AS ""MetaBlocksetID""
FROM
""FilesetEntry"" A,
""File"" B,
""Metadataset"" C,
""Blockset"" D,
""BlocksetEntry"" E,
""Block"" F
WHERE
""A"".""FileID"" = ""B"".""ID""
AND ""B"".""MetadataID"" = ""C"".""ID""
AND ""C"".""BlocksetID"" = ""D"".""ID""
AND ""E"".""BlocksetID"" = ""C"".""BlocksetID""
AND ""E"".""BlockID"" = ""F"".""ID""
AND ""E"".""Index"" = 0
AND (
""B"".""BlocksetID"" = @FolderBlocksetId
OR ""B"".""BlocksetID"" = @SymlinkBlocksetId
)
AND ""A"".""FilesetID"" = @FilesetId
) G
LEFT OUTER JOIN ""BlocklistHash"" H
ON ""H"".""BlocksetID"" = ""G"".""MetaBlocksetID""
ORDER BY
""G"".""Path"",
""H"".""Index""
";
public async Task WriteFileset(Volumes.FilesetVolumeWriter filesetvolume, long filesetId)
{
using var cmd = m_connection.CreateCommand()
.SetCommandAndParameters(LIST_FOLDERS_AND_SYMLINKS)
.SetTransaction(m_rtr)
.SetParameterValue("@FilesetId", filesetId)
.SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID)
.SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID);
string? lastpath = null;
using (var rd = await cmd.ExecuteReaderAsync())
while (await rd.ReadAsync())
{
var blocksetID = rd.ConvertValueToInt64(0, -1);
var path = rd.ConvertValueToString(2);
var metalength = rd.ConvertValueToInt64(3, -1);
var metahash = rd.ConvertValueToString(4);
var metablockhash = rd.ConvertValueToString(6);
var metablocklisthash = rd.ConvertValueToString(7);
if (path == lastpath)
Logging.Log.WriteWarningMessage(LOGTAG, "DuplicatePathFound", null, "Duplicate path detected: {0}", path);
lastpath = path;
if (blocksetID == FOLDER_BLOCKSET_ID)
filesetvolume.AddDirectory(path, metahash, metalength, metablockhash, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash });
else if (blocksetID == SYMLINK_BLOCKSET_ID)
filesetvolume.AddSymlink(path, metahash, metalength, metablockhash, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash });
}
// TODO: Perhaps run the above query after recreate and compare count(*) with count(*) from filesetentry where id = x
cmd.SetCommandAndParameters(LIST_FILESETS);
cmd.SetParameterValue("@FilesetId", filesetId);
using (var rd = await cmd.ExecuteReaderAsync())
if (await rd.ReadAsync())
{
var more = false;
do
{
var path = rd.ConvertValueToString(0);
var filehash = rd.ConvertValueToString(3);
var size = rd.ConvertValueToInt64(2);
var lastmodified = new DateTime(rd.ConvertValueToInt64(1, 0), DateTimeKind.Utc);
var metahash = rd.ConvertValueToString(4);
var metasize = rd.ConvertValueToInt64(5, -1);
var p = rd.GetValue(6);
var blrd = (p == null || p == DBNull.Value) ? null : new BlocklistHashEnumerable(rd);
var blockhash = rd.ConvertValueToString(7);
var blocksize = rd.ConvertValueToInt64(8, -1);
var metablockhash = rd.ConvertValueToString(9);
//var metablocksize = rd.ConvertValueToInt64(10, -1);
var metablocklisthash = rd.ConvertValueToString(11);
if (blockhash == filehash)
blockhash = null;
if (metablockhash == metahash)
metablockhash = null;
filesetvolume.AddFile(path, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blrd, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash });
if (blrd == null)
more = await rd.ReadAsync();
else
more = blrd.MoreData;
} while (more);
}
}
public async Task LinkFilesetToVolume(long filesetid, long volumeid)
{
using var cmd = await m_connection.CreateCommandAsync(@"
UPDATE ""Fileset""
SET ""VolumeID"" = @VolumeId
WHERE ""ID"" = @FilesetId
");
var c = await cmd.SetTransaction(m_rtr)
.SetParameterValue("@VolumeId", volumeid)
.SetParameterValue("@FilesetId", filesetid)
.ExecuteNonQueryAsync();
if (c != 1)
throw new Exception($"Failed to link filesetid {filesetid} to volumeid {volumeid}");
}
public async Task PushTimestampChangesToPreviousVersion(long filesetId)
{
var query = @"
UPDATE FilesetEntry AS oldVersion
SET Lastmodified = tempVersion.Lastmodified
FROM FilesetEntry AS tempVersion
WHERE
oldVersion.FileID = tempVersion.FileID
AND tempVersion.FilesetID = @FilesetId
AND oldVersion.FilesetID = (
SELECT ID
FROM Fileset
WHERE ID != @FilesetId
ORDER BY Timestamp DESC
LIMIT 1
)
";
using var cmd = await m_connection.CreateCommandAsync(query);
await cmd.SetTransaction(m_rtr)
.SetParameterValue("@FilesetId", filesetId)
.ExecuteNonQueryAsync();
}
/// <summary>
/// Keeps a list of filenames in a temporary table with a single column Path
/// </summary>
public class FilteredFilenameTable : IDisposable
{
public string Tablename { get; private set; }
private readonly LocalDatabase m_db;
private FilteredFilenameTable(LocalDatabase db)
{
Tablename = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
m_db = db;
}
[Obsolete("Calling this constructor will throw an exception. Use the Create method instead.")]
public FilteredFilenameTable(SqliteConnection connection, IFilter filter)
{
throw new NotImplementedException("Use the Create method instead.");
}
public static async Task<FilteredFilenameTable> CreateFilteredFilenameTableAsync(LocalDatabase db, IFilter filter)
{
var ftt = new FilteredFilenameTable(db);
// TODO factory
var type = FilterType.Regexp;
if (filter is FilterExpression expression)
type = expression.Type;
// Bugfix: SQLite does not handle case-insensitive LIKE with non-ascii characters
if (type != FilterType.Regexp && !Library.Utility.Utility.IsFSCaseSensitive && filter.ToString()!.Any(x => x > 127))
type = FilterType.Regexp;
if (filter.Empty)
{
using var cmd = await db.Connection.CreateCommandAsync($@"
CREATE TEMPORARY TABLE ""{ftt.Tablename}"" AS
SELECT DISTINCT ""Path""
FROM ""File""
");
await cmd.SetTransaction(db.Transaction)
.ExecuteNonQueryAsync();
return ftt;
}
if (type == FilterType.Regexp || type == FilterType.Group)
{
// TODO: Optimize this to not rely on the "File" view, and not instantiate the paths in full
using var cmd = await db.Connection.CreateCommandAsync($@"
CREATE TEMPORARY TABLE ""{ftt.Tablename}"" (
""Path"" TEXT NOT NULL
)
");
await cmd.SetTransaction(db.Transaction)
.ExecuteNonQueryAsync();
cmd.SetCommandAndParameters($@"
INSERT INTO ""{ftt.Tablename}"" (""Path"")
VALUES (@Path)
");
using var c2 = await db.Connection.CreateCommandAsync(@"
SELECT DISTINCT ""Path""
FROM ""File""
");
c2.SetTransaction(db.Transaction);
using (var rd = await c2.ExecuteReaderAsync())
while (await rd.ReadAsync())
{
var p = rd.ConvertValueToString(0);
if (FilterExpression.Matches(filter, p))
{
await cmd.SetParameterValue("@Path", p)
.ExecuteNonQueryAsync();
}
}
await db.Transaction.CommitAsync();
}
else
{
var sb = new StringBuilder();
var args = new Dictionary<string, object?>();
foreach (var f in ((FilterExpression)filter).GetSimpleList())
{
if (sb.Length != 0)
sb.Append(" OR ");
var argName = $"@Arg{args.Count}";
if (type == FilterType.Wildcard)
{
sb.Append(@$"""Path"" LIKE {argName}");
args.Add(argName, f.Replace('*', '%').Replace('?', '_'));
}
else
{
sb.Append(@$"""Path"" = {argName}");
args.Add(argName, f);
}
}
using var cmd = db.Connection.CreateCommand();
cmd.SetTransaction(db.Transaction);
await cmd.ExecuteNonQueryAsync($@"
CREATE TEMPORARY TABLE ""{ftt.Tablename}"" (
""Path"" TEXT NOT NULL
)
");
await cmd.ExecuteNonQueryAsync($@"
INSERT INTO ""{ftt.Tablename}""
SELECT DISTINCT ""Path""
FROM ""File""
WHERE {sb}
", args);
await db.Transaction.CommitAsync();
}
return ftt;
}
public void Dispose()
{
DisposeAsync().Await();
}
public async Task DisposeAsync()
{
if (Tablename != null)
try
{
using var cmd = m_db.Connection.CreateCommand();
await cmd.ExecuteNonQueryAsync(@$"
DROP TABLE IF EXISTS ""{Tablename}""
");
}
catch { }
finally { Tablename = null!; }
}
}
public async Task RenameRemoteFile(string oldname, string newname)
{
//Rename the old entry, to preserve ID links
using var cmd = await m_connection.CreateCommandAsync(@"
UPDATE ""Remotevolume""
SET ""Name"" = @Newname
WHERE ""Name"" = @Oldname
");
var c = await cmd.SetTransaction(m_rtr)
.SetParameterValue("@Newname", newname)
.SetParameterValue("@Oldname", oldname)
.ExecuteNonQueryAsync();
if (c != 1)
throw new Exception($"Unexpected result from renaming \"{oldname}\" to \"{newname}\", expected {1} got {c}");
// Grab the type of entry
cmd.SetCommandAndParameters(@"
SELECT ""Type""
FROM ""Remotevolume""
WHERE ""Name"" = @Name
")
.SetParameterValue("@Name", newname);
var type = (RemoteVolumeType)Enum.Parse(
typeof(RemoteVolumeType), (await cmd.ExecuteScalarAsync())?.ToString() ?? "",
true);
//Create a fake new entry with the old name and mark as deleting
// as this ensures we will remove it, if it shows up in some later listing
await RegisterRemoteVolume(oldname, type, RemoteVolumeState.Deleting);
await m_rtr.CommitAsync();
}
/// <summary>
/// Creates a timestamped backup operation to correctly associate the fileset with the time it was created.
/// </summary>
/// <param name="volumeid">The ID of the fileset volume to update</param>
/// <param name="timestamp">The timestamp of the operation to create</param>
/// <param name="transaction">An optional external transaction</param>
public virtual async Task<long> CreateFileset(long volumeid, DateTime timestamp)
{
using var cmd = await m_connection.CreateCommandAsync(@"
INSERT INTO ""Fileset"" (
""OperationID"",
""Timestamp"",
""VolumeID"",
""IsFullBackup""
)
VALUES (
@OperationId,
@Timestamp,
@VolumeId,
@IsFullBackup
);
SELECT last_insert_rowid();
");
var id = await cmd.SetTransaction(m_rtr)
.SetParameterValue("@OperationId", m_operationid)
.SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(timestamp))
.SetParameterValue("@VolumeId", volumeid)
.SetParameterValue("@IsFullBackup", BackupType.PARTIAL_BACKUP)
.ExecuteScalarInt64Async(-1);
await m_rtr.CommitAsync();
return id;
}
/// <summary>
/// Adds a link between an index volume and a block volume.
/// </summary>
/// <param name="indexVolumeID">The ID of the index volume.</param>
/// <param name="blockVolumeID">The ID of the block volume.</param>
/// <param name="transaction">An optional transaction.</param>
public async Task AddIndexBlockLink(long indexVolumeID, long blockVolumeID)
{
if (indexVolumeID <= 0)
throw new ArgumentOutOfRangeException(nameof(indexVolumeID), "Index volume ID must be greater than 0.");
if (blockVolumeID <= 0)
throw new ArgumentOutOfRangeException(nameof(blockVolumeID), "Block volume ID must be greater than 0.");
await m_insertIndexBlockLink.SetTransaction(m_rtr)
.SetParameterValue("@IndexVolumeId", indexVolumeID)
.SetParameterValue("@BlockVolumeId", blockVolumeID)
.ExecuteNonQueryAsync();
}
/// <summary>
/// Returns all unique blocklists for a given volume
/// </summary>
/// <param name="volumeid">The volume ID to get blocklists for</param>
/// <param name="blocksize">The blocksize</param>
/// <param name="hashsize">The size of the hash</param>
/// <param name="transaction">An optional external transaction</param>
/// <returns>An enumerable of tuples containing the blocklist hash, the blocklist data and the length of the data</returns>
public async IAsyncEnumerable<Tuple<string, byte[], int>> GetBlocklists(long volumeid, long blocksize, int hashsize)
{
// Group subquery by hash to ensure that each blocklist hash appears only once in the result
using var cmd = await m_connection.CreateCommandAsync($@"
SELECT
""A"".""Hash"",
""C"".""Hash""
FROM (
SELECT
""BlocklistHash"".""BlocksetID"",
""Block"".""Hash"",
""BlocklistHash"".""Index""
FROM
""BlocklistHash"",
""Block""
WHERE
""BlocklistHash"".""Hash"" = ""Block"".""Hash""
AND ""Block"".""VolumeID"" = @VolumeId
GROUP BY
""Block"".""Hash"",
""Block"".""Size""
) A,
""BlocksetEntry"" B,
""Block"" C
WHERE
""B"".""BlocksetID"" = ""A"".""BlocksetID""
AND ""B"".""Index"" >= (""A"".""Index"" * {blocksize / hashsize})
AND ""B"".""Index"" < ((""A"".""Index"" + 1) * {blocksize / hashsize})
AND ""C"".""ID"" = ""B"".""BlockID""
ORDER BY
""A"".""BlocksetID"",
""B"".""Index""
");
string? curHash = null;
var count = 0;
var buffer = new byte[blocksize];
cmd.SetTransaction(m_rtr)
.SetParameterValue("@VolumeId", volumeid);
using (var rd = await cmd.ExecuteReaderAsync())
while (await rd.ReadAsync())
{
var blockhash = rd.ConvertValueToString(0);
if ((blockhash != curHash && curHash != null) || count + hashsize > buffer.Length)
{
yield return new Tuple<string, byte[], int>(curHash!, buffer, count);
buffer = new byte[blocksize];
count = 0;
}
var hash = Convert.FromBase64String(rd.ConvertValueToString(1) ?? throw new Exception("Hash is null"));
Array.Copy(hash, 0, buffer, count, hashsize);
curHash = blockhash;
count += hashsize;
}
if (curHash != null)
yield return new Tuple<string, byte[], int>(curHash, buffer, count);
}
/// <summary>
/// Update fileset with full backup state
/// </summary>
/// <param name="fileSetId">Existing file set to update</param>
/// <param name="isFullBackup">Full backup state</param>
/// <param name="transaction">An optional external transaction</param>
public async Task UpdateFullBackupStateInFileset(long fileSetId, bool isFullBackup)
{
using var cmd = await m_connection.CreateCommandAsync(@"
UPDATE ""Fileset""
SET ""IsFullBackup"" = @IsFullBackup
WHERE ""ID"" = @FilesetId
");
await cmd.SetTransaction(m_rtr)
.SetParameterValue("@FilesetId", fileSetId)
.SetParameterValue("@IsFullBackup", isFullBackup ? BackupType.FULL_BACKUP : BackupType.PARTIAL_BACKUP)
.ExecuteNonQueryAsync();
await m_rtr.CommitAsync();
}
/// <summary>
/// Removes all entries in the fileset entry table for a given fileset ID
/// </summary>
/// <param name="filesetId">The fileset ID to clear</param>
/// <param name="transaction">The transaction to use</param>
public async Task ClearFilesetEntries(long filesetId)
{
using var cmd = await m_connection.CreateCommandAsync(@"
DELETE FROM ""FilesetEntry""
WHERE ""FilesetID"" = @FilesetId
");
await cmd.SetTransaction(m_rtr)
.SetParameterValue("@FilesetId", filesetId)
.ExecuteNonQueryAsync();
}
/// <summary>
/// Gets the last previous fileset that was incomplete
/// </summary>
/// <param name="transaction">The transaction to use</param>
/// <returns>The last incomplete fileset or default</returns>
public async Task<RemoteVolumeEntry> GetLastIncompleteFilesetVolume()
{
var candidates = GetIncompleteFilesets()
.OrderBy(x => x.Value);
if (await candidates.AnyAsync())
return await GetRemoteVolumeFromFilesetID((await candidates.LastAsync()).Key);
return default;
}
/// <summary>
/// Gets a list of incomplete filesets
/// </summary>
/// <param name="transaction">An optional transaction</param>
/// <returns>A list of fileset IDs and timestamps</returns>
public async IAsyncEnumerable<KeyValuePair<long, DateTime>> GetIncompleteFilesets()
{
using var cmd = await m_connection.CreateCommandAsync(@$"
SELECT DISTINCT
""Fileset"".""ID"",
""Fileset"".""Timestamp""
FROM
""Fileset"",
""RemoteVolume""
WHERE
""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID""
AND ""Fileset"".""ID"" IN (
SELECT ""FilesetID""
FROM ""FilesetEntry""
)
AND (
""RemoteVolume"".""State"" = '{RemoteVolumeState.Uploading}'
OR ""RemoteVolume"".""State"" = '{RemoteVolumeState.Temporary}'
)
");
using var rd = await cmd.SetTransaction(m_rtr)
.ExecuteReaderAsync();
while (await rd.ReadAsync())
{
yield return new KeyValuePair<long, DateTime>(
rd.ConvertValueToInt64(0),
ParseFromEpochSeconds(rd.ConvertValueToInt64(1))
.ToLocalTime()
);
}
}
/// <summary>
/// Gets the remote volume entry from the fileset ID
/// </summary>
/// <param name="filesetID">The fileset ID</param>
/// <param name="transaction">An optional transaction</param>
/// <returns>The remote volume entry or default</returns>
public async Task<RemoteVolumeEntry> GetRemoteVolumeFromFilesetID(long filesetID)
{
using var cmd = await m_connection.CreateCommandAsync(@"
SELECT
""RemoteVolume"".""ID"",
""Name"",
""Type"",
""Size"",
""Hash"",
""State"",
""DeleteGraceTime"",
""ArchiveTime""
FROM
""RemoteVolume"",
""Fileset""
WHERE
""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID""
AND ""Fileset"".""ID"" = @FilesetId
");
using var rd = await cmd.SetTransaction(m_rtr)
.SetParameterValue("@FilesetId", filesetID)
.ExecuteReaderAsync();
if (await rd.ReadAsync())
return new RemoteVolumeEntry(
rd.ConvertValueToInt64(0, -1),
rd.ConvertValueToString(1),
rd.ConvertValueToString(4),
rd.ConvertValueToInt64(3, -1),
(RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), rd.ConvertValueToString(2) ?? ""),
(RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.ConvertValueToString(5) ?? ""),
ParseFromEpochSeconds(rd.ConvertValueToInt64(6)).ToLocalTime(),
ParseFromEpochSeconds(rd.ConvertValueToInt64(7)).ToLocalTime()
);
else
return default(RemoteVolumeEntry);
}
public async Task PurgeLogData(DateTime threshold)
{
using var transaction = m_connection.BeginTransaction();
using var cmd = m_connection.CreateCommand();
cmd.Transaction = transaction;
var t = Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(threshold);
await cmd.SetCommandAndParameters(@"
DELETE FROM ""LogData""
WHERE ""Timestamp"" < @Timestamp
")
.SetParameterValue("@Timestamp", t)
.ExecuteNonQueryAsync();
await cmd.SetCommandAndParameters(@"
DELETE FROM ""RemoteOperation""
WHERE ""Timestamp"" < @Timestamp
")
.SetParameterValue("@Timestamp", t)
.ExecuteNonQueryAsync();
await transaction.CommitAsync();
}
public async Task PurgeDeletedVolumes(DateTime threshold)
{
using var transaction = m_connection.BeginTransaction();
await m_removedeletedremotevolumeCommand
.SetTransaction(transaction)
.SetParameterValue("@Now", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(threshold))
.ExecuteNonQueryAsync();
await transaction.CommitAsync();
}
public virtual void Dispose()
{
DisposeAsync().Await();
}
public virtual async Task DisposeAsync()
{
if (IsDisposed)
return;
DisposeAllFields<SqliteCommand>(this, false);
if (ShouldCloseConnection && m_connection != null)
{
if (m_connection.State == ConnectionState.Open && !m_hasExecutedVacuum)
{
using (var transaction = m_connection.BeginTransaction())
using (var command = m_connection.CreateCommand())
{
command.Transaction = transaction;
// SQLite recommends that PRAGMA optimize is run just before closing each database connection.
await command.ExecuteNonQueryAsync("PRAGMA optimize");
try
{
await transaction.CommitAsync();
}
catch (Exception ex)
{
Logging.Log.WriteVerboseMessage(LOGTAG, "FailedToCommitTransaction", ex, "Failed to commit transaction after pragma optimize, usually caused by the a no-op transaction");
}
}
await m_connection.CloseAsync();
}
await m_connection.DisposeAsync();
}
IsDisposed = true;
}
/// <summary>
/// Disposes all fields of a certain type, in the instance and its bases
/// </summary>
/// <typeparam name="T">The type of fields to find</typeparam>
/// <param name="item">The item to dispose</param>
/// <param name="throwExceptions"><c>True</c> if an aggregate exception should be thrown, or <c>false</c> if exceptions are silently captured</param>
public static void DisposeAllFields<T>(object item, bool throwExceptions)
where T : IDisposable
{
var typechain = new List<Type>();
var cur = item.GetType();
var exceptions = new List<Exception>();
while (cur != null && cur != typeof(object))
{
typechain.Add(cur);
cur = cur.BaseType;
}
var fields =
typechain.SelectMany(x =>
x.GetFields(
System.Reflection.BindingFlags.NonPublic |
System.Reflection.BindingFlags.Public |
System.Reflection.BindingFlags.Instance |
System.Reflection.BindingFlags.FlattenHierarchy
)
).Distinct()
.Where(x => x.FieldType.IsAssignableFrom(typeof(T)));
foreach (var p in fields)
try
{
var val = p.GetValue(item);
if (val != null)
((T)val).Dispose();
}
catch (Exception ex)
{
if (throwExceptions)
exceptions.Add(ex);
}
if (exceptions.Count > 0)
throw new AggregateException(exceptions);
}
public async Task WriteResults(IBasicResults result)
{
if (IsDisposed)
return;
if (m_connection != null && result != null)
{
if (result is BasicResults basicResults)
{
basicResults.FlushLog(this);
if (basicResults.EndTime.Ticks == 0)
basicResults.EndTime = DateTime.UtcNow;
}
var serializer = new JsonFormatSerializer();
await LogMessage("Result",
serializer.SerializeResults(result),
null
);
}
}
/// <summary>
/// The current index into the path prefix buffer
/// </summary>
private int m_pathPrefixIndex = 0;
/// <summary>
/// The path prefix lookup list
/// </summary>
private readonly KeyValuePair<string, long>[] m_pathPrefixLookup = new KeyValuePair<string, long>[5];
/// <summary>
/// Gets the path prefix ID, optionally creating it in the process.
/// </summary>
/// <returns>The path prefix ID.</returns>
/// <param name="prefix">The path to get the prefix for.</param>
/// <param name="transaction">The transaction to use for insertion, or null for no transaction</param>
public async Task<long> GetOrCreatePathPrefix(string prefix)
{
// Ring-buffer style lookup
for (var i = 0; i < m_pathPrefixLookup.Length; i++)
{
var ix = (i + m_pathPrefixIndex) % m_pathPrefixLookup.Length;
if (string.Equals(m_pathPrefixLookup[ix].Key, prefix, StringComparison.Ordinal))
return m_pathPrefixLookup[ix].Value;
}
var id = await m_findpathprefixCommand.SetTransaction(m_rtr)
.SetParameterValue("@Prefix", prefix)
.ExecuteScalarInt64Async();
if (id < 0)
{
id = await m_insertpathprefixCommand.SetTransaction(m_rtr)
.SetParameterValue("@Prefix", prefix)
.ExecuteScalarInt64Async();
}
m_pathPrefixIndex = (m_pathPrefixIndex + 1) % m_pathPrefixLookup.Length;
m_pathPrefixLookup[m_pathPrefixIndex] = new KeyValuePair<string, long>(prefix, id);
return id;
}
/// <summary>
/// The path separators on this system
/// </summary>
private static readonly char[] _pathseparators = [
Path.DirectorySeparatorChar,
Path.AltDirectorySeparatorChar,
];
/// <summary>
/// Helper method that splits a path on the last path separator
/// </summary>
/// <returns>The prefix and name.</returns>
/// <param name="path">The path to split.</param>
public static KeyValuePair<string, string> SplitIntoPrefixAndName(string path)
{
if (string.IsNullOrEmpty(path))
throw new ArgumentException($"Invalid path: {path}", nameof(path));
int nLast = path.TrimEnd(_pathseparators).LastIndexOfAny(_pathseparators);
if (nLast >= 0)
return new KeyValuePair<string, string>(path.Substring(0, nLast + 1), path.Substring(nLast + 1));
return new KeyValuePair<string, string>(string.Empty, path);
}
}
/// <summary>
/// Defines the backups types
/// </summary>
public static class BackupType
{
public const int PARTIAL_BACKUP = 0;
public const int FULL_BACKUP = 1;
}
}
Reference in New Issue View Git Blame Copy Permalink