2208 lines
97 KiB
C#
2208 lines
97 KiB
C#
// Copyright (C) 2025, The Duplicati Team
|
|
// https://duplicati.com, hello@duplicati.com
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a
|
|
// copy of this software and associated documentation files (the "Software"),
|
|
// to deal in the Software without restriction, including without limitation
|
|
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
// and/or sell copies of the Software, and to permit persons to whom the
|
|
// Software is furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in
|
|
// all copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
|
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
// DEALINGS IN THE SOFTWARE.
|
|
|
|
#nullable enable
|
|
|
|
using System;
|
|
using System.Collections.Concurrent;
|
|
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using System.Text;
|
|
using System.Threading.Tasks;
|
|
using Duplicati.Library.Common.IO;
|
|
using Duplicati.Library.Main.Operation.Restore;
|
|
using Duplicati.Library.Main.Volumes;
|
|
using Duplicati.Library.SQLiteHelper;
|
|
using Duplicati.Library.Utility;
|
|
using Microsoft.Data.Sqlite;
|
|
|
|
namespace Duplicati.Library.Main.Database
|
|
{
|
|
internal class LocalRestoreDatabase : LocalDatabase
|
|
{
|
|
/// <summary>
|
|
/// The tag used for logging
|
|
/// </summary>
|
|
private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalRestoreDatabase));
|
|
|
|
protected readonly string m_temptabsetguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
|
|
/// <summary>
|
|
/// The name of the temporary table in the database, which is used to store the list of files to restore.
|
|
/// </summary>
|
|
protected string? m_tempfiletable;
|
|
protected string? m_tempblocktable;
|
|
protected ConcurrentBag<(SqliteConnection, ReusableTransaction)> m_connection_pool = [];
|
|
protected string? m_latestblocktable;
|
|
protected string? m_fileprogtable;
|
|
protected string? m_totalprogtable;
|
|
protected string? m_filesnewlydonetable;
|
|
|
|
protected DateTime m_restoreTime;
|
|
|
|
public DateTime RestoreTime { get { return m_restoreTime; } }
|
|
|
|
public static async Task<LocalRestoreDatabase> CreateAsync(string path, long pagecachesize, LocalRestoreDatabase? dbnew = null)
|
|
{
|
|
dbnew ??= new LocalRestoreDatabase();
|
|
|
|
dbnew = (LocalRestoreDatabase)await CreateLocalDatabaseAsync(path, "Restore", false, pagecachesize, dbnew);
|
|
dbnew.ShouldCloseConnection = true;
|
|
|
|
return dbnew;
|
|
}
|
|
|
|
public static async Task<LocalRestoreDatabase> CreateAsync(LocalDatabase dbparent, LocalRestoreDatabase? dbnew = null)
|
|
{
|
|
dbnew ??= new LocalRestoreDatabase();
|
|
|
|
return (LocalRestoreDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Create tables and triggers for automatic tracking of progress during a restore operation.
|
|
/// This replaces continuous requerying of block progress by iterating over blocks table.
|
|
/// SQLite is much faster keeping information up to date with internal triggers.
|
|
/// </summary>
|
|
/// <param name="createFilesNewlyDoneTracker"> This allows to create another table that keeps track
|
|
/// of all files that are done (all data blocks restored). </param>
|
|
/// <remarks>
|
|
/// The method is prepared to create a table that keeps track of all files being done completely.
|
|
/// That means, it fires for files where the number of restored blocks equals the number of all blocks.
|
|
/// It is intended to be used for fast identification of fully restored files to trigger their verification.
|
|
/// It should be read after a commit and truncated after putting the files to a verification queue.
|
|
/// Note: If a file is done once and then set back to a none restored state, the file is not automatically removed.
|
|
/// But if it reaches a restored state later, it will be re-added (trigger will fire)
|
|
/// </remarks>
|
|
public async Task CreateProgressTracker(bool createFilesNewlyDoneTracker)
|
|
{
|
|
m_fileprogtable = "FileProgress-" + m_temptabsetguid;
|
|
m_totalprogtable = "TotalProgress-" + m_temptabsetguid;
|
|
m_filesnewlydonetable = createFilesNewlyDoneTracker ? "FilesNewlyDone-" + m_temptabsetguid : null;
|
|
|
|
using var cmd = m_connection.CreateCommand()
|
|
.SetTransaction(m_rtr);
|
|
|
|
// How to handle METADATA?
|
|
await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_fileprogtable}"" ");
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
CREATE TEMPORARY TABLE ""{m_fileprogtable}"" (
|
|
""FileId"" INTEGER PRIMARY KEY,
|
|
""TotalBlocks"" INTEGER NOT NULL,
|
|
""TotalSize"" INTEGER NOT NULL,
|
|
""BlocksRestored"" INTEGER NOT NULL,
|
|
""SizeRestored"" INTEGER NOT NULL
|
|
)
|
|
");
|
|
|
|
await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_totalprogtable}"" ");
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
CREATE TEMPORARY TABLE ""{m_totalprogtable}"" (
|
|
""TotalFiles"" INTEGER NOT NULL,
|
|
""TotalBlocks"" INTEGER NOT NULL,
|
|
""TotalSize"" INTEGER NOT NULL,
|
|
""FilesFullyRestored"" INTEGER NOT NULL,
|
|
""FilesPartiallyRestored"" INTEGER NOT NULL,
|
|
""BlocksRestored"" INTEGER NOT NULL,
|
|
""SizeRestored"" INTEGER NOT NULL
|
|
)
|
|
");
|
|
|
|
if (createFilesNewlyDoneTracker)
|
|
{
|
|
await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}"" ");
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
CREATE TEMPORARY TABLE ""{m_filesnewlydonetable}"" (
|
|
""ID"" INTEGER PRIMARY KEY
|
|
)
|
|
");
|
|
}
|
|
|
|
try
|
|
{
|
|
// Initialize statistics with File- and Block-Data (it is valid to already have restored blocks in files)
|
|
// A rebuild with this function should be valid anytime.
|
|
// Note: FilesNewlyDone is NOT initialized, as in initialization nothing is really new.
|
|
|
|
// We use a LEFT JOIN to allow for empty files (no data Blocks)
|
|
|
|
// Will be one row per file.
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
INSERT INTO ""{m_fileprogtable}"" (
|
|
""FileId"",
|
|
""TotalBlocks"",
|
|
""TotalSize"",
|
|
""BlocksRestored"",
|
|
""SizeRestored""
|
|
)
|
|
SELECT
|
|
""F"".""ID"",
|
|
IFNULL(COUNT(""B"".""ID""), 0),
|
|
IFNULL(SUM(""B"".""Size""), 0),
|
|
IFNULL(
|
|
COUNT(
|
|
CASE ""B"".""Restored""
|
|
WHEN 1
|
|
THEN ""B"".""ID""
|
|
ELSE NULL
|
|
END
|
|
),
|
|
0
|
|
),
|
|
IFNULL(
|
|
SUM(
|
|
CASE ""B"".""Restored""
|
|
WHEN 1
|
|
THEN ""B"".""Size""
|
|
ELSE 0
|
|
END
|
|
),
|
|
0
|
|
)
|
|
FROM ""{m_tempfiletable}"" ""F""
|
|
LEFT JOIN ""{m_tempblocktable}"" ""B""
|
|
ON ""B"".""FileID"" = ""F"".""ID""
|
|
WHERE ""B"".""Metadata"" IS NOT 1
|
|
GROUP BY ""F"".""ID""
|
|
");
|
|
|
|
// Will result in a single line (no support to also track metadata)
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
INSERT INTO ""{m_totalprogtable}"" (
|
|
""TotalFiles"",
|
|
""TotalBlocks"",
|
|
""TotalSize"",
|
|
""FilesFullyRestored"",
|
|
""FilesPartiallyRestored"",
|
|
""BlocksRestored"",
|
|
""SizeRestored""
|
|
)
|
|
SELECT
|
|
IFNULL(COUNT(""P"".""FileId""), 0),
|
|
IFNULL(SUM(""P"".""TotalBlocks""), 0),
|
|
IFNULL(SUM(""P"".""TotalSize""), 0),
|
|
IFNULL(
|
|
COUNT(
|
|
CASE
|
|
WHEN ""P"".""BlocksRestored"" = ""P"".""TotalBlocks""
|
|
THEN 1
|
|
ELSE NULL
|
|
END
|
|
),
|
|
0
|
|
),
|
|
IFNULL(
|
|
COUNT(
|
|
CASE
|
|
WHEN
|
|
""P"".""BlocksRestored"" BETWEEN 1
|
|
AND ""P"".""TotalBlocks"" - 1
|
|
THEN 1
|
|
ELSE NULL
|
|
END
|
|
),
|
|
0
|
|
),
|
|
IFNULL(SUM(""P"".""BlocksRestored""), 0),
|
|
IFNULL(SUM(""P"".""SizeRestored""), 0)
|
|
FROM ""{m_fileprogtable}"" ""P""
|
|
");
|
|
|
|
// Finally we create TRIGGERs to keep all our statistics up to date.
|
|
// This is lightning fast, as SQLite uses internal hooks and our indices to do the update magic.
|
|
// Note: We do assume that neither files nor blocks will be added or deleted during restore process
|
|
// and that the size of each block stays constant so there is no need to track that information
|
|
// with additional INSERT and DELETE triggers.
|
|
|
|
// A trigger to update the file-stat entry each time a block changes restoration state.
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
CREATE TEMPORARY TRIGGER ""TrackRestoredBlocks_{m_tempblocktable}""
|
|
AFTER UPDATE OF ""Restored""
|
|
ON ""{m_tempblocktable}""
|
|
WHEN OLD.""Restored"" != NEW.""Restored""
|
|
AND NEW.""Metadata"" = 0
|
|
BEGIN UPDATE ""{m_fileprogtable}""
|
|
SET
|
|
""BlocksRestored"" =
|
|
""{m_fileprogtable}"".""BlocksRestored""
|
|
+ (NEW.""Restored"" - OLD.""Restored""),
|
|
""SizeRestored"" =
|
|
""{m_fileprogtable}"".""SizeRestored""
|
|
+ ((NEW.""Restored"" - OLD.""Restored"") * NEW.Size)
|
|
WHERE ""{m_fileprogtable}"".""FileId"" = NEW.""FileID""
|
|
; END
|
|
");
|
|
|
|
// A trigger to update total stats each time a file stat changed (nested triggering by file-stats)
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
CREATE TEMPORARY TRIGGER ""UpdateTotalStats_{m_fileprogtable}""
|
|
AFTER UPDATE ON ""{m_fileprogtable}""
|
|
BEGIN UPDATE ""{m_totalprogtable}""
|
|
SET
|
|
""FilesFullyRestored"" =
|
|
""{m_totalprogtable}"".""FilesFullyRestored""
|
|
+ (CASE
|
|
WHEN NEW.""BlocksRestored"" = NEW.""TotalBlocks""
|
|
THEN 1
|
|
ELSE 0
|
|
END)
|
|
- (CASE
|
|
WHEN OLD.""BlocksRestored"" = OLD.""TotalBlocks""
|
|
THEN 1
|
|
ELSE 0
|
|
END),
|
|
""FilesPartiallyRestored"" =
|
|
""{m_totalprogtable}"".""FilesPartiallyRestored""
|
|
+ (CASE
|
|
WHEN
|
|
NEW.""BlocksRestored"" BETWEEN 1
|
|
AND NEW.""TotalBlocks"" - 1
|
|
THEN 1
|
|
ELSE 0
|
|
END)
|
|
- (CASE
|
|
WHEN
|
|
OLD.""BlocksRestored"" BETWEEN 1
|
|
AND OLD.""TotalBlocks"" - 1
|
|
THEN 1
|
|
ELSE 0
|
|
END),
|
|
""BlocksRestored"" =
|
|
""{m_totalprogtable}"".""BlocksRestored""
|
|
+ NEW.""BlocksRestored""
|
|
- OLD.""BlocksRestored"",
|
|
""SizeRestored"" =
|
|
""{m_totalprogtable}"".""SizeRestored""
|
|
+ NEW.""SizeRestored""
|
|
- OLD.""SizeRestored""
|
|
; END
|
|
");
|
|
|
|
|
|
if (createFilesNewlyDoneTracker)
|
|
{
|
|
// A trigger checking if a file is done (all blocks restored in file-stat) (nested triggering by file-stats)
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
CREATE TEMPORARY TRIGGER ""UpdateFilesNewlyDone_{m_fileprogtable}""
|
|
AFTER UPDATE OF
|
|
""BlocksRestored"",
|
|
""TotalBlocks""
|
|
ON ""{m_fileprogtable}""
|
|
WHEN NEW.""BlocksRestored"" = NEW.""TotalBlocks""
|
|
BEGIN
|
|
INSERT OR IGNORE INTO ""{m_filesnewlydonetable}"" (""ID"")
|
|
VALUES (NEW.""FileId"");
|
|
END
|
|
");
|
|
}
|
|
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
m_fileprogtable = null;
|
|
m_totalprogtable = null;
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "ProgressTrackerSetupError", ex, "Failed to set up progress tracking tables");
|
|
throw;
|
|
}
|
|
finally
|
|
{
|
|
await m_rtr.CommitAsync();
|
|
}
|
|
}
|
|
|
|
public async Task<Tuple<long, long>> PrepareRestoreFilelist(DateTime restoretime, long[] versions, IFilter filter)
|
|
{
|
|
m_tempfiletable = "Fileset-" + m_temptabsetguid;
|
|
m_tempblocktable = "Blocks-" + m_temptabsetguid;
|
|
|
|
using (var cmd = m_connection.CreateCommand())
|
|
{
|
|
cmd.SetTransaction(m_rtr);
|
|
var filesetIds = await GetFilesetIDs(Library.Utility.Utility.NormalizeDateTime(restoretime), versions).ToListAsync();
|
|
while (filesetIds.Count > 0)
|
|
{
|
|
var filesetId = filesetIds[0];
|
|
filesetIds.RemoveAt(0);
|
|
|
|
cmd.SetCommandAndParameters(@"
|
|
SELECT ""Timestamp""
|
|
FROM ""Fileset""
|
|
WHERE ""ID"" = @FilesetId
|
|
")
|
|
.SetParameterValue("@FilesetId", filesetId);
|
|
m_restoreTime = ParseFromEpochSeconds(await cmd.ExecuteScalarInt64Async(0));
|
|
|
|
var ix = await FilesetTimes()
|
|
.Select((value, index) => new { value.Key, index })
|
|
.Where(n => n.Key == filesetId)
|
|
.Select(pair => pair.index + 1)
|
|
.FirstOrDefaultAsync() - 1;
|
|
|
|
Logging.Log.WriteInformationMessage(LOGTAG, "SearchingBackup", "Searching backup {0} ({1}) ...", ix, m_restoreTime);
|
|
|
|
cmd.Parameters.Clear();
|
|
|
|
await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempfiletable}"" ");
|
|
await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempblocktable}"" ");
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
CREATE TEMPORARY TABLE ""{m_tempfiletable}"" (
|
|
""ID"" INTEGER PRIMARY KEY,
|
|
""Path"" TEXT NOT NULL,
|
|
""BlocksetID"" INTEGER NOT NULL,
|
|
""MetadataID"" INTEGER NOT NULL,
|
|
""TargetPath"" TEXT NULL,
|
|
""DataVerified"" BOOLEAN NOT NULL,
|
|
""LatestBlocksetId"" INTEGER,
|
|
""LocalSourceExists"" BOOLEAN
|
|
)
|
|
");
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
CREATE TEMPORARY TABLE ""{m_tempblocktable}"" (
|
|
""ID"" INTEGER PRIMARY KEY,
|
|
""FileID"" INTEGER NOT NULL,
|
|
""Index"" INTEGER NOT NULL,
|
|
""Hash"" TEXT NOT NULL,
|
|
""Size"" INTEGER NOT NULL,
|
|
""Restored"" BOOLEAN NOT NULL,
|
|
""Metadata"" BOOLEAN NOT NULL,
|
|
""VolumeID"" INTEGER NOT NULL,
|
|
""BlockID"" INTEGER NOT NULL
|
|
)
|
|
");
|
|
|
|
// TODO: Optimize to use the path prefix
|
|
|
|
if (filter == null || filter.Empty)
|
|
{
|
|
// Simple case, restore everything
|
|
await cmd.SetCommandAndParameters($@"
|
|
INSERT INTO ""{m_tempfiletable}"" (
|
|
""ID"",
|
|
""Path"",
|
|
""BlocksetID"",
|
|
""MetadataID"",
|
|
""DataVerified""
|
|
)
|
|
SELECT
|
|
""File"".""ID"",
|
|
""File"".""Path"",
|
|
""File"".""BlocksetID"",
|
|
""File"".""MetadataID"",
|
|
0
|
|
FROM
|
|
""File"",
|
|
""FilesetEntry""
|
|
WHERE
|
|
""File"".""ID"" = ""FilesetEntry"".""FileID""
|
|
AND ""FilesetEntry"".""FilesetID"" = @FilesetId
|
|
")
|
|
.SetParameterValue("@FilesetId", filesetId)
|
|
.ExecuteNonQueryAsync();
|
|
}
|
|
else if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == FilterType.Simple)
|
|
{
|
|
using (new Logging.Timer(LOGTAG, "CommitBeforePrepareFileset", "CommitBeforePrepareFileset"))
|
|
await m_rtr.CommitAsync();
|
|
cmd.SetTransaction(m_rtr);
|
|
// If we get a list of filenames, the lookup table is faster
|
|
// unfortunately we cannot do this if the filesystem is case sensitive as
|
|
// SQLite only supports ASCII compares
|
|
var p = expression.GetSimpleList();
|
|
var m_filenamestable = "Filenames-" + m_temptabsetguid;
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
CREATE TEMPORARY TABLE ""{m_filenamestable}"" (
|
|
""Path"" TEXT NOT NULL
|
|
)
|
|
");
|
|
cmd.SetCommandAndParameters($@"
|
|
INSERT INTO ""{m_filenamestable}"" (""Path"")
|
|
VALUES (@Path)
|
|
");
|
|
|
|
foreach (var s in p)
|
|
{
|
|
await cmd.SetParameterValue("@Path", s)
|
|
.ExecuteNonQueryAsync();
|
|
}
|
|
|
|
var c = await cmd.SetCommandAndParameters($@"
|
|
INSERT INTO ""{m_tempfiletable}"" (
|
|
""ID"",
|
|
""Path"",
|
|
""BlocksetID"",
|
|
""MetadataID"",
|
|
""DataVerified""
|
|
)
|
|
SELECT
|
|
""File"".""ID"",
|
|
""File"".""Path"",
|
|
""File"".""BlocksetID"",
|
|
""File"".""MetadataID"",
|
|
0
|
|
FROM
|
|
""File"",
|
|
""FilesetEntry""
|
|
WHERE
|
|
""File"".""ID"" = ""FilesetEntry"".""FileID""
|
|
AND ""FilesetEntry"".""FilesetID"" = @FilesetId
|
|
AND ""Path"" IN (
|
|
SELECT DISTINCT ""Path""
|
|
FROM ""{m_filenamestable}""
|
|
)
|
|
")
|
|
.SetParameterValue("@FilesetId", filesetId)
|
|
.ExecuteNonQueryAsync();
|
|
|
|
if (c != p.Length && c != 0)
|
|
{
|
|
var sb = new StringBuilder();
|
|
sb.AppendLine();
|
|
|
|
cmd.SetCommandAndParameters($@"
|
|
SELECT ""Path""
|
|
FROM ""{m_filenamestable}""
|
|
WHERE ""Path"" NOT IN (
|
|
SELECT ""Path""
|
|
FROM ""{m_tempfiletable}""
|
|
)
|
|
");
|
|
using (var rd = await cmd.ExecuteReaderAsync())
|
|
while (await rd.ReadAsync())
|
|
sb.AppendLine(rd.ConvertValueToString(0));
|
|
|
|
cmd.SetCommandAndParameters(@"
|
|
SELECT ""Timestamp""
|
|
FROM ""Fileset""
|
|
WHERE ""ID"" = @FilesetId
|
|
")
|
|
.SetParameterValue("@FilesetId", filesetId);
|
|
var actualrestoretime = ParseFromEpochSeconds(await cmd.ExecuteScalarInt64Async(0));
|
|
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "FilesNotFoundInBackupList", null, "{0} File(s) were not found in list of files for backup at {1}, will not be restored: {2}", p.Length - c, actualrestoretime.ToLocalTime(), sb);
|
|
cmd.Parameters.Clear();
|
|
}
|
|
|
|
await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filenamestable}"" ");
|
|
|
|
using (new Logging.Timer(LOGTAG, "CommitAfterPrepareFileset", "CommitAfterPrepareFileset"))
|
|
await m_rtr.CommitAsync();
|
|
cmd.SetTransaction(m_rtr);
|
|
}
|
|
else
|
|
{
|
|
// Restore but filter elements based on the filter expression
|
|
// If this is too slow, we could add a special handler for wildcard searches too
|
|
cmd.SetCommandAndParameters(@"
|
|
SELECT
|
|
""File"".""ID"",
|
|
""File"".""Path"",
|
|
""File"".""BlocksetID"",
|
|
""File"".""MetadataID""
|
|
FROM
|
|
""File"",
|
|
""FilesetEntry""
|
|
WHERE
|
|
""File"".""ID"" = ""FilesetEntry"".""FileID""
|
|
AND ""FilesetID"" = @FilesetId
|
|
")
|
|
.SetParameterValue("@FilesetId", filesetId);
|
|
|
|
object[] values = new object[4];
|
|
using var cmd2 = m_connection.CreateCommand($@"
|
|
INSERT INTO ""{m_tempfiletable}"" (
|
|
""ID"",
|
|
""Path"",
|
|
""BlocksetID"",
|
|
""MetadataID"",
|
|
""DataVerified""
|
|
)
|
|
VALUES (
|
|
@ID,
|
|
@Path,
|
|
@BlocksetID,
|
|
@MetadataID,
|
|
0
|
|
)
|
|
");
|
|
using var rd = await cmd.ExecuteReaderAsync();
|
|
while (await rd.ReadAsync())
|
|
{
|
|
rd.GetValues(values);
|
|
if (values[1] != null && values[1] != DBNull.Value && FilterExpression.Matches(filter, values[1].ToString()))
|
|
{
|
|
await cmd2.SetParameterValue("@ID", values[0])
|
|
.SetParameterValue("@Path", values[1])
|
|
.SetParameterValue("@BlocksetID", values[2])
|
|
.SetParameterValue("@MetadataID", values[3])
|
|
.ExecuteNonQueryAsync();
|
|
}
|
|
}
|
|
}
|
|
|
|
//creating indexes after insertion is much faster
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
CREATE INDEX ""{m_tempfiletable}_ID""
|
|
ON ""{m_tempfiletable}"" (""ID"")
|
|
");
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
CREATE INDEX ""{m_tempfiletable}_TargetPath""
|
|
ON ""{m_tempfiletable}"" (""TargetPath"")
|
|
");
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
CREATE INDEX ""{m_tempfiletable}_Path""
|
|
ON ""{m_tempfiletable}"" (""Path"")
|
|
");
|
|
|
|
cmd.SetCommandAndParameters($@"
|
|
SELECT
|
|
COUNT(DISTINCT ""{m_tempfiletable}"".""Path""),
|
|
SUM(""Blockset"".""Length"")
|
|
FROM
|
|
""{m_tempfiletable}"",
|
|
""Blockset""
|
|
WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID""
|
|
");
|
|
using (var rd = await cmd.ExecuteReaderAsync())
|
|
{
|
|
var filecount = 0L;
|
|
var filesize = 0L;
|
|
|
|
if (await rd.ReadAsync())
|
|
{
|
|
filecount = rd.ConvertValueToInt64(0, 0);
|
|
filesize = rd.ConvertValueToInt64(1, 0);
|
|
}
|
|
|
|
if (filecount > 0)
|
|
{
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreTargetFileCount", "Needs to restore {0} files ({1})", filecount, Library.Utility.Utility.FormatSizeString(filesize));
|
|
return new Tuple<long, long>(filecount, filesize);
|
|
}
|
|
}
|
|
}
|
|
|
|
await m_rtr.CommitAsync();
|
|
}
|
|
|
|
return new Tuple<long, long>(0, 0);
|
|
}
|
|
|
|
public async Task<string?> GetFirstPath()
|
|
{
|
|
using var cmd = m_connection.CreateCommand($@"
|
|
SELECT ""Path""
|
|
FROM ""{m_tempfiletable}""
|
|
ORDER BY LENGTH(""Path"") DESC
|
|
LIMIT 1
|
|
")
|
|
.SetTransaction(m_rtr);
|
|
|
|
var v0 = await cmd.ExecuteScalarAsync();
|
|
if (v0 == null || v0 == DBNull.Value)
|
|
return null;
|
|
|
|
await m_rtr.CommitAsync();
|
|
|
|
return v0.ToString();
|
|
}
|
|
|
|
public async Task<string> GetLargestPrefix()
|
|
{
|
|
using var cmd = m_connection.CreateCommand($@"
|
|
SELECT ""Path""
|
|
FROM ""{m_tempfiletable}""
|
|
ORDER BY LENGTH(""Path"") DESC
|
|
LIMIT 1
|
|
")
|
|
.SetTransaction(m_rtr);
|
|
|
|
var v0 = await cmd.ExecuteScalarAsync();
|
|
var maxpath = "";
|
|
if (v0 != null && v0 != DBNull.Value)
|
|
maxpath = v0.ToString()!;
|
|
|
|
var dirsep = Util.GuessDirSeparator(maxpath);
|
|
|
|
var filecount = await cmd.ExecuteScalarInt64Async($@"
|
|
SELECT COUNT(*)
|
|
FROM ""{m_tempfiletable}""
|
|
", -1);
|
|
var foundfiles = -1L;
|
|
|
|
//TODO: Handle FS case-sensitive?
|
|
cmd.SetCommandAndParameters($@"
|
|
SELECT COUNT(*)
|
|
FROM ""{m_tempfiletable}""
|
|
WHERE SUBSTR(""Path"", 1, @PrefixLength) = @Prefix
|
|
");
|
|
|
|
while (filecount != foundfiles && maxpath.Length > 0)
|
|
{
|
|
var mp = Util.AppendDirSeparator(maxpath, dirsep);
|
|
foundfiles = await cmd.SetParameterValue("@PrefixLength", mp.Length)
|
|
.SetParameterValue("@Prefix", mp)
|
|
.ExecuteScalarInt64Async(-1);
|
|
|
|
if (filecount != foundfiles)
|
|
{
|
|
var oldlen = maxpath.Length;
|
|
|
|
var lix = maxpath.LastIndexOf(dirsep, maxpath.Length - 2, StringComparison.Ordinal);
|
|
maxpath = maxpath.Substring(0, lix + 1);
|
|
if (string.IsNullOrWhiteSpace(maxpath) || maxpath.Length == oldlen)
|
|
maxpath = "";
|
|
}
|
|
}
|
|
|
|
await m_rtr.CommitAsync();
|
|
|
|
return maxpath == "" ? "" : Util.AppendDirSeparator(maxpath, dirsep);
|
|
}
|
|
|
|
public async Task SetTargetPaths(string largest_prefix, string destination)
|
|
{
|
|
var dirsep = Util.GuessDirSeparator(string.IsNullOrWhiteSpace(largest_prefix) ? await GetFirstPath() : largest_prefix);
|
|
|
|
using var cmd = m_connection.CreateCommand()
|
|
.SetTransaction(m_rtr);
|
|
if (string.IsNullOrEmpty(destination))
|
|
{
|
|
//The string fixing here is meant to provide some non-random
|
|
// defaults when restoring cross OS, e.g. backup on Linux, restore on Windows
|
|
//This is mostly meaningless, and the user really should use --restore-path
|
|
|
|
if ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) && dirsep == "\\")
|
|
{
|
|
// For Win -> Linux, we remove the colon from the drive letter, and use the drive letter as root folder
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
UPDATE ""{m_tempfiletable}""
|
|
SET ""Targetpath"" =
|
|
CASE
|
|
WHEN SUBSTR(""Path"", 2, 1) == ':'
|
|
THEN '\\' || SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3)
|
|
ELSE ""Path""
|
|
END
|
|
");
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
UPDATE ""{m_tempfiletable}""
|
|
SET ""Targetpath"" =
|
|
CASE
|
|
WHEN SUBSTR(""Path"", 1, 2) == '\\'
|
|
THEN '\\' || SUBSTR(""Path"", 2)
|
|
ELSE ""Path""
|
|
END
|
|
");
|
|
|
|
}
|
|
else if (OperatingSystem.IsWindows() && dirsep == "/")
|
|
{
|
|
// For Linux -> Win, we use the temporary folder's drive as the root path
|
|
await cmd.SetCommandAndParameters($@"
|
|
UPDATE ""{m_tempfiletable}""
|
|
SET ""Targetpath"" =
|
|
CASE
|
|
WHEN SUBSTR(""Path"", 1, 1) == '/'
|
|
THEN @Path || SUBSTR(""Path"", 2)
|
|
ELSE ""Path""
|
|
END
|
|
")
|
|
.SetParameterValue("@Path", Util.AppendDirSeparator(System.IO.Path.GetPathRoot(Library.Utility.TempFolder.SystemTempPath)).Replace("\\", "/"))
|
|
.ExecuteNonQueryAsync();
|
|
}
|
|
else
|
|
{
|
|
// Same OS, just use the path directly
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
UPDATE ""{m_tempfiletable}""
|
|
SET ""Targetpath"" = ""Path""
|
|
");
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (string.IsNullOrEmpty(largest_prefix))
|
|
{
|
|
//Special case, restoring to new folder, but files are from different drives (no shared root on Windows)
|
|
|
|
// We use the format <restore path> / <drive letter> / <source path>
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
UPDATE ""{m_tempfiletable}""
|
|
SET ""TargetPath"" =
|
|
CASE
|
|
WHEN SUBSTR(""Path"", 2, 1) == ':'
|
|
THEN SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3)
|
|
ELSE ""Path""
|
|
END
|
|
");
|
|
|
|
// For UNC paths, we use \\server\folder -> <restore path> / <servername> / <source path>
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
UPDATE ""{m_tempfiletable}""
|
|
SET ""TargetPath"" =
|
|
CASE
|
|
WHEN SUBSTR(""Path"", 1, 2) == '\\'
|
|
THEN SUBSTR(""Path"", 2)
|
|
ELSE ""TargetPath""
|
|
END
|
|
");
|
|
}
|
|
else
|
|
{
|
|
largest_prefix = Util.AppendDirSeparator(largest_prefix, dirsep);
|
|
await cmd.SetCommandAndParameters($@"
|
|
UPDATE ""{m_tempfiletable}""
|
|
SET ""TargetPath"" = SUBSTR(""Path"", @PrefixLength)
|
|
")
|
|
.SetParameterValue("@PrefixLength", largest_prefix.Length + 1)
|
|
.ExecuteNonQueryAsync();
|
|
}
|
|
}
|
|
|
|
// Cross-os path remapping support
|
|
if ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) && dirsep == "\\")
|
|
// For Win paths on Linux
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
UPDATE ""{m_tempfiletable}""
|
|
SET ""TargetPath"" = REPLACE(""TargetPath"", '\', '/')
|
|
");
|
|
else if (OperatingSystem.IsWindows() && dirsep == "/")
|
|
// For Linux paths on Windows
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
UPDATE ""{m_tempfiletable}""
|
|
SET ""TargetPath"" = REPLACE(REPLACE(""TargetPath"", '\', '_'), '/', '\')
|
|
");
|
|
|
|
if (!string.IsNullOrEmpty(destination))
|
|
{
|
|
// Paths are now relative with target-os naming system
|
|
// so we prefix them with the target path
|
|
await cmd.SetCommandAndParameters($@"
|
|
UPDATE ""{m_tempfiletable}""
|
|
SET ""TargetPath"" = @Destination || ""TargetPath""
|
|
")
|
|
.SetParameterValue("@Destination", Util.AppendDirSeparator(destination))
|
|
.ExecuteNonQueryAsync();
|
|
}
|
|
|
|
await m_rtr.CommitAsync();
|
|
}
|
|
|
|
public async Task FindMissingBlocks(bool skipMetadata)
|
|
{
|
|
using var cmd = m_connection.CreateCommand()
|
|
.SetTransaction(m_rtr);
|
|
var p1 = await cmd.ExecuteNonQueryAsync($@"
|
|
INSERT INTO ""{m_tempblocktable}"" (
|
|
""FileID"",
|
|
""Index"",
|
|
""Hash"",
|
|
""Size"",
|
|
""Restored"",
|
|
""Metadata"",
|
|
""VolumeId"",
|
|
""BlockId""
|
|
)
|
|
SELECT DISTINCT
|
|
""{m_tempfiletable}"".""ID"",
|
|
""BlocksetEntry"".""Index"",
|
|
""Block"".""Hash"",
|
|
""Block"".""Size"",
|
|
0,
|
|
0,
|
|
""Block"".""VolumeID"",
|
|
""Block"".""ID""
|
|
FROM
|
|
""{m_tempfiletable}"",
|
|
""BlocksetEntry"",
|
|
""Block""
|
|
WHERE
|
|
""{m_tempfiletable}"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID""
|
|
AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID""
|
|
");
|
|
|
|
var p2 = 0;
|
|
if (!skipMetadata)
|
|
p2 = await cmd.ExecuteNonQueryAsync($@"
|
|
INSERT INTO ""{m_tempblocktable}"" (
|
|
""FileID"",
|
|
""Index"",
|
|
""Hash"",
|
|
""Size"",
|
|
""Restored"",
|
|
""Metadata"",
|
|
""VolumeId"",
|
|
""BlockId""
|
|
)
|
|
SELECT DISTINCT
|
|
""{m_tempfiletable}"".""ID"",
|
|
""BlocksetEntry"".""Index"",
|
|
""Block"".""Hash"",
|
|
""Block"".""Size"",
|
|
0,
|
|
1,
|
|
""Block"".""VolumeID"",
|
|
""Block"".""ID""
|
|
FROM
|
|
""{m_tempfiletable}"",
|
|
""BlocksetEntry"",
|
|
""Block"",
|
|
""Metadataset""
|
|
WHERE
|
|
""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID""
|
|
AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID""
|
|
AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID""
|
|
");
|
|
|
|
//creating indexes after insertion is much faster
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
CREATE INDEX ""{m_tempblocktable}_HashSizeIndex""
|
|
ON ""{m_tempblocktable}"" (""Hash"", ""Size"")
|
|
");
|
|
// better suited to speed up commit on UpdateBlocks
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
CREATE INDEX ""{m_tempblocktable}_FileIdIndexIndex""
|
|
ON ""{m_tempblocktable}"" (""FileId"", ""Index"")
|
|
");
|
|
|
|
var size = await cmd.ExecuteScalarInt64Async($@"
|
|
SELECT SUM(""Size"")
|
|
FROM ""{m_tempblocktable}""
|
|
", 0);
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreSourceSize", "Restore list contains {0} blocks with a total size of {1}", p1 + p2, Library.Utility.Utility.FormatSizeString(size));
|
|
|
|
await m_rtr.CommitAsync();
|
|
}
|
|
|
|
public async Task UpdateTargetPath(long ID, string newname)
|
|
{
|
|
using var cmd = m_connection.CreateCommand($@"
|
|
UPDATE ""{m_tempfiletable}""
|
|
SET ""TargetPath"" = @TargetPath
|
|
WHERE ""ID"" = @ID
|
|
");
|
|
await cmd.SetTransaction(m_rtr)
|
|
.SetParameterValue("@TargetPath", newname)
|
|
.SetParameterValue("@ID", ID)
|
|
.ExecuteNonQueryAsync();
|
|
|
|
await m_rtr.CommitAsync();
|
|
}
|
|
|
|
public interface IExistingFileBlock
|
|
{
|
|
string Hash { get; }
|
|
long Index { get; }
|
|
long Size { get; }
|
|
}
|
|
|
|
public interface IExistingFile
|
|
{
|
|
string TargetPath { get; }
|
|
string TargetHash { get; }
|
|
long TargetFileID { get; }
|
|
long Length { get; }
|
|
IAsyncEnumerable<IExistingFileBlock> Blocks();
|
|
}
|
|
|
|
public interface IBlockSource
|
|
{
|
|
string Path { get; }
|
|
long Offset { get; }
|
|
bool IsMetadata { get; }
|
|
}
|
|
|
|
public interface IBlockDescriptor
|
|
{
|
|
string Hash { get; }
|
|
long Size { get; }
|
|
long Offset { get; }
|
|
long Index { get; }
|
|
bool IsMetadata { get; }
|
|
IAsyncEnumerable<IBlockSource> BlockSources();
|
|
}
|
|
|
|
public interface ILocalBlockSource
|
|
{
|
|
string TargetPath { get; }
|
|
long TargetFileID { get; }
|
|
IAsyncEnumerable<IBlockDescriptor> Blocks();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Interface for an object describing a file to restore.
|
|
/// </summary>
|
|
public interface IFileToRestore
|
|
{
|
|
string Path { get; }
|
|
string Hash { get; }
|
|
long Length { get; }
|
|
}
|
|
|
|
public interface IPatchBlock
|
|
{
|
|
long Offset { get; }
|
|
long Size { get; }
|
|
string Key { get; }
|
|
}
|
|
|
|
public interface IVolumePatch
|
|
{
|
|
string Path { get; }
|
|
long FileID { get; }
|
|
IAsyncEnumerable<IPatchBlock> Blocks();
|
|
}
|
|
|
|
private class ExistingFile : IExistingFile
|
|
{
|
|
private readonly SqliteDataReader m_reader;
|
|
|
|
public ExistingFile(SqliteDataReader rd) { m_reader = rd; HasMore = true; }
|
|
|
|
public string TargetPath { get { return m_reader.ConvertValueToString(0) ?? ""; } }
|
|
public string TargetHash { get { return m_reader.ConvertValueToString(1) ?? ""; } }
|
|
public long TargetFileID { get { return m_reader.ConvertValueToInt64(2); } }
|
|
public long Length { get { return m_reader.ConvertValueToInt64(3); } }
|
|
|
|
public bool HasMore { get; private set; }
|
|
|
|
private class ExistingFileBlock : IExistingFileBlock
|
|
{
|
|
private readonly SqliteDataReader m_reader;
|
|
|
|
public ExistingFileBlock(SqliteDataReader rd) { m_reader = rd; }
|
|
|
|
public string Hash { get { return m_reader.ConvertValueToString(4) ?? ""; } }
|
|
public long Index { get { return m_reader.ConvertValueToInt64(5); } }
|
|
public long Size { get { return m_reader.ConvertValueToInt64(6); } }
|
|
}
|
|
|
|
public async IAsyncEnumerable<IExistingFileBlock> Blocks()
|
|
{
|
|
string p = TargetPath;
|
|
while (HasMore && p == TargetPath)
|
|
{
|
|
yield return new ExistingFileBlock(m_reader);
|
|
HasMore = await m_reader.ReadAsync();
|
|
}
|
|
}
|
|
|
|
public static async IAsyncEnumerable<IExistingFile> GetExistingFilesWithBlocks(LocalDatabase db, string tablename)
|
|
{
|
|
using var cmd = db.Connection.CreateCommand($@"
|
|
SELECT
|
|
""{tablename}"".""TargetPath"",
|
|
""Blockset"".""FullHash"",
|
|
""{tablename}"".""ID"",
|
|
""Blockset"".""Length"",
|
|
""Block"".""Hash"",
|
|
""BlocksetEntry"".""Index"",
|
|
""Block"".""Size""
|
|
FROM
|
|
""{tablename}"",
|
|
""Blockset"",
|
|
""BlocksetEntry"",
|
|
""Block""
|
|
WHERE
|
|
""{tablename}"".""BlocksetID"" = ""Blockset"".""ID""
|
|
AND ""BlocksetEntry"".""BlocksetID"" = ""{tablename}"".""BlocksetID""
|
|
AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID""
|
|
ORDER BY
|
|
""{tablename}"".""TargetPath"",
|
|
""BlocksetEntry"".""Index""
|
|
")
|
|
.SetTransaction(db.Transaction);
|
|
using var rd = await cmd.ExecuteReaderAsync();
|
|
if (await rd.ReadAsync())
|
|
{
|
|
var more = true;
|
|
while (more)
|
|
{
|
|
var f = new ExistingFile(rd);
|
|
string current = f.TargetPath;
|
|
yield return f;
|
|
|
|
more = f.HasMore;
|
|
while (more && current == f.TargetPath)
|
|
more = await rd.ReadAsync();
|
|
}
|
|
}
|
|
await db.Transaction.CommitAsync();
|
|
}
|
|
}
|
|
|
|
public IAsyncEnumerable<IExistingFile> GetExistingFilesWithBlocks()
|
|
{
|
|
if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable))
|
|
throw new InvalidOperationException("No temporary file table set up for this restore.");
|
|
return ExistingFile.GetExistingFilesWithBlocks(this, m_tempfiletable);
|
|
}
|
|
|
|
private class LocalBlockSource : ILocalBlockSource
|
|
{
|
|
private class BlockDescriptor : IBlockDescriptor
|
|
{
|
|
private class BlockSource : IBlockSource
|
|
{
|
|
private readonly SqliteDataReader m_reader;
|
|
public BlockSource(SqliteDataReader rd) { m_reader = rd; }
|
|
|
|
public string Path { get { return m_reader.ConvertValueToString(6) ?? ""; } }
|
|
public long Offset { get { return m_reader.ConvertValueToInt64(7); } }
|
|
public bool IsMetadata { get { return false; } }
|
|
}
|
|
|
|
private readonly SqliteDataReader m_reader;
|
|
public BlockDescriptor(SqliteDataReader rd) { m_reader = rd; HasMore = true; }
|
|
|
|
private string TargetPath { get { return m_reader.ConvertValueToString(0) ?? ""; } }
|
|
|
|
public string Hash { get { return m_reader.ConvertValueToString(2) ?? ""; } }
|
|
public long Offset { get { return m_reader.ConvertValueToInt64(3); } }
|
|
public long Index { get { return m_reader.ConvertValueToInt64(4); } }
|
|
public long Size { get { return m_reader.ConvertValueToInt64(5); } }
|
|
public bool IsMetadata { get { return !(m_reader.ConvertValueToInt64(9) == 0); } }
|
|
|
|
public bool HasMore { get; private set; }
|
|
|
|
public async IAsyncEnumerable<IBlockSource> BlockSources()
|
|
{
|
|
var p = TargetPath;
|
|
var h = Hash;
|
|
var s = Size;
|
|
|
|
while (HasMore && p == TargetPath && h == Hash && s == Size)
|
|
{
|
|
yield return new BlockSource(m_reader);
|
|
HasMore = await m_reader.ReadAsync();
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
private readonly SqliteDataReader m_reader;
|
|
public LocalBlockSource(SqliteDataReader rd) { m_reader = rd; HasMore = true; }
|
|
|
|
public string TargetPath { get { return m_reader.ConvertValueToString(0) ?? ""; } }
|
|
public long TargetFileID { get { return m_reader.ConvertValueToInt64(1); } }
|
|
|
|
public bool HasMore { get; private set; }
|
|
|
|
public async IAsyncEnumerable<IBlockDescriptor> Blocks()
|
|
{
|
|
var p = TargetPath;
|
|
while (HasMore && p == TargetPath)
|
|
{
|
|
var c = new BlockDescriptor(m_reader);
|
|
var h = c.Hash;
|
|
var s = c.Size;
|
|
|
|
yield return c;
|
|
|
|
HasMore = c.HasMore;
|
|
while (HasMore && c.Hash == h && c.Size == s && TargetPath == p)
|
|
HasMore = await m_reader.ReadAsync();
|
|
}
|
|
}
|
|
|
|
public static async IAsyncEnumerable<ILocalBlockSource> GetFilesAndSourceBlocks(LocalDatabase db, string filetablename, string blocktablename, long blocksize, bool skipMetadata)
|
|
{
|
|
// TODO: Skip metadata as required
|
|
// Have to order by target path and hash, to ensure BlockDescriptor and BlockSource match adjacent rows
|
|
using var cmd = db.Connection.CreateCommand($@"
|
|
SELECT DISTINCT
|
|
""A"".""TargetPath"",
|
|
""A"".""ID"",
|
|
""B"".""Hash"",
|
|
(""B"".""Index"" * {blocksize}),
|
|
""B"".""Index"",
|
|
""B"".""Size"",
|
|
""C"".""Path"",
|
|
(""D"".""Index"" * {blocksize}),
|
|
""E"".""Size"",
|
|
""B"".""Metadata""
|
|
FROM
|
|
""{filetablename}"" ""A"",
|
|
""{blocktablename}"" ""B"",
|
|
""File"" ""C"",
|
|
""BlocksetEntry"" ""D"",
|
|
""Block"" E
|
|
WHERE
|
|
""A"".""ID"" = ""B"".""FileID""
|
|
AND ""C"".""BlocksetID"" = ""D"".""BlocksetID""
|
|
AND ""D"".""BlockID"" = ""E"".""ID""
|
|
AND ""B"".""Hash"" = ""E"".""Hash""
|
|
AND ""B"".""Size"" = ""E"".""Size""
|
|
AND ""B"".""Restored"" = 0
|
|
ORDER BY
|
|
""A"".""TargetPath"",
|
|
""B"".""Index""
|
|
")
|
|
.SetTransaction(db.Transaction);
|
|
using var rd = await cmd.ExecuteReaderAsync();
|
|
var more = await rd.ReadAsync();
|
|
|
|
while (more)
|
|
{
|
|
var f = new LocalBlockSource(rd);
|
|
string current = f.TargetPath;
|
|
yield return f;
|
|
|
|
more = f.HasMore;
|
|
while (more && current == f.TargetPath)
|
|
more = await rd.ReadAsync();
|
|
}
|
|
|
|
await db.Transaction.CommitAsync();
|
|
}
|
|
}
|
|
|
|
public IAsyncEnumerable<ILocalBlockSource> GetFilesAndSourceBlocks(bool skipMetadata, long blocksize)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable))
|
|
throw new InvalidOperationException("No temporary file table set up for this restore.");
|
|
return LocalBlockSource.GetFilesAndSourceBlocks(this, m_tempfiletable, m_tempblocktable, blocksize, skipMetadata);
|
|
}
|
|
|
|
public async IAsyncEnumerable<IRemoteVolume> GetMissingVolumes()
|
|
{
|
|
using var cmd = m_connection.CreateCommand($@"
|
|
SELECT
|
|
""RV"".""Name"",
|
|
""RV"".""Hash"",
|
|
""RV"".""Size"",
|
|
""BB"".""MaxIndex""
|
|
FROM ""RemoteVolume"" ""RV""
|
|
INNER JOIN (
|
|
SELECT
|
|
""TB"".""VolumeID"",
|
|
MAX(""TB"".""Index"") as ""MaxIndex""
|
|
FROM ""{m_tempblocktable}"" ""TB""
|
|
WHERE ""TB"".""Restored"" = 0
|
|
GROUP BY ""TB"".""VolumeID""
|
|
) as ""BB""
|
|
ON ""RV"".""ID"" = ""BB"".""VolumeID""
|
|
ORDER BY ""BB"".""MaxIndex""
|
|
")
|
|
.SetTransaction(m_rtr);
|
|
|
|
// Return order from SQLite-DISTINCT is likely to be sorted by Name, which is bad for restore.
|
|
// If the end of very large files (e.g. iso's) is restored before the beginning, most OS write out zeros to fill the file.
|
|
// If we manage to get the volumes in an order restoring front blocks first, this can save time.
|
|
// An optimal algorithm would build a dependency net with cycle resolution to find the best near topological
|
|
// order of volumes, but this is a bit too fancy here.
|
|
// We will just put a very simple heuristic to work, that will try to prefer volumes containing lower block indexes:
|
|
// We just order all volumes by the maximum block index they contain. This query is slow, but should be worth the effort.
|
|
// Now it is likely to restore all files from front to back. Large files will always be done last.
|
|
// One could also use like the average block number in a volume, that needs to be measured.
|
|
|
|
using var rd = await cmd.ExecuteReaderAsync();
|
|
object[] r = new object[3];
|
|
while (await rd.ReadAsync())
|
|
{
|
|
rd.GetValues(r);
|
|
yield return new RemoteVolume(
|
|
rd.ConvertValueToString(0),
|
|
rd.ConvertValueToString(1),
|
|
rd.ConvertValueToInt64(2, -1)
|
|
);
|
|
}
|
|
|
|
await m_rtr.CommitAsync();
|
|
}
|
|
|
|
public interface IFilesAndMetadata : IDisposable
|
|
{
|
|
IAsyncEnumerable<IVolumePatch> FilesWithMissingBlocks();
|
|
IAsyncEnumerable<IVolumePatch> MetadataWithMissingBlocks();
|
|
}
|
|
|
|
private class FilesAndMetadata : IFilesAndMetadata
|
|
{
|
|
private string m_tmptable = null!;
|
|
private string m_filetablename = null!;
|
|
private string m_blocktablename = null!;
|
|
private long m_blocksize;
|
|
|
|
private LocalDatabase m_db = null!;
|
|
|
|
[Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")]
|
|
public FilesAndMetadata(SqliteConnection connection, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume)
|
|
{
|
|
throw new NotImplementedException("Use CreateAsync instead of the constructor");
|
|
}
|
|
|
|
private FilesAndMetadata() { }
|
|
|
|
public static async Task<FilesAndMetadata> CreateAsync(LocalDatabase db, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume)
|
|
{
|
|
var fam = new FilesAndMetadata()
|
|
{
|
|
m_db = db,
|
|
m_filetablename = filetablename,
|
|
m_blocktablename = blocktablename,
|
|
m_blocksize = blocksize,
|
|
};
|
|
|
|
using var c = db.Connection.CreateCommand()
|
|
.SetTransaction(db.Transaction);
|
|
fam.m_tmptable = "VolumeFiles-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
|
|
await c.ExecuteNonQueryAsync($@"
|
|
CREATE TEMPORARY TABLE ""{fam.m_tmptable}"" (
|
|
""Hash"" TEXT NOT NULL,
|
|
""Size"" INTEGER NOT NULL
|
|
)
|
|
");
|
|
|
|
c.SetCommandAndParameters($@"
|
|
INSERT INTO ""{fam.m_tmptable}"" (
|
|
""Hash"",
|
|
""Size""
|
|
)
|
|
VALUES (
|
|
@Hash,
|
|
@Size)
|
|
");
|
|
foreach (var s in curvolume.Blocks)
|
|
{
|
|
await c.SetParameterValue("@Hash", s.Key)
|
|
.SetParameterValue("@Size", s.Value)
|
|
.ExecuteNonQueryAsync();
|
|
}
|
|
|
|
// The index _HashSizeIndex is not needed anymore. Index on "Blocks-..." is used on Join in GetMissingBlocks
|
|
|
|
await db.Transaction.CommitAsync();
|
|
|
|
return fam;
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
DisposeAsync().Await();
|
|
}
|
|
|
|
public async Task DisposeAsync()
|
|
{
|
|
if (m_tmptable != null)
|
|
{
|
|
using var c = m_db.Connection.CreateCommand(@"DROP TABLE IF EXISTS ""{m_tmptable}""")
|
|
.SetTransaction(m_db.Transaction);
|
|
await c.ExecuteNonQueryAsync();
|
|
await m_db.Transaction.CommitAsync();
|
|
}
|
|
}
|
|
|
|
private class VolumePatch : IVolumePatch
|
|
{
|
|
private class PatchBlock : IPatchBlock
|
|
{
|
|
private readonly SqliteDataReader m_reader;
|
|
public PatchBlock(SqliteDataReader rd) { m_reader = rd; }
|
|
|
|
public long Offset { get { return m_reader.ConvertValueToInt64(2); } }
|
|
public long Size { get { return m_reader.ConvertValueToInt64(3); } }
|
|
public string Key { get { return m_reader.ConvertValueToString(4) ?? ""; } }
|
|
}
|
|
|
|
private readonly SqliteDataReader m_reader;
|
|
public VolumePatch(SqliteDataReader rd) { m_reader = rd; HasMore = true; }
|
|
|
|
public string Path { get { return m_reader.ConvertValueToString(0) ?? ""; } }
|
|
public long FileID { get { return m_reader.ConvertValueToInt64(1); } }
|
|
public bool HasMore { get; private set; }
|
|
|
|
public async IAsyncEnumerable<IPatchBlock> Blocks()
|
|
{
|
|
string p = Path;
|
|
while (HasMore && p == Path)
|
|
{
|
|
yield return new PatchBlock(m_reader);
|
|
HasMore = await m_reader.ReadAsync();
|
|
}
|
|
}
|
|
}
|
|
|
|
public async IAsyncEnumerable<IVolumePatch> FilesWithMissingBlocks()
|
|
{
|
|
// The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here!
|
|
using var cmd = m_db.Connection.CreateCommand($@"
|
|
SELECT DISTINCT
|
|
""A"".""TargetPath"",
|
|
""BB"".""FileID"",
|
|
(""BB"".""Index"" * {m_blocksize}),
|
|
""BB"".""Size"",
|
|
""BB"".""Hash""
|
|
FROM
|
|
""{m_filetablename}"" ""A"",
|
|
""{m_blocktablename}"" ""BB""
|
|
WHERE
|
|
""A"".""ID"" = ""BB"".""FileID""
|
|
AND ""BB"".""Restored"" = 0
|
|
AND ""BB"".""Metadata"" = {"0"}
|
|
AND ""BB"".""ID"" IN (
|
|
SELECT ""B"".""ID""
|
|
FROM
|
|
""{m_blocktablename}"" ""B"",
|
|
""{m_tmptable}"" ""C""
|
|
WHERE
|
|
""B"".""Hash"" = ""C"".""Hash""
|
|
AND ""B"".""Size"" = ""C"".""Size""
|
|
)
|
|
ORDER BY
|
|
""A"".""TargetPath"",
|
|
""BB"".""Index""
|
|
")
|
|
.SetTransaction(m_db.Transaction);
|
|
|
|
using var rd = await cmd.ExecuteReaderAsync();
|
|
if (await rd.ReadAsync())
|
|
{
|
|
var more = true;
|
|
while (more)
|
|
{
|
|
var f = new VolumePatch(rd);
|
|
var current = f.Path;
|
|
yield return f;
|
|
|
|
more = f.HasMore;
|
|
while (more && current == f.Path)
|
|
more = await rd.ReadAsync();
|
|
}
|
|
}
|
|
await m_db.Transaction.Transaction.CommitAsync();
|
|
}
|
|
|
|
public async IAsyncEnumerable<IVolumePatch> MetadataWithMissingBlocks()
|
|
{
|
|
// The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here!
|
|
using var cmd = m_db.Connection.CreateCommand($@"
|
|
SELECT DISTINCT
|
|
""A"".""TargetPath"",
|
|
""BB"".""FileID"",
|
|
(""BB"".""Index"" * {m_blocksize}),
|
|
""BB"".""Size"",
|
|
""BB"".""Hash""
|
|
FROM
|
|
""{m_filetablename}"" ""A"",
|
|
""{m_blocktablename}"" ""BB""
|
|
WHERE
|
|
""A"".""ID"" = ""BB"".""FileID""
|
|
AND ""BB"".""Restored"" = 0
|
|
AND ""BB"".""Metadata"" = {"1"}
|
|
AND ""BB"".""ID"" IN (
|
|
SELECT ""B"".""ID""
|
|
FROM
|
|
""{m_blocktablename}"" ""B"",
|
|
""{m_tmptable}"" ""C""
|
|
WHERE
|
|
""B"".""Hash"" = ""C"".""Hash""
|
|
AND ""B"".""Size"" = ""C"".""Size""
|
|
)
|
|
ORDER BY
|
|
""A"".""TargetPath"",
|
|
""BB"".""Index""
|
|
")
|
|
.SetTransaction(m_db.Transaction);
|
|
|
|
using var rd = await cmd.ExecuteReaderAsync();
|
|
if (await rd.ReadAsync())
|
|
{
|
|
var more = true;
|
|
while (more)
|
|
{
|
|
var f = new VolumePatch(rd);
|
|
string current = f.Path;
|
|
yield return f;
|
|
|
|
more = f.HasMore;
|
|
while (more && current == f.Path)
|
|
more = await rd.ReadAsync();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
public async Task<IFilesAndMetadata> GetMissingBlockData(BlockVolumeReader curvolume, long blocksize)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable))
|
|
throw new InvalidOperationException("No temporary file table set up for this restore.");
|
|
return await FilesAndMetadata.CreateAsync(this, m_tempfiletable, m_tempblocktable, blocksize, curvolume);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns a connection from the connection pool.
|
|
/// </summary>
|
|
/// <returns>A connection from the connection pool.</returns>
|
|
public async Task<(SqliteConnection, ReusableTransaction)> GetConnectionFromPool()
|
|
{
|
|
if (!m_connection_pool.TryTake(out var entry))
|
|
{
|
|
var connection = await SQLiteLoader.LoadConnectionAsync();
|
|
connection.ConnectionString = m_connection.ConnectionString + ";Cache=Shared";
|
|
await connection.OpenAsync();
|
|
await SQLiteLoader.ApplyCustomPragmasAsync(connection, m_pagecachesize);
|
|
var transaction = new ReusableTransaction(connection);
|
|
|
|
return (connection, transaction);
|
|
}
|
|
|
|
return entry;
|
|
}
|
|
|
|
private class FileToRestore : IFileToRestore
|
|
{
|
|
public string Path { get; private set; }
|
|
public string Hash { get; private set; }
|
|
public long Length { get; private set; }
|
|
|
|
public FileToRestore(long id, string path, string hash, long length)
|
|
{
|
|
Path = path;
|
|
Hash = hash;
|
|
Length = length;
|
|
}
|
|
}
|
|
|
|
public async IAsyncEnumerable<IFileToRestore> GetFilesToRestore(bool onlyNonVerified)
|
|
{
|
|
using var cmd = m_connection.CreateCommand($@"
|
|
SELECT
|
|
""{m_tempfiletable}"".""ID"",
|
|
""{m_tempfiletable}"".""TargetPath"",
|
|
""Blockset"".""FullHash"",
|
|
""Blockset"".""Length""
|
|
FROM
|
|
""{m_tempfiletable}"",
|
|
""Blockset""
|
|
WHERE
|
|
""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID""
|
|
AND ""{m_tempfiletable}"".""DataVerified"" <= @Verified
|
|
")
|
|
.SetTransaction(m_rtr)
|
|
.SetParameterValue("@Verified", !onlyNonVerified);
|
|
|
|
using var rd = await cmd.ExecuteReaderAsync();
|
|
while (await rd.ReadAsync())
|
|
yield return new FileToRestore(
|
|
rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? "", rd.ConvertValueToString(2) ?? "", rd.ConvertValueToInt64(3));
|
|
|
|
await m_rtr.CommitAsync();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns a list of files and symlinks to restore.
|
|
/// </summary>
|
|
/// <returns>A list of files and symlinks to restore.</returns>
|
|
public async IAsyncEnumerable<FileRequest> GetFilesAndSymlinksToRestore()
|
|
{
|
|
using var cmd = m_connection.CreateCommand($@"
|
|
SELECT
|
|
F.ID,
|
|
F.Path,
|
|
F.TargetPath,
|
|
IFNULL(B.FullHash, ''),
|
|
IFNULL(B.Length, 0),
|
|
F.BlocksetID
|
|
FROM ""{m_tempfiletable}"" F
|
|
LEFT JOIN Blockset B
|
|
ON F.BlocksetID = B.ID
|
|
WHERE F.BlocksetID != {FOLDER_BLOCKSET_ID}
|
|
")
|
|
.SetTransaction(m_rtr);
|
|
using var rd = await cmd.ExecuteReaderAsync();
|
|
while (await rd.ReadAsync())
|
|
yield return new FileRequest(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), rd.ConvertValueToString(2), rd.ConvertValueToString(3), rd.ConvertValueToInt64(4), rd.ConvertValueToInt64(5));
|
|
|
|
await m_rtr.CommitAsync();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns a list of folders to restore. Used to restore folder metadata.
|
|
/// </summary>
|
|
/// <returns>A list of folders to restore.</returns>
|
|
public async IAsyncEnumerable<FileRequest> GetFolderMetadataToRestore()
|
|
{
|
|
using var cmd = m_connection.CreateCommand();
|
|
cmd.SetTransaction(m_rtr);
|
|
using var rd = await cmd.ExecuteReaderAsync($@"
|
|
SELECT
|
|
F.ID,
|
|
'',
|
|
F.TargetPath,
|
|
'',
|
|
0,
|
|
{FOLDER_BLOCKSET_ID}
|
|
FROM ""{m_tempfiletable}"" F
|
|
WHERE
|
|
F.BlocksetID = {FOLDER_BLOCKSET_ID}
|
|
AND F.MetadataID IS NOT NULL
|
|
AND F.MetadataID >= 0
|
|
");
|
|
|
|
while (await rd.ReadAsync())
|
|
yield return new FileRequest(
|
|
rd.ConvertValueToInt64(0),
|
|
rd.ConvertValueToString(1),
|
|
rd.ConvertValueToString(2),
|
|
rd.ConvertValueToString(3),
|
|
rd.ConvertValueToInt64(4),
|
|
rd.ConvertValueToInt64(5)
|
|
);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns a list of blocks and their volume IDs. Used by the <see cref="BlockManager"/> to keep track of blocks and volumes to automatically evict them from the respective caches.
|
|
/// </summary>
|
|
/// <param name="skipMetadata">Flag indicating whether the returned blocks should exclude the metadata blocks.</param>
|
|
/// <returns>A list of tuples containing the block ID and the volume ID of the block.</returns>
|
|
public async IAsyncEnumerable<(long, long)> GetBlocksAndVolumeIDs(bool skipMetadata)
|
|
{
|
|
var metadata_query = skipMetadata ? "" : $@"
|
|
UNION ALL
|
|
SELECT
|
|
""Block"".""ID"",
|
|
""Block"".""VolumeID""
|
|
FROM ""{m_tempfiletable}""
|
|
INNER JOIN ""Metadataset""
|
|
ON ""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID""
|
|
INNER JOIN ""BlocksetEntry""
|
|
ON ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID""
|
|
INNER JOIN ""Block""
|
|
ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID""
|
|
";
|
|
using var cmd = Connection.CreateCommand($@"
|
|
SELECT
|
|
""Block"".""ID"",
|
|
""Block"".""VolumeID""
|
|
FROM ""BlocksetEntry""
|
|
INNER JOIN ""{m_tempfiletable}""
|
|
ON ""BlocksetEntry"".""BlocksetID"" = ""{m_tempfiletable}"".""BlocksetID""
|
|
INNER JOIN ""Block""
|
|
ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID""
|
|
{metadata_query}
|
|
")
|
|
.SetTransaction(m_rtr);
|
|
using var reader = await cmd.ExecuteReaderAsync();
|
|
while (await reader.ReadAsync())
|
|
yield return (
|
|
reader.ConvertValueToInt64(0),
|
|
reader.ConvertValueToInt64(1)
|
|
);
|
|
|
|
await m_rtr.CommitAsync();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns a list of <see cref="BlockRequest"/> for the given blockset ID. It is used by the <see cref="FileProcessor"/> to restore the blocks of a file.
|
|
/// </summary>
|
|
/// <param name="blocksetID">The BlocksetID of the file.</param>
|
|
/// <returns>A list of <see cref="BlockRequest"/> needed to restore the given file.</returns>
|
|
public async IAsyncEnumerable<BlockRequest> GetBlocksFromFile(long blocksetID)
|
|
{
|
|
var (connection, transaction) = await GetConnectionFromPool();
|
|
try
|
|
{
|
|
|
|
using var cmd = connection.CreateCommand(@$"
|
|
SELECT
|
|
""Block"".""ID"",
|
|
""Block"".""Hash"",
|
|
""Block"".""Size"",
|
|
""Block"".""VolumeID""
|
|
FROM ""BlocksetEntry""
|
|
INNER JOIN ""Block""
|
|
ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID""
|
|
WHERE ""BlocksetEntry"".""BlocksetID"" = @BlocksetID
|
|
")
|
|
.SetTransaction(transaction)
|
|
.SetParameterValue("@BlocksetID", blocksetID);
|
|
|
|
using var reader = await cmd.ExecuteReaderAsync();
|
|
for (long i = 0; await reader.ReadAsync(); i++)
|
|
yield return new BlockRequest(
|
|
reader.ConvertValueToInt64(0),
|
|
i,
|
|
reader.ConvertValueToString(1),
|
|
reader.ConvertValueToInt64(2),
|
|
reader.ConvertValueToInt64(3),
|
|
false
|
|
);
|
|
}
|
|
finally
|
|
{
|
|
// Return the connection to the pool
|
|
m_connection_pool.Add((connection, transaction));
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns a list of <see cref="BlockRequest"/> for the metadata blocks of the given file. It is used by the <see cref="FileProcessor"/> to restore the metadata of a file.
|
|
/// </summary>
|
|
/// <param name="fileID">The ID of the file.</param>
|
|
/// <returns>A list of <see cref="BlockRequest"/> needed to restore the metadata of the given file.</returns>
|
|
public async IAsyncEnumerable<BlockRequest> GetMetadataBlocksFromFile(long fileID)
|
|
{
|
|
var (connection, transaction) = await GetConnectionFromPool();
|
|
try
|
|
{
|
|
using var cmd = connection.CreateCommand($@"
|
|
SELECT
|
|
""Block"".""ID"",
|
|
""Block"".""Hash"",
|
|
""Block"".""Size"",
|
|
""Block"".""VolumeID""
|
|
FROM ""File""
|
|
INNER JOIN ""Metadataset""
|
|
ON ""File"".""MetadataID"" = ""Metadataset"".""ID""
|
|
INNER JOIN ""BlocksetEntry""
|
|
ON ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID""
|
|
INNER JOIN ""Block""
|
|
ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID""
|
|
WHERE ""File"".""ID"" = @FileID
|
|
")
|
|
.SetTransaction(transaction)
|
|
.SetParameterValue("@FileID", fileID);
|
|
|
|
using var reader = await cmd.ExecuteReaderAsync();
|
|
for (long i = 0; await reader.ReadAsync(); i++)
|
|
{
|
|
yield return new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false);
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
// Return the connection to the pool
|
|
m_connection_pool.Add((connection, transaction));
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns the volume information for the given volume ID. It is used by the <see cref="VolumeManager"/> to get the volume information for the given volume ID.
|
|
/// </summary>
|
|
/// <param name="VolumeID">The ID of the volume.</param>
|
|
/// <returns>A tuple containing the name, size, and hash of the volume.</returns>
|
|
public async IAsyncEnumerable<(string, long, string)> GetVolumeInfo(long VolumeID)
|
|
{
|
|
using var cmd = m_connection.CreateCommand(@"
|
|
SELECT
|
|
Name,
|
|
Size,
|
|
Hash
|
|
FROM RemoteVolume
|
|
WHERE ID = @VolumeID
|
|
")
|
|
.SetTransaction(m_rtr)
|
|
.SetParameterValue("@VolumeID", VolumeID);
|
|
|
|
using var reader = await cmd.ExecuteReaderAsync();
|
|
while (await reader.ReadAsync())
|
|
yield return (reader.ConvertValueToString(0) ?? "", reader.ConvertValueToInt64(1), reader.ConvertValueToString(2) ?? "");
|
|
|
|
await m_rtr.CommitAsync();
|
|
}
|
|
|
|
public async Task DropRestoreTable()
|
|
{
|
|
using var cmd = m_connection.CreateCommand()
|
|
.SetTransaction(m_rtr);
|
|
|
|
if (m_tempfiletable != null)
|
|
try
|
|
{
|
|
await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempfiletable}""");
|
|
}
|
|
catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); }
|
|
finally { m_tempfiletable = null; }
|
|
|
|
if (m_tempblocktable != null)
|
|
try
|
|
{
|
|
await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempblocktable}""");
|
|
}
|
|
catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); }
|
|
finally { m_tempblocktable = null; }
|
|
|
|
if (m_latestblocktable != null)
|
|
try
|
|
{
|
|
await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_latestblocktable}""");
|
|
}
|
|
catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); }
|
|
finally { m_latestblocktable = null; }
|
|
|
|
if (m_fileprogtable != null)
|
|
try
|
|
{
|
|
await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_fileprogtable}""");
|
|
}
|
|
catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); }
|
|
finally { m_fileprogtable = null; }
|
|
|
|
if (m_totalprogtable != null)
|
|
try
|
|
{
|
|
await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_totalprogtable}""");
|
|
}
|
|
catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); }
|
|
finally { m_totalprogtable = null; }
|
|
|
|
if (m_filesnewlydonetable != null)
|
|
try
|
|
{
|
|
await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}""");
|
|
}
|
|
catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); }
|
|
finally { m_filesnewlydonetable = null; }
|
|
|
|
await m_rtr.CommitAsync();
|
|
}
|
|
|
|
public interface IBlockMarker : IDisposable
|
|
{
|
|
Task SetBlockRestored(long targetfileid, long index, string hash, long blocksize, bool metadata);
|
|
Task SetAllBlocksMissing(long targetfileid);
|
|
Task SetAllBlocksRestored(long targetfileid, bool includeMetadata);
|
|
Task SetFileDataVerified(long targetfileid);
|
|
Task CommitAsync();
|
|
Task UpdateProcessed(IOperationProgressUpdater writer);
|
|
}
|
|
|
|
/// <summary>
|
|
/// A new implementation of IBlockMarker, marking the blocks directly in the blocks table as restored
|
|
/// and reading statistics about progress from DB (kept up-to-date by triggers).
|
|
/// There is no negative influence on performance, esp. since the block table is temporary anyway.
|
|
/// </summary>
|
|
private class DirectBlockMarker : IBlockMarker
|
|
{
|
|
private SqliteCommand m_insertblockCommand = null!;
|
|
private SqliteCommand m_resetfileCommand = null!;
|
|
private SqliteCommand m_updateAsRestoredCommand = null!;
|
|
private SqliteCommand m_updateFileAsDataVerifiedCommand = null!;
|
|
private SqliteCommand m_statUpdateCommand = null!;
|
|
private ReusableTransaction m_rtr = null!;
|
|
private bool m_hasUpdates = false;
|
|
|
|
private string m_blocktablename = null!;
|
|
private string m_filetablename = null!;
|
|
|
|
[Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")]
|
|
public DirectBlockMarker(SqliteConnection connection, string blocktablename, string filetablename, string statstablename)
|
|
{
|
|
throw new NotImplementedException("Use CreateAsync instead of the constructor");
|
|
}
|
|
|
|
private DirectBlockMarker() { }
|
|
|
|
public static async Task<DirectBlockMarker> CreateAsync(LocalDatabase db, string blocktablename, string filetablename, string statstablename)
|
|
{
|
|
var dbm = new DirectBlockMarker()
|
|
{
|
|
m_rtr = db.Transaction,
|
|
m_blocktablename = blocktablename,
|
|
m_filetablename = filetablename,
|
|
|
|
m_insertblockCommand = await db.Connection.CreateCommandAsync($@"
|
|
UPDATE ""{blocktablename}"" SET ""Restored"" = 1
|
|
WHERE
|
|
""FileID"" = @TargetFileId
|
|
AND ""Index"" = @Index
|
|
AND ""Hash"" = @Hash
|
|
AND ""Size"" = @Size
|
|
AND ""Metadata"" = @Metadata
|
|
AND ""Restored"" = 0
|
|
"),
|
|
|
|
m_resetfileCommand = await db.Connection.CreateCommandAsync($@"
|
|
UPDATE ""{blocktablename}""
|
|
SET ""Restored"" = 0
|
|
WHERE ""FileID"" = @TargetFileId
|
|
"),
|
|
|
|
m_updateAsRestoredCommand = await db.Connection.CreateCommandAsync($@"
|
|
UPDATE ""{blocktablename}""
|
|
SET ""Restored"" = 1
|
|
WHERE
|
|
""FileID"" = @TargetFileId
|
|
AND ""Metadata"" <= @Metadata
|
|
"),
|
|
|
|
m_updateFileAsDataVerifiedCommand = await db.Connection.CreateCommandAsync($@"
|
|
UPDATE ""{filetablename}""
|
|
SET ""DataVerified"" = 1
|
|
WHERE ""ID"" = @TargetFileId
|
|
"),
|
|
|
|
m_statUpdateCommand = statstablename == null ?
|
|
// very slow fallback if stats tables were not created
|
|
await db.Connection.CreateCommandAsync($@"
|
|
SELECT
|
|
COUNT(DISTINCT ""FileID""),
|
|
SUM(""Size"")
|
|
FROM ""{blocktablename}""
|
|
WHERE ""Restored"" = 1
|
|
")
|
|
:
|
|
// Fields in Stats: TotalFiles, TotalBlocks, TotalSize
|
|
// FilesFullyRestored, FilesPartiallyRestored, BlocksRestored, SizeRestored
|
|
await db.Connection.CreateCommandAsync($@"
|
|
SELECT
|
|
SUM(""FilesFullyRestored""),
|
|
SUM(""SizeRestored"")
|
|
FROM ""{statstablename}""
|
|
")
|
|
};
|
|
|
|
dbm.m_insertblockCommand.SetTransaction(db.Transaction);
|
|
dbm.m_resetfileCommand.SetTransaction(db.Transaction);
|
|
dbm.m_updateAsRestoredCommand.SetTransaction(db.Transaction);
|
|
dbm.m_updateFileAsDataVerifiedCommand.SetTransaction(db.Transaction);
|
|
dbm.m_statUpdateCommand.SetTransaction(db.Transaction);
|
|
|
|
return dbm;
|
|
}
|
|
|
|
public async Task UpdateProcessed(IOperationProgressUpdater updater)
|
|
{
|
|
if (!m_hasUpdates)
|
|
return;
|
|
|
|
m_hasUpdates = false;
|
|
using var rd = await m_statUpdateCommand.ExecuteReaderAsync();
|
|
var filesprocessed = 0L;
|
|
var processedsize = 0L;
|
|
|
|
if (rd.Read())
|
|
{
|
|
filesprocessed += rd.ConvertValueToInt64(0, 0);
|
|
processedsize += rd.ConvertValueToInt64(1, 0);
|
|
}
|
|
|
|
updater.UpdatefilesProcessed(filesprocessed, processedsize);
|
|
}
|
|
|
|
public async Task SetAllBlocksMissing(long targetfileid)
|
|
{
|
|
m_hasUpdates = true;
|
|
m_resetfileCommand.SetParameterValue("@TargetFileId", targetfileid);
|
|
|
|
var r = await m_resetfileCommand.ExecuteNonQueryAsync();
|
|
if (r <= 0)
|
|
throw new Exception("Unexpected reset result");
|
|
}
|
|
|
|
public async Task SetAllBlocksRestored(long targetfileid, bool includeMetadata)
|
|
{
|
|
m_hasUpdates = true;
|
|
m_updateAsRestoredCommand.SetParameterValue("@TargetFileId", targetfileid)
|
|
.SetParameterValue("@Metadata", includeMetadata ? 1 : 0);
|
|
|
|
var r = await m_updateAsRestoredCommand.ExecuteNonQueryAsync();
|
|
if (r <= 0)
|
|
throw new Exception("Unexpected reset result");
|
|
}
|
|
|
|
public async Task SetFileDataVerified(long targetfileid)
|
|
{
|
|
m_hasUpdates = true;
|
|
m_updateFileAsDataVerifiedCommand.SetParameterValue("@TargetFileId", targetfileid);
|
|
|
|
var r = await m_updateFileAsDataVerifiedCommand.ExecuteNonQueryAsync();
|
|
if (r != 1)
|
|
throw new Exception("Unexpected result when marking file as verified.");
|
|
}
|
|
|
|
public async Task SetBlockRestored(long targetfileid, long index, string hash, long size, bool metadata)
|
|
{
|
|
m_hasUpdates = true;
|
|
m_insertblockCommand.SetParameterValue("@TargetFileId", targetfileid)
|
|
.SetParameterValue("@Index", index)
|
|
.SetParameterValue("@Hash", hash)
|
|
.SetParameterValue("@Size", size)
|
|
.SetParameterValue("@Metadata", metadata);
|
|
|
|
var r = await m_insertblockCommand.ExecuteNonQueryAsync();
|
|
if (r != 1)
|
|
throw new Exception("Unexpected result when marking block.");
|
|
}
|
|
|
|
public async Task CommitAsync()
|
|
{
|
|
using (new Logging.Timer(LOGTAG, "CommitBlockMarker", "CommitBlockMarker"))
|
|
await m_rtr.CommitAsync();
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
m_insertblockCommand?.Dispose();
|
|
m_resetfileCommand?.Dispose();
|
|
m_updateAsRestoredCommand?.Dispose();
|
|
m_updateFileAsDataVerifiedCommand?.Dispose();
|
|
m_statUpdateCommand?.Dispose();
|
|
}
|
|
}
|
|
|
|
[Obsolete("Calling this constructor will throw an exception. Use CreateBlockMarkerAsync instead.")]
|
|
public IBlockMarker CreateBlockMarker()
|
|
{
|
|
throw new NotImplementedException("Use CreateBlockMarkerAsync instead of the constructor");
|
|
}
|
|
|
|
public async Task<IBlockMarker> CreateBlockMarkerAsync()
|
|
{
|
|
if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable))
|
|
throw new InvalidOperationException("No temporary file table set up for this restore.");
|
|
if (string.IsNullOrWhiteSpace(m_totalprogtable))
|
|
throw new InvalidOperationException("No progress table set up for this restore.");
|
|
return await DirectBlockMarker.CreateAsync(this, m_tempblocktable, m_tempfiletable, m_totalprogtable);
|
|
}
|
|
|
|
public override void Dispose()
|
|
{
|
|
DisposeAsync().Await();
|
|
}
|
|
|
|
public override async Task DisposeAsync()
|
|
{
|
|
await DisposePoolAsync();
|
|
await DropRestoreTable();
|
|
await base.DisposeAsync();
|
|
}
|
|
|
|
public async Task DisposePoolAsync()
|
|
{
|
|
foreach (var (connection, transaction) in m_connection_pool)
|
|
{
|
|
await transaction.DisposeAsync();
|
|
await connection.CloseAsync();
|
|
await connection.DisposeAsync();
|
|
}
|
|
m_connection_pool.Clear();
|
|
}
|
|
|
|
public async IAsyncEnumerable<string> GetTargetFolders()
|
|
{
|
|
using var cmd = m_connection.CreateCommand($@"
|
|
SELECT ""TargetPath""
|
|
FROM ""{m_tempfiletable}""
|
|
WHERE ""BlocksetID"" == @BlocksetID
|
|
")
|
|
.SetTransaction(m_rtr)
|
|
.SetParameterValue("@BlocksetID", FOLDER_BLOCKSET_ID);
|
|
|
|
using var rd = await cmd.ExecuteReaderAsync();
|
|
while (await rd.ReadAsync())
|
|
yield return rd.ConvertValueToString(0) ?? "";
|
|
|
|
await m_rtr.CommitAsync();
|
|
}
|
|
|
|
public interface IFastSource
|
|
{
|
|
string TargetPath { get; }
|
|
long TargetFileID { get; }
|
|
string SourcePath { get; }
|
|
IAsyncEnumerable<IBlockEntry> Blocks();
|
|
}
|
|
|
|
public interface IBlockEntry
|
|
{
|
|
long Offset { get; }
|
|
long Size { get; }
|
|
long Index { get; }
|
|
string Hash { get; }
|
|
}
|
|
|
|
private class FastSource : IFastSource
|
|
{
|
|
private class BlockEntry : IBlockEntry
|
|
{
|
|
private readonly SqliteDataReader m_rd;
|
|
private readonly long m_blocksize;
|
|
public BlockEntry(SqliteDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; }
|
|
public long Offset { get { return m_rd.ConvertValueToInt64(3) * m_blocksize; } }
|
|
public long Index { get { return m_rd.ConvertValueToInt64(3); } }
|
|
public long Size { get { return m_rd.ConvertValueToInt64(5); } }
|
|
public string Hash { get { return m_rd.ConvertValueToString(4) ?? ""; } }
|
|
}
|
|
|
|
private readonly SqliteDataReader m_rd;
|
|
private readonly long m_blocksize;
|
|
public FastSource(SqliteDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; MoreData = true; }
|
|
public bool MoreData { get; private set; }
|
|
public string TargetPath { get { return m_rd.ConvertValueToString(0) ?? ""; } }
|
|
public long TargetFileID { get { return m_rd.ConvertValueToInt64(2); } }
|
|
public string SourcePath { get { return m_rd.ConvertValueToString(1) ?? ""; } }
|
|
|
|
public async IAsyncEnumerable<IBlockEntry> Blocks()
|
|
{
|
|
var tid = TargetFileID;
|
|
|
|
do
|
|
{
|
|
yield return new BlockEntry(m_rd, m_blocksize);
|
|
} while ((MoreData = await m_rd.ReadAsync()) && tid == TargetFileID);
|
|
}
|
|
}
|
|
|
|
public async IAsyncEnumerable<IFastSource> GetFilesAndSourceBlocksFast(long blocksize)
|
|
{
|
|
using (var cmdReader = m_connection.CreateCommand())
|
|
using (var cmd = m_connection.CreateCommand())
|
|
{
|
|
cmdReader.SetTransaction(m_rtr);
|
|
cmd.SetTransaction(m_rtr);
|
|
cmd.SetCommandAndParameters($@"
|
|
UPDATE ""{m_tempfiletable}""
|
|
SET ""LocalSourceExists"" = 1
|
|
WHERE Path = @Path
|
|
");
|
|
cmdReader.SetCommandAndParameters($@"
|
|
SELECT DISTINCT ""{m_tempfiletable}"".""Path""
|
|
FROM ""{m_tempfiletable}""
|
|
");
|
|
using (var rd = await cmdReader.ExecuteReaderAsync())
|
|
{
|
|
while (await rd.ReadAsync())
|
|
{
|
|
var sourcepath = rd.ConvertValueToString(0);
|
|
if (SystemIO.IO_OS.FileExists(sourcepath))
|
|
{
|
|
await cmd.SetParameterValue("@Path", sourcepath)
|
|
.ExecuteNonQueryAsync();
|
|
}
|
|
else
|
|
{
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "LocalSourceMissing", "Local source file not found: {0}", sourcepath);
|
|
}
|
|
}
|
|
}
|
|
|
|
//This localSourceExists index will make the query engine to start by searching FileSet table. As the result is ordered by FileSet.ID, we will get the cursor "instantly"
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
CREATE INDEX ""{m_tempfiletable}_LocalSourceExists""
|
|
ON ""{m_tempfiletable}"" (""LocalSourceExists"")
|
|
");
|
|
|
|
await m_rtr.CommitAsync();
|
|
}
|
|
|
|
m_latestblocktable = "LatestBlocksetIds-" + m_temptabsetguid;
|
|
|
|
var whereclause = $@"
|
|
""{m_tempfiletable}"".""LocalSourceExists"" = 1
|
|
AND ""{m_tempblocktable}"".""Restored"" = 0
|
|
AND ""{m_tempblocktable}"".""Metadata"" = 0
|
|
AND ""{m_tempfiletable}"".""TargetPath"" != ""{m_tempfiletable}"".""Path""
|
|
";
|
|
|
|
var latestBlocksetIds = $@"
|
|
SELECT
|
|
""File"".""Path"" AS ""PATH"",
|
|
""File"".""BlocksetID"" AS ""BlocksetID"",
|
|
MAX(""Fileset"".""Timestamp"") AS ""Timestamp""
|
|
FROM
|
|
""File"",
|
|
""FilesetEntry"",
|
|
""Fileset""
|
|
WHERE
|
|
""File"".""ID"" = ""FilesetEntry"".""FileID""
|
|
AND ""FilesetEntry"".""FilesetID"" = ""Fileset"".""ID""
|
|
AND ""File"".""Path"" IN (
|
|
SELECT DISTINCT ""{m_tempfiletable}"".""Path""
|
|
FROM
|
|
""{m_tempfiletable}"",
|
|
""{m_tempblocktable}""
|
|
WHERE
|
|
""{m_tempfiletable}"".""ID"" = ""{m_tempblocktable}"".""FileID""
|
|
AND {whereclause}
|
|
)
|
|
GROUP BY ""File"".""Path""
|
|
";
|
|
|
|
using (var cmd = m_connection.CreateCommand())
|
|
{
|
|
cmd.SetTransaction(m_rtr);
|
|
await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_latestblocktable}"" ");
|
|
await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{m_latestblocktable}"" AS {latestBlocksetIds}");
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
CREATE INDEX ""{m_latestblocktable}_path""
|
|
ON ""{m_latestblocktable}"" (""Path"")
|
|
");
|
|
|
|
await cmd.ExecuteNonQueryAsync($@"
|
|
UPDATE ""{m_tempfiletable}""
|
|
SET LatestBlocksetId = (
|
|
SELECT BlocksetId
|
|
FROM ""{m_latestblocktable}""
|
|
WHERE Path = ""{m_tempfiletable}"".Path
|
|
)
|
|
");
|
|
}
|
|
|
|
var sources = $@"
|
|
SELECT DISTINCT
|
|
""{m_tempfiletable}"".""TargetPath"",
|
|
""{m_tempfiletable}"".""Path"",
|
|
""{m_tempfiletable}"".""ID"",
|
|
""{m_tempblocktable}"".""Index"",
|
|
""{m_tempblocktable}"".""Hash"",
|
|
""{m_tempblocktable}"".""Size""
|
|
FROM
|
|
""{m_tempfiletable}"",
|
|
""{m_tempblocktable}"",
|
|
""BlocksetEntry""
|
|
WHERE
|
|
""{m_tempfiletable}"".""ID"" = ""{m_tempblocktable}"".""FileID""
|
|
AND ""BlocksetEntry"".""BlocksetID"" = ""{m_tempfiletable}"".""LatestBlocksetID""
|
|
AND ""BlocksetEntry"".""BlockID"" = ""{m_tempblocktable}"".""BlockID""
|
|
AND ""BlocksetEntry"".""Index"" = ""{m_tempblocktable}"".""Index""
|
|
AND {whereclause}
|
|
ORDER BY
|
|
""{m_tempfiletable}"".""ID"",
|
|
""{m_tempblocktable}"".""Index""
|
|
";
|
|
|
|
using (var cmd = m_connection.CreateCommand())
|
|
{
|
|
cmd.SetTransaction(m_rtr);
|
|
using var rd = await cmd.ExecuteReaderAsync(sources);
|
|
if (await rd.ReadAsync())
|
|
{
|
|
bool more;
|
|
do
|
|
{
|
|
var n = new FastSource(rd, blocksize);
|
|
var tid = n.TargetFileID;
|
|
yield return n;
|
|
|
|
more = n.MoreData;
|
|
while (more && n.TargetFileID == tid)
|
|
more = await rd.ReadAsync();
|
|
|
|
} while (more);
|
|
}
|
|
}
|
|
|
|
await m_rtr.CommitAsync();
|
|
}
|
|
|
|
}
|
|
}
|