haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Moved all database operations to the LocalRestoreDatabase.

Browse Source 
Also changed the old changes, so that the functions related to the legacy restore remains untouched.
This commit is contained in:
Carl Johnsen
2024-12-03 08:35:14 +01:00
parent 6413014c7d
commit 6ba956dfcd
6 changed files with 163 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Main/Database/LocalRestoreDatabase.cs
+103 -21
View File
@@ -20,10 +20,11 @@
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using System.Text;
using Duplicati.Library.Common;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Main.Operation.Restore;
using Duplicati.Library.Main.Volumes;
using Duplicati.Library.Utility;
@@ -36,11 +37,11 @@ namespace Duplicati.Library.Main.Database
/// </summary>
private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalRestoreDatabase));
public readonly string m_temptabsetguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
protected readonly string m_temptabsetguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
/// <summary>
/// The name of the temporary table in the database, which is used to store the list of files to restore.
/// </summary>
public string m_tempfiletable { get; private set; }
protected string m_tempfiletable;
protected string m_tempblocktable;
protected string m_fileprogtable;
protected string m_totalprogtable;
@@ -544,12 +545,9 @@ namespace Duplicati.Library.Main.Database
/// </summary>
public interface IFileToRestore
{
public long ID { get; }
public string Name { get; }
public string Path { get; }
public string Hash { get; }
public long Length { get; }
public long BlocksetID { get; }
string Path { get; }
string Hash { get; }
long Length { get; }
}
public interface IPatchBlock
@@ -943,21 +941,15 @@ namespace Duplicati.Library.Main.Database
private class FileToRestore : IFileToRestore
{
public long ID { get; private set; }
public string Name { get; private set; }
public string Path { get; private set; }
public string Hash { get; private set; }
public long Length { get; private set; }
public long BlocksetID { get; private set; }
public FileToRestore(long id, string name, string path, string hash, long length, long blocksetid)
public FileToRestore(long id, string path, string hash, long length)
{
this.ID = id;
this.Name = name;
this.Path = path;
this.Hash = hash;
this.Length = length;
this.BlocksetID = blocksetid;
}
}
@@ -966,9 +958,9 @@ namespace Duplicati.Library.Main.Database
using (var cmd = m_connection.CreateCommand())
{
cmd.AddParameter(!onlyNonVerified);
using (var rd = cmd.ExecuteReader(string.Format(@"SELECT ""{0}"".ID, ""{0}"".""Path"", ""{0}"".""TargetPath"", ""Blockset"".""FullHash"", ""Blockset"".""Length"", ""Blockset"".""ID"" FROM ""{0}"",""Blockset"" WHERE ""{0}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""{0}"".""DataVerified"" <= ?", m_tempfiletable)))
using (var rd = cmd.ExecuteReader(string.Format(@"SELECT ""{0}"".""ID"", ""{0}"".""TargetPath"", ""Blockset"".""FullHash"", ""Blockset"".""Length"" FROM ""{0}"",""Blockset"" WHERE ""{0}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""{0}"".""DataVerified"" <= ?", m_tempfiletable)))
while (rd.Read())
yield return new FileToRestore(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), rd.ConvertValueToString(2), rd.ConvertValueToString(3), rd.ConvertValueToInt64(4), rd.ConvertValueToInt64(5));
yield return new FileToRestore(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), rd.ConvertValueToString(2), rd.ConvertValueToInt64(3));
}
}
@@ -977,7 +969,7 @@ namespace Duplicati.Library.Main.Database
/// </summary>
/// <param name="onlyNonVerified">Flag to indicate if only files with non-verified data should be returned.</param>
/// <returns>A list of files and symlinks to restore.</returns>
public IEnumerable<IFileToRestore> GetFilesAndSymlinksToRestore(bool onlyNonVerified)
public IEnumerable<FileRequest> GetFilesAndSymlinksToRestore(bool onlyNonVerified)
{
using var cmd = m_connection.CreateCommand();
cmd.AddParameter(!onlyNonVerified);
@@ -985,9 +977,99 @@ namespace Duplicati.Library.Main.Database
SELECT F.ID, F.Path, F.TargetPath, IFNULL(B.FullHash, ''), IFNULL(B.Length, 0), F.BlocksetID
FROM ""{m_tempfiletable}"" F
LEFT JOIN Blockset B ON F.BlocksetID = B.ID
WHERE F.BlocksetID != {LocalDatabase.FOLDER_BLOCKSET_ID}");
WHERE F.BlocksetID != {FOLDER_BLOCKSET_ID}");
while (rd.Read())
yield return new FileToRestore(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), rd.ConvertValueToString(2), rd.ConvertValueToString(3), rd.ConvertValueToInt64(4), rd.ConvertValueToInt64(5));
yield return new FileRequest(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), rd.ConvertValueToString(2), rd.ConvertValueToString(3), rd.ConvertValueToInt64(4), rd.ConvertValueToInt64(5));
}
/// <summary>
/// Returns a list of blocks and their volume IDs. Used by the <see cref="BlockManager"/> to keep track of blocks and volumes to automatically evict them from the respective caches.
/// </summary>
/// <param name="skipMetadata">Flag indicating whether the returned blocks should exclude the metadata blocks.</param>
/// <returns>A list of tuples containing the block ID and the volume ID of the block.</returns>
public IEnumerable<(long, long)> GetBlocksAndVolumeIDs(bool skipMetadata)
{
using var cmd = Connection.CreateCommand();
using var reader = cmd.ExecuteReader($@"
SELECT Block.ID, Block.VolumeID
FROM BlocksetEntry
INNER JOIN ""{m_tempfiletable}"" ON BlocksetEntry.BlocksetID = ""{m_tempfiletable}"".BlocksetID
INNER JOIN Block ON BlocksetEntry.BlockID = Block.ID
"
+ (skipMetadata ? "" : $@"
UNION ALL
SELECT Block.ID, Block.VolumeID
FROM ""{m_tempfiletable}""
INNER JOIN Metadataset ON ""{m_tempfiletable}"".MetadataID = Metadataset.ID
INNER JOIN BlocksetEntry ON Metadataset.BlocksetID = BlocksetEntry.BlocksetID
INNER JOIN Block ON BlocksetEntry.BlockID = Block.ID
WHERE ""{m_tempfiletable}"".BlocksetID IS NOT {FOLDER_BLOCKSET_ID}
"));
while (reader.Read())
yield return (reader.ConvertValueToInt64(0), reader.ConvertValueToInt64(1));
}
/// <summary>
/// Returns a list of <see cref="BlockRequest"/> for the given blockset ID. It is used by the <see cref="FileProcessor"/> to restore the blocks of a file.
/// </summary>
/// <param name="blocksetID">The BlocksetID of the file.</param>
/// <returns>A list of <see cref="BlockRequest"/> needed to restore the given file.</returns>
public IEnumerable<BlockRequest> GetBlocksFromFile(long blocksetID)
{
using var cmd = m_connection.CreateCommand();
cmd.CommandText = @$"
SELECT Block.ID, Block.Hash, Block.Size, Block.VolumeID
FROM BlocksetEntry INNER JOIN Block
ON BlocksetEntry.BlockID = Block.ID
WHERE BlocksetEntry.BlocksetID = ?";
cmd.AddParameter();
cmd.SetParameterValue(0, blocksetID);
using var reader = cmd.ExecuteReader();
for (long i = 0; reader.Read(); i++)
{
yield return new BlockRequest(reader.GetInt64(0), i, reader.GetString(1), reader.GetInt64(2), reader.GetInt64(3), false);
}
}
/// <summary>
/// Returns a list of <see cref="BlockRequest"/> for the metadata blocks of the given file. It is used by the <see cref="FileProcessor"/> to restore the metadata of a file.
/// </summary>
/// <param name="fileID">The ID of the file.</param>
/// <returns>A list of <see cref="BlockRequest"/> needed to restore the metadata of the given file.</returns>
public IEnumerable<BlockRequest> GetMetadataBlocksFromFile(long fileID)
{
using var cmd = m_connection.CreateCommand();
cmd.CommandText = $@"
SELECT Block.ID, Block.Hash, Block.Size, Block.VolumeID
FROM ""{m_tempfiletable}""
INNER JOIN Metadataset ON ""{m_tempfiletable}"".MetadataID = Metadataset.ID
INNER JOIN BlocksetEntry ON Metadataset.BlocksetID = BlocksetEntry.BlocksetID
INNER JOIN Block ON BlocksetEntry.BlockID = Block.ID
WHERE ""{m_tempfiletable}"".ID = ?
";
cmd.AddParameter();
cmd.SetParameterValue(0, fileID);
using var reader = cmd.ExecuteReader();
for (long i = 0; reader.Read(); i++)
{
yield return new BlockRequest(reader.GetInt64(0), i, reader.GetString(1), reader.GetInt64(2), reader.GetInt64(3), false);
}
}
/// <summary>
/// Returns the volume information for the given volume ID. It is used by the <see cref="VolumeManager"/> to get the volume information for the given volume ID.
/// </summary>
/// <param name="VolumeID">The ID of the volume.</param>
/// <returns>A tuple containing the name, size, and hash of the volume.</returns>
public IEnumerable<(string,long,string)> GetVolumeInfo(long VolumeID)
{
using var cmd = m_connection.CreateCommand();
cmd.CommandText = "SELECT Name, Size, Hash FROM RemoteVolume WHERE ID = ?";
cmd.AddParameter();
cmd.SetParameterValue(0, VolumeID);
using var reader = cmd.ExecuteReader();
while (reader.Read())
yield return (reader.GetString(0), reader.GetInt64(1), reader.GetString(2));
}
public void DropRestoreTable()
Duplicati/Library/Main/Operation/Restore/BlockManager.cs
+5 -23
View File
@@ -114,30 +114,12 @@ namespace Duplicati.Library.Main.Operation.Restore
sw_checkcounts = options.InternalProfiling ? new () : null;
sw_get_wait = options.InternalProfiling ? new () : null;
var cmd = db.Connection.CreateCommand();
var blockcounts = cmd.ExecuteReaderEnumerable($@"
SELECT Block.ID, Block.VolumeID
FROM BlocksetEntry
INNER JOIN ""{db.m_tempfiletable}"" ON BlocksetEntry.BlocksetID = ""{db.m_tempfiletable}"".BlocksetID
INNER JOIN Block ON BlocksetEntry.BlockID = Block.ID
"
+ (options.SkipMetadata ? "" : $@"
UNION ALL
SELECT Block.ID, Block.VolumeID
FROM ""{db.m_tempfiletable}""
INNER JOIN Metadataset ON ""{db.m_tempfiletable}"".MetadataID = Metadataset.ID
INNER JOIN BlocksetEntry ON Metadataset.BlocksetID = BlocksetEntry.BlocksetID
INNER JOIN Block ON BlocksetEntry.BlockID = Block.ID
WHERE ""{db.m_tempfiletable}"".BlocksetID IS NOT {LocalDatabase.FOLDER_BLOCKSET_ID}
"));
foreach (var row in blockcounts)
foreach (var (block_id, volume_id) in db.GetBlocksAndVolumeIDs(options.SkipMetadata))
{
var bid = row.GetInt64(0);
var bc = m_blockcount.TryGetValue(bid, out var c);
m_blockcount[bid] = bc ? c + 1 : 1;
var vid = row.GetInt64(1);
var vc = m_volumecount.TryGetValue(vid, out var v);
m_volumecount[vid] = vc ? v + 1 : 1;
var bc = m_blockcount.TryGetValue(block_id, out var c);
m_blockcount[block_id] = bc ? c + 1 : 1;
var vc = m_volumecount.TryGetValue(volume_id, out var v);
m_volumecount[volume_id] = vc ? v + 1 : 1;
}
}
Duplicati/Library/Main/Operation/Restore/Channels.cs
+1 -1
View File
@@ -41,7 +41,7 @@ namespace Duplicati.Library.Main.Operation.Restore
/// <summary>
/// Channel between <see cref="FileLister"/> and <see cref="FileProcessor"/>.
/// </summary>
public static readonly ChannelMarkerWrapper<Database.LocalRestoreDatabase.IFileToRestore> FilesToRestore = new(new ChannelNameAttribute("FilesToRestore", BufferSize));
public static readonly ChannelMarkerWrapper<FileRequest> FilesToRestore = new(new ChannelNameAttribute("FilesToRestore", BufferSize));
/// <summary>
/// Channel between <see cref="BlockManager"/> and <see cref="VolumeManager"/>.
Duplicati/Library/Main/Operation/Restore/FileProcessor.cs
+34 -62
View File
@@ -81,24 +81,6 @@ namespace Duplicati.Library.Main.Operation.Restore
{
using var filehasher = HashFactory.CreateHasher(options.FileHashAlgorithm);
using var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm);
using var findBlocksCmd = db.Connection.CreateCommand();
findBlocksCmd.CommandText = @$"
SELECT Block.ID, Block.Hash, Block.Size, Block.VolumeID
FROM BlocksetEntry INNER JOIN Block
ON BlocksetEntry.BlockID = Block.ID
WHERE BlocksetEntry.BlocksetID = ?";
findBlocksCmd.AddParameter();
var findMetaBlocksCmd = db.Connection.CreateCommand();
findMetaBlocksCmd.CommandText = $@"
SELECT Block.ID, Block.Hash, Block.Size, Block.VolumeID
FROM ""{db.m_tempfiletable}""
INNER JOIN Metadataset ON ""{db.m_tempfiletable}"".MetadataID = Metadataset.ID
INNER JOIN BlocksetEntry ON Metadataset.BlocksetID = BlocksetEntry.BlocksetID
INNER JOIN Block ON BlocksetEntry.BlockID = Block.ID
WHERE ""{db.m_tempfiletable}"".ID = ?
";
findMetaBlocksCmd.AddParameter();
while (true)
{
@@ -109,12 +91,7 @@ namespace Duplicati.Library.Main.Operation.Restore
// Get information about the blocks for the file
sw_block?.Start();
findBlocksCmd.SetParameterValue(0, file.BlocksetID);
var blocks = findBlocksCmd.ExecuteReaderEnumerable()
.Select((b, i) =>
new BlockRequest(b.GetInt64(0), i, b.GetString(1), b.GetInt64(2), b.GetInt64(3), false)
)
.ToArray();
var blocks = db.GetBlocksFromFile(file.BlocksetID).ToArray();
sw_block?.Stop();
sw_work?.Start();
@@ -135,18 +112,18 @@ namespace Duplicati.Library.Main.Operation.Restore
{
if (options.Dryrun)
{
Logging.Log.WriteDryrunMessage(LOGTAG, "DryrunRestore", @$"Would have created empty file ""{file.Path}""");
Logging.Log.WriteDryrunMessage(LOGTAG, "DryrunRestore", @$"Would have created empty file ""{file.TargetPath}""");
}
else
{
var foldername = SystemIO.IO_OS.PathGetDirectoryName(file.Path);
var foldername = SystemIO.IO_OS.PathGetDirectoryName(file.TargetPath);
if (!Directory.Exists(foldername))
{
Directory.CreateDirectory(foldername);
}
// Create an empty file, or truncate to 0
using var fs = new FileStream(file.Path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.None);
using var fs = new FileStream(file.TargetPath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.None);
fs.SetLength(0);
if (missing_blocks.Count != 0)
{
@@ -159,7 +136,7 @@ namespace Duplicati.Library.Main.Operation.Restore
{
if (missing_blocks.Any(x => x.VolumeID < 0))
{
Logging.Log.WriteWarningMessage(LOGTAG, "NegativeVolumeID", null, $"{file.Path} has a negative volume ID, skipping");
Logging.Log.WriteWarningMessage(LOGTAG, "NegativeVolumeID", null, $"{file.TargetPath} has a negative volume ID, skipping");
continue;
}
@@ -172,14 +149,14 @@ namespace Duplicati.Library.Main.Operation.Restore
if (options.Dryrun)
{
// If dryrun, open the file for read only to verify the file hash.
if (File.Exists(file.Path))
if (File.Exists(file.TargetPath))
{
fs = new FileStream(file.Path, FileMode.Open, FileAccess.Read);
fs = new FileStream(file.TargetPath, FileMode.Open, FileAccess.Read);
}
}
else
{
fs = new FileStream(file.Path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
fs = new FileStream(file.TargetPath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
}
sw_work?.Stop();
@@ -239,14 +216,14 @@ namespace Duplicati.Library.Main.Operation.Restore
if (options.Dryrun)
{
Logging.Log.WriteDryrunMessage(LOGTAG, "DryrunRestore", @$"Would have restored {bytes_written} bytes of ""{file.Path}""");
Logging.Log.WriteDryrunMessage(LOGTAG, "DryrunRestore", @$"Would have restored {bytes_written} bytes of ""{file.TargetPath}""");
}
// Verify the file hash
filehasher.TransformFinalBlock([], 0, 0);
if (Convert.ToBase64String(filehasher.Hash) != file.Hash)
{
Logging.Log.WriteErrorMessage(LOGTAG, "FileHashMismatch", null, $"File hash mismatch for {file.Path} - expected: {file.Hash}, actual: {Convert.ToBase64String(filehasher.Hash)}");
Logging.Log.WriteErrorMessage(LOGTAG, "FileHashMismatch", null, $"File hash mismatch for {file.TargetPath} - expected: {file.Hash}, actual: {Convert.ToBase64String(filehasher.Hash)}");
throw new Exception("File hash mismatch");
}
@@ -255,7 +232,7 @@ namespace Duplicati.Library.Main.Operation.Restore
{
if (options.Dryrun)
{
Logging.Log.WriteDryrunMessage(LOGTAG, "DryrunRestore", @$"Would have truncated ""{file.Path}"" from {fs.Length} to {file.Length}");
Logging.Log.WriteDryrunMessage(LOGTAG, "DryrunRestore", @$"Would have truncated ""{file.TargetPath}"" from {fs.Length} to {file.Length}");
}
else
{
@@ -267,7 +244,7 @@ namespace Duplicati.Library.Main.Operation.Restore
{
lock (results)
{
results.BrokenLocalFiles.Add(file.Path);
results.BrokenLocalFiles.Add(file.TargetPath);
}
throw;
}
@@ -278,7 +255,7 @@ namespace Duplicati.Library.Main.Operation.Restore
}
if (!options.SkipMetadata) {
await RestoreMetadata(findMetaBlocksCmd, file, block_request, block_response, options, sw_meta, sw_work, sw_req, sw_resp);
await RestoreMetadata(db, file, block_request, block_response, options, sw_meta, sw_work, sw_req, sw_resp);
}
// Keep track of the restored files and their sizes
@@ -324,16 +301,11 @@ namespace Duplicati.Library.Main.Operation.Restore
/// <param name="sw_work">The stopwatch for internal profiling of the general processing.</param>
/// <param name="sw_req">The stopwatch for internal profiling of the block requests.</param>
/// <param name="sw_resp">The stopwatch for internal profiling of the block responses.</param>
private static async Task RestoreMetadata(IDbCommand cmd, LocalRestoreDatabase.IFileToRestore file, IChannel<BlockRequest> block_request, IChannel<byte[]> block_response, Options options, Stopwatch sw_meta, Stopwatch sw_work, Stopwatch sw_req, Stopwatch sw_resp)
private static async Task RestoreMetadata(LocalRestoreDatabase db, FileRequest file, IChannel<BlockRequest> block_request, IChannel<byte[]> block_response, Options options, Stopwatch sw_meta, Stopwatch sw_work, Stopwatch sw_req, Stopwatch sw_resp)
{
sw_work?.Stop();
sw_meta?.Start();
cmd.SetParameterValue(0, file.ID);
var blocks = cmd.ExecuteReaderEnumerable()
.Select((b, i) =>
new BlockRequest(b.GetInt64(0), i, b.GetString(1), b.GetInt64(2), b.GetInt64(3), false)
)
.ToArray();
var blocks = db.GetMetadataBlocksFromFile(file.ID).ToArray();
sw_meta?.Stop();
sw_work?.Start();
@@ -353,7 +325,7 @@ namespace Duplicati.Library.Main.Operation.Restore
}
ms.Seek(0, SeekOrigin.Begin);
RestoreHandler.ApplyMetadata(file.Path, ms, options.RestorePermissions, options.RestoreSymlinkMetadata, options.Dryrun);
RestoreHandler.ApplyMetadata(file.TargetPath, ms, options.RestorePermissions, options.RestoreSymlinkMetadata, options.Dryrun);
}
/// <summary>
@@ -367,17 +339,17 @@ namespace Duplicati.Library.Main.Operation.Restore
/// <param name="results">The restoration results.</param>
/// <param name="block_request">The channel to request blocks from the block manager. Used to inform the block manager which blocks are already present.</param>
/// <returns>An awaitable `Task`, which returns a collection of data blocks that are missing.</returns>
private static async Task<List<BlockRequest>> VerifyTargetBlocks(LocalRestoreDatabase.IFileToRestore file, BlockRequest[] blocks, System.Security.Cryptography.HashAlgorithm filehasher, System.Security.Cryptography.HashAlgorithm blockhasher, Options options, RestoreResults results, IChannel<BlockRequest> block_request)
private static async Task<List<BlockRequest>> VerifyTargetBlocks(FileRequest file, BlockRequest[] blocks, System.Security.Cryptography.HashAlgorithm filehasher, System.Security.Cryptography.HashAlgorithm blockhasher, Options options, RestoreResults results, IChannel<BlockRequest> block_request)
{
List<BlockRequest> missing_blocks = [];
// Check if the file exists
if (File.Exists(file.Path))
if (File.Exists(file.TargetPath))
{
filehasher.Initialize();
try
{
using var f = new FileStream(file.Path, FileMode.Open, FileAccess.Read);
using var f = new FileStream(file.TargetPath, FileMode.Open, FileAccess.Read);
var buffer = new byte[options.Blocksize];
long bytes_read = 0;
for (int i = 0; i < blocks.Length; i++)
@@ -418,7 +390,7 @@ namespace Duplicati.Library.Main.Operation.Restore
{
lock (results)
{
results.BrokenLocalFiles.Add(file.Path);
results.BrokenLocalFiles.Add(file.TargetPath);
}
}
@@ -430,18 +402,18 @@ namespace Duplicati.Library.Main.Operation.Restore
if (Convert.ToBase64String(filehasher.Hash) == file.Hash)
{
// Truncate the file if it is larger than the expected size.
FileInfo fi = new (file.Path);
FileInfo fi = new (file.TargetPath);
if (file.Length < fi.Length)
{
if (options.Dryrun)
{
Logging.Log.WriteDryrunMessage(LOGTAG, "DryrunRestore", @$"Would have truncated ""{file.Path}"" from {fi.Length} to {file.Length}");
Logging.Log.WriteDryrunMessage(LOGTAG, "DryrunRestore", @$"Would have truncated ""{file.TargetPath}"" from {fi.Length} to {file.Length}");
}
else
{
// Reopen file with write permission
fi.IsReadOnly = false; // The metadata handler will revert this back later.
using var f = new FileStream(file.Path, FileMode.Open, FileAccess.ReadWrite);
using var f = new FileStream(file.TargetPath, FileMode.Open, FileAccess.ReadWrite);
f.SetLength(file.Length);
}
}
@@ -469,18 +441,18 @@ namespace Duplicati.Library.Main.Operation.Restore
/// <param name="results">The restoration results.</param>
/// <param name="block_request">The channel to request blocks from the block manager. Used to inform the block manager which blocks are already present.</param>
/// <returns>An awaitable `Task`, which returns a collection of data blocks that are missing.</returns>
private static async Task<(long, List<BlockRequest>)> VerifyLocalBlocks(LocalRestoreDatabase.IFileToRestore file, List<BlockRequest> blocks, long total_blocks, System.Security.Cryptography.HashAlgorithm filehasher, System.Security.Cryptography.HashAlgorithm blockhasher, Options options, RestoreResults results, IChannel<BlockRequest> block_request)
private static async Task<(long, List<BlockRequest>)> VerifyLocalBlocks(FileRequest file, List<BlockRequest> blocks, long total_blocks, System.Security.Cryptography.HashAlgorithm filehasher, System.Security.Cryptography.HashAlgorithm blockhasher, Options options, RestoreResults results, IChannel<BlockRequest> block_request)
{
List<BlockRequest> missing_blocks = [];
// Check if the file exists
if (File.Exists(file.Name))
if (File.Exists(file.OriginalPath))
{
filehasher.Initialize();
// Open both files, as the target file is still being read to produce the overall file hash, if all the blocks are present across both the target and original files.
using var f_original = new FileStream(file.Name, FileMode.Open, FileAccess.Read);
using var f_target = new FileStream(file.Path, FileMode.OpenOrCreate, FileAccess.ReadWrite);
using var f_original = new FileStream(file.OriginalPath, FileMode.Open, FileAccess.Read);
using var f_target = new FileStream(file.TargetPath, FileMode.OpenOrCreate, FileAccess.ReadWrite);
var buffer = new byte[options.Blocksize];
long bytes_read = 0;
long bytes_written = 0;
@@ -501,7 +473,7 @@ namespace Duplicati.Library.Main.Operation.Restore
{
lock (results)
{
results.BrokenLocalFiles.Add(file.Path);
results.BrokenLocalFiles.Add(file.TargetPath);
}
throw;
}
@@ -524,7 +496,7 @@ namespace Duplicati.Library.Main.Operation.Restore
{
lock (results)
{
results.BrokenLocalFiles.Add(file.Path);
results.BrokenLocalFiles.Add(file.TargetPath);
}
throw;
}
@@ -558,7 +530,7 @@ namespace Duplicati.Library.Main.Operation.Restore
{
lock (results)
{
results.BrokenLocalFiles.Add(file.Path);
results.BrokenLocalFiles.Add(file.TargetPath);
}
throw;
}
@@ -577,7 +549,7 @@ namespace Duplicati.Library.Main.Operation.Restore
{
if (options.Dryrun)
{
Logging.Log.WriteDryrunMessage(LOGTAG, "DryrunRestore", @$"Would have truncated ""{file.Path}"" from {f_target.Length} to {file.Length}");
Logging.Log.WriteDryrunMessage(LOGTAG, "DryrunRestore", @$"Would have truncated ""{file.TargetPath}"" from {f_target.Length} to {file.Length}");
}
else
{
@@ -589,7 +561,7 @@ namespace Duplicati.Library.Main.Operation.Restore
{
lock (results)
{
results.BrokenLocalFiles.Add(file.Path);
results.BrokenLocalFiles.Add(file.TargetPath);
}
throw;
}
@@ -598,10 +570,10 @@ namespace Duplicati.Library.Main.Operation.Restore
}
else
{
Logging.Log.WriteErrorMessage(LOGTAG, "FileHashMismatch", null, $"File hash mismatch for {file.Path} - expected: {file.Hash}, actual: {Convert.ToBase64String(filehasher.Hash)}");
Logging.Log.WriteErrorMessage(LOGTAG, "FileHashMismatch", null, $"File hash mismatch for {file.TargetPath} - expected: {file.Hash}, actual: {Convert.ToBase64String(filehasher.Hash)}");
lock (results)
{
results.BrokenLocalFiles.Add(file.Path);
results.BrokenLocalFiles.Add(file.TargetPath);
}
}
}
Duplicati/Library/Main/Operation/Restore/Interfaces.cs
+19
View File
@@ -41,4 +41,23 @@ namespace Duplicati.Library.Main.Operation.Restore
public bool CacheDecrEvict { get; set; } = cacheDecrEvict;
}
/// <summary>
/// Represents a file request that the `FileLister` process sends to the `FileProcessor` process.
/// </summary>
/// <param name="ID">The File ID.</param>
/// <param name="OriginalPath">The original path of the file.</param>
/// <param name="TargetPath">The target path of the file.</param>
/// <param name="Hash">The file hash.</param>
/// <param name="Length">The length of the file.</param>
/// <param name="BlocksetID">The BlocksetID of the file.</param>
public class FileRequest(long ID, string OriginalPath, string TargetPath, string Hash, long Length, long BlocksetID)
{
public long ID { get; } = ID;
public string OriginalPath { get; } = OriginalPath;
public string TargetPath { get; } = TargetPath;
public string Hash { get; } = Hash;
public long Length { get; } = Length;
public long BlocksetID { get; } = BlocksetID;
}
}
Duplicati/Library/Main/Operation/Restore/VolumeManager.cs
+1 -8
View File
@@ -73,11 +73,6 @@ namespace Duplicati.Library.Main.Operation.Restore
Stopwatch sw_request = options.InternalProfiling ? new () : null;
Stopwatch sw_wakeup = options.InternalProfiling ? new () : null;
// Prepare the command to get the volume information
using var cmd = db.Connection.CreateCommand();
cmd.CommandText = "SELECT Name, Size, Hash FROM RemoteVolume WHERE ID = ?";
cmd.AddParameter();
try
{
while (true)
@@ -113,9 +108,7 @@ namespace Duplicati.Library.Main.Operation.Restore
else
{
sw_query?.Start();
cmd.SetParameterValue(0, request.VolumeID);
var (volume_name, volume_size, volume_hash) = cmd.ExecuteReaderEnumerable()
.Select(x => (x.GetString(0), x.GetInt64(1), x.GetString(2))).First();
var (volume_name, volume_size, volume_hash) = db.GetVolumeInfo(request.VolumeID).First();
sw_query?.Stop();
sw_backend?.Start();
var handle = backend.GetAsync(volume_name, volume_size, volume_hash);