548 lines
27 KiB
C#
548 lines
27 KiB
C#
// Copyright (C) 2024, The Duplicati Team
|
|
// https://duplicati.com, hello@duplicati.com
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a
|
|
// copy of this software and associated documentation files (the "Software"),
|
|
// to deal in the Software without restriction, including without limitation
|
|
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
// and/or sell copies of the Software, and to permit persons to whom the
|
|
// Software is furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in
|
|
// all copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
|
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
// DEALINGS IN THE SOFTWARE.
|
|
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Threading.Tasks;
|
|
using CoCoL;
|
|
using Duplicati.Library.Main.Database;
|
|
using Duplicati.Library.Utility;
|
|
|
|
namespace Duplicati.Library.Main.Operation.Restore
|
|
{
|
|
|
|
/// <summary>
|
|
/// Process that handles each file that needs to be restored. It starts by
|
|
/// identifying the blocks that need to be restored. Then it verifies the
|
|
/// which of the target file blocks are missing. Then it checks whether it
|
|
/// can use local blocks to restore the file. Finally, it restores the file
|
|
/// by downloading the missing blocks and writing them to the target file.
|
|
/// It also verifies the file hash and truncates the file if necessary.
|
|
/// </summary>
|
|
internal class FileProcessor
|
|
{
|
|
private static readonly string LOGTAG = Logging.Log.LogTagFromType<FileProcessor>();
|
|
|
|
public static Task Run(LocalRestoreDatabase db, IChannel<BlockRequest> block_request, IChannel<byte[]> block_response, RestoreHandlerMetadataStorage metadatastorage, Options options, RestoreResults results)
|
|
{
|
|
return AutomationExtensions.RunTask(
|
|
new
|
|
{
|
|
Input = Channels.filesToRestore.ForRead,
|
|
},
|
|
async self =>
|
|
{
|
|
// TODO preallocate the file size to avoid fragmentation / help the operating system / filesystem. Verify this in a benchmark - I think it relies on OS and filesystem.
|
|
// using (var fs = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.None) { fs.SetLength(size); fs.Seek(0, SeekOrigin.Begin); }
|
|
|
|
try
|
|
{
|
|
using var filehasher = HashFactory.CreateHasher(options.FileHashAlgorithm);
|
|
using var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm);
|
|
using var cmd = db.Connection.CreateCommand();
|
|
cmd.CommandText = @$"
|
|
SELECT Block.ID, Block.Hash, Block.Size, Block.VolumeID
|
|
FROM BlocksetEntry INNER JOIN Block
|
|
ON BlocksetEntry.BlockID = Block.ID
|
|
WHERE BlocksetEntry.BlocksetID = ?";
|
|
cmd.AddParameter();
|
|
|
|
while (true)
|
|
{
|
|
// Get the next file to restore.
|
|
var file = await self.Input.ReadAsync();
|
|
|
|
// Get information about the blocks for the file
|
|
cmd.SetParameterValue(0, file.BlocksetID);
|
|
var blocks = cmd.ExecuteReaderEnumerable()
|
|
.Select((b, i) =>
|
|
new BlockRequest(b.GetInt64(0), i, b.GetString(1), b.GetInt64(2), b.GetInt64(3), false)
|
|
)
|
|
.ToArray();
|
|
|
|
// Verify the target file blocks that may already exist.
|
|
var missing_blocks = await VerifyTargetBlocks(file, blocks, filehasher, blockhasher, options, results, block_request);
|
|
|
|
long bytes_written = 0;
|
|
|
|
if (missing_blocks.Count > 0 && options.UseLocalBlocks && missing_blocks.Count > 0)
|
|
{
|
|
// Verify the local blocks at the original restore path that may be used to restore the file.
|
|
var (bw, new_missing_blocks) = await VerifyLocalBlocks(file, missing_blocks, blocks.Length, filehasher, blockhasher, options, results, block_request);
|
|
bytes_written = bw;
|
|
missing_blocks = new_missing_blocks;
|
|
}
|
|
|
|
if (file.BlocksetID != LocalDatabase.SYMLINK_BLOCKSET_ID && (missing_blocks.Count == 0 || (missing_blocks.Count == 1 && missing_blocks[0].BlockSize == 0)))
|
|
{
|
|
if (options.Dryrun)
|
|
{
|
|
Logging.Log.WriteDryrunMessage(LOGTAG, "DryrunRestore", @$"Would have created empty file ""{file.Path}""");
|
|
}
|
|
else
|
|
{
|
|
// Create an empty file, or truncate to 0
|
|
using var fs = new System.IO.FileStream(file.Path, System.IO.FileMode.OpenOrCreate, System.IO.FileAccess.Write, System.IO.FileShare.None);
|
|
fs.SetLength(0);
|
|
if (missing_blocks.Count != 0)
|
|
{
|
|
blocks[0].CacheDecrEvict = true;
|
|
await block_request.WriteAsync(blocks[0]);
|
|
}
|
|
}
|
|
}
|
|
else if (missing_blocks.Count > 0)
|
|
{
|
|
if (missing_blocks.Any(x => x.VolumeID < 0))
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "NegativeVolumeID", null, $"{file.Path} has a negative volume ID, skipping");
|
|
continue;
|
|
}
|
|
|
|
filehasher.Initialize();
|
|
|
|
FileStream fs = null;
|
|
try
|
|
{
|
|
// Open the target file
|
|
if (options.Dryrun)
|
|
{
|
|
// If dryrun, open the file for read only to verify the file hash.
|
|
if (System.IO.File.Exists(file.Path))
|
|
{
|
|
fs = new System.IO.FileStream(file.Path, System.IO.FileMode.Open, System.IO.FileAccess.Read);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
fs = new System.IO.FileStream(file.Path, System.IO.FileMode.OpenOrCreate, System.IO.FileAccess.ReadWrite, System.IO.FileShare.None);
|
|
}
|
|
|
|
// Burst the block requests to speed up the restore
|
|
int burst = Channels.bufferSize;
|
|
int j = 0;
|
|
for (int i = 0; i < (int) Math.Min(missing_blocks.Count, burst); i++)
|
|
{
|
|
await block_request.WriteAsync(missing_blocks[i]);
|
|
}
|
|
for (int i = 0; i < blocks.Length; i++)
|
|
{
|
|
// Each response block is not verified against the block hash, as the processes downloading the blocks should do that.
|
|
if (j < missing_blocks.Count && missing_blocks[j].BlockOffset == i)
|
|
{
|
|
// Read the block from the response and issue a new request, if more blocks are missing
|
|
var data = await block_response.ReadAsync();
|
|
if (j < missing_blocks.Count - burst)
|
|
{
|
|
await block_request.WriteAsync(missing_blocks[j + burst]);
|
|
}
|
|
|
|
// Hash the block to verify the file hash
|
|
filehasher.TransformBlock(data, 0, data.Length, data, 0);
|
|
if (options.Dryrun)
|
|
{
|
|
// Simulate writing the block
|
|
fs.Seek(fs.Position + blocks[i].BlockSize, System.IO.SeekOrigin.Begin);
|
|
}
|
|
else
|
|
{
|
|
// Write the block to the file
|
|
await fs.WriteAsync(data);
|
|
}
|
|
|
|
// Keep track of metrics
|
|
bytes_written += data.Length;
|
|
j++;
|
|
}
|
|
else
|
|
{
|
|
// No more blocks are missing, so read the rest of the blocks to verify the file hash.
|
|
var data = new byte[blocks[i].BlockSize];
|
|
var read = await fs.ReadAsync(data, 0, data.Length);
|
|
filehasher.TransformBlock(data, 0, read, data, 0);
|
|
}
|
|
}
|
|
|
|
if (options.Dryrun)
|
|
{
|
|
Logging.Log.WriteDryrunMessage(LOGTAG, "DryrunRestore", @$"Would have restored {bytes_written} bytes of ""{file.Path}""");
|
|
}
|
|
|
|
// Verify the file hash
|
|
filehasher.TransformFinalBlock([], 0, 0);
|
|
if (Convert.ToBase64String(filehasher.Hash) != file.Hash)
|
|
{
|
|
Logging.Log.WriteErrorMessage(LOGTAG, "FileHashMismatch", null, $"File hash mismatch for {file.Path} - expected: {file.Hash}, actual: {Convert.ToBase64String(filehasher.Hash)}");
|
|
throw new Exception("File hash mismatch");
|
|
}
|
|
|
|
// Truncate the file if it is larger than the expected size.
|
|
if (fs?.Length > file.Length)
|
|
{
|
|
if (options.Dryrun)
|
|
{
|
|
Logging.Log.WriteDryrunMessage(LOGTAG, "DryrunRestore", @$"Would have truncated ""{file.Path}"" from {fs.Length} to {file.Length}");
|
|
}
|
|
else
|
|
{
|
|
fs.SetLength(file.Length);
|
|
}
|
|
}
|
|
}
|
|
catch (Exception)
|
|
{
|
|
lock (results)
|
|
{
|
|
results.BrokenLocalFiles.Add(file.Path);
|
|
}
|
|
throw;
|
|
}
|
|
finally
|
|
{
|
|
fs?.Dispose();
|
|
}
|
|
}
|
|
|
|
if (!options.SkipMetadata) {
|
|
await RestoreMetadata(db, file, block_request, block_response, options, results);
|
|
}
|
|
|
|
// Keep track of the restored files and their sizes
|
|
lock (results)
|
|
{
|
|
results.RestoredFiles++;
|
|
results.SizeOfRestoredFiles += bytes_written;
|
|
}
|
|
}
|
|
}
|
|
catch (RetiredException)
|
|
{
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "RetiredProcess", null, "File processor retired");
|
|
block_request.Retire();
|
|
block_response.Retire();
|
|
return;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logging.Log.WriteErrorMessage(LOGTAG, "FileProcessingError", ex, "Error during file processing");
|
|
block_request.Retire();
|
|
block_response.Retire();
|
|
throw;
|
|
}
|
|
});
|
|
}
|
|
|
|
private static async Task<bool> RestoreMetadata(LocalRestoreDatabase db, LocalRestoreDatabase.IFileToRestore file, IChannel<BlockRequest> block_request, IChannel<byte[]> block_response, Options options, RestoreResults results)
|
|
{
|
|
var cmd = db.Connection.CreateCommand();
|
|
cmd.CommandText = $@"
|
|
SELECT Block.ID, Block.Hash, Block.Size, Block.VolumeID
|
|
FROM ""{db.m_tempfiletable}""
|
|
INNER JOIN Metadataset ON ""{db.m_tempfiletable}"".MetadataID = Metadataset.ID
|
|
INNER JOIN BlocksetEntry ON Metadataset.BlocksetID = BlocksetEntry.BlocksetID
|
|
INNER JOIN Block ON BlocksetEntry.BlockID = Block.ID
|
|
WHERE ""{db.m_tempfiletable}"".ID = ?
|
|
";
|
|
cmd.AddParameter();
|
|
cmd.SetParameterValue(0, file.ID);
|
|
|
|
var blocks = cmd.ExecuteReaderEnumerable()
|
|
.Select((b, i) =>
|
|
new BlockRequest(b.GetInt64(0), i, b.GetString(1), b.GetInt64(2), b.GetInt64(3), false)
|
|
)
|
|
.ToArray();
|
|
|
|
using var ms = new MemoryStream();
|
|
|
|
foreach (var block in blocks)
|
|
{
|
|
await block_request.WriteAsync(block);
|
|
var data = await block_response.ReadAsync();
|
|
ms.Write(data, 0, data.Length);
|
|
}
|
|
ms.Seek(0, SeekOrigin.Begin);
|
|
|
|
RestoreHandler.ApplyMetadata(file.Path, ms, options.RestorePermissions, options.RestoreSymlinkMetadata, options.Dryrun);
|
|
|
|
return true;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies the target blocks of a file that may already exist.
|
|
/// </summary>
|
|
/// <param name="file">The target file.</param>
|
|
/// <param name="blocks">The metadata about the blocks that make up the file.</param>
|
|
/// <param name="filehasher">A hasher for the file.</param>
|
|
/// <param name="blockhasher">A hasher for a data block.</param>
|
|
/// <param name="options">The Duplicati configuration options.</param>
|
|
/// <param name="results">The restoration results.</param>
|
|
/// <returns>An awaitable `Task`, which returns a collection of data blocks that are missing.</returns>
|
|
private static async Task<List<BlockRequest>> VerifyTargetBlocks(LocalRestoreDatabase.IFileToRestore file, BlockRequest[] blocks, System.Security.Cryptography.HashAlgorithm filehasher, System.Security.Cryptography.HashAlgorithm blockhasher, Options options, RestoreResults results, IChannel<BlockRequest> block_request)
|
|
{
|
|
List<BlockRequest> missing_blocks = [];
|
|
|
|
// Check if the file exists
|
|
if (System.IO.File.Exists(file.Path))
|
|
{
|
|
filehasher.Initialize();
|
|
try
|
|
{
|
|
using var f = new System.IO.FileStream(file.Path, System.IO.FileMode.Open, System.IO.FileAccess.Read);
|
|
var buffer = new byte[options.Blocksize];
|
|
long bytes_read = 0;
|
|
for (int i = 0; i < blocks.Length; i++)
|
|
{
|
|
var read = await f.ReadAsync(buffer, 0, (int)blocks[i].BlockSize);
|
|
if (read == blocks[i].BlockSize)
|
|
{
|
|
// Block is present
|
|
filehasher.TransformBlock(buffer, 0, read, buffer, 0);
|
|
var blockhash = Convert.ToBase64String(blockhasher.ComputeHash(buffer, 0, read));
|
|
if (blockhash == blocks[i].BlockHash)
|
|
{
|
|
// Block matches
|
|
bytes_read += read;
|
|
blocks[i].CacheDecrEvict = true;
|
|
await block_request.WriteAsync(blocks[i]);
|
|
}
|
|
else
|
|
{
|
|
// Block mismatch
|
|
missing_blocks.Add(blocks[i]);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// Block is missing
|
|
missing_blocks.Add(blocks[i]);
|
|
if (f.Position == f.Length)
|
|
{
|
|
// No more file - the rest of the blocks are missing.
|
|
missing_blocks.AddRange(blocks.Skip(i + 1));
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception)
|
|
{
|
|
lock (results)
|
|
{
|
|
results.BrokenLocalFiles.Add(file.Path);
|
|
}
|
|
}
|
|
|
|
// If all of the individual blocks have been verified.
|
|
if (missing_blocks.Count == 0)
|
|
{
|
|
// Verify the file hash
|
|
filehasher.TransformFinalBlock([], 0, 0);
|
|
if (Convert.ToBase64String(filehasher.Hash) == file.Hash)
|
|
{
|
|
// Truncate the file if it is larger than the expected size.
|
|
FileInfo fi = new FileInfo(file.Path);
|
|
if (file.Length < fi.Length)
|
|
{
|
|
if (options.Dryrun)
|
|
{
|
|
Logging.Log.WriteDryrunMessage(LOGTAG, "DryrunRestore", @$"Would have truncated ""{file.Path}"" from {fi.Length} to {file.Length}");
|
|
}
|
|
else
|
|
{
|
|
// Reopen file with write permission
|
|
fi.IsReadOnly = false; // The metadata handler will revert this back later.
|
|
using var f = new System.IO.FileStream(file.Path, System.IO.FileMode.Open, System.IO.FileAccess.ReadWrite);
|
|
f.SetLength(file.Length);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// The file doesn't exist, so all blocks are missing.
|
|
missing_blocks.AddRange(blocks);
|
|
}
|
|
|
|
return missing_blocks;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies the local blocks at the original restore path that may be used to restore the file.
|
|
/// </summary>
|
|
/// <param name="file">The file to restore. Contains both the target and original paths.</param>
|
|
/// <param name="blocks">The collection of blocks that are currently missing.</param>
|
|
/// <param name="total_blocks">The total number of blocks for the file.</param>
|
|
/// <param name="filehasher">A hasher for the file.</param>
|
|
/// <param name="blockhasher">A hasher for the data block.</param>
|
|
/// <param name="options">The Duplicati configuration options.</param>
|
|
/// <param name="results">The restoration results.</param>
|
|
/// <returns>An awaitable `Task`, which returns a collection of data blocks that are missing.</returns>
|
|
private static async Task<(long, List<BlockRequest>)> VerifyLocalBlocks(LocalRestoreDatabase.IFileToRestore file, List<BlockRequest> blocks, long total_blocks, System.Security.Cryptography.HashAlgorithm filehasher, System.Security.Cryptography.HashAlgorithm blockhasher, Options options, RestoreResults results, IChannel<BlockRequest> block_request)
|
|
{
|
|
List<BlockRequest> missing_blocks = [];
|
|
|
|
// Check if the file exists
|
|
if (System.IO.File.Exists(file.Name))
|
|
{
|
|
filehasher.Initialize();
|
|
|
|
// Open both files, as the target file is still being read to produce the overall file hash, if all the blocks are present across both the target and original files.
|
|
using var f_original = new System.IO.FileStream(file.Name, System.IO.FileMode.Open, System.IO.FileAccess.Read);
|
|
using var f_target = new System.IO.FileStream(file.Path, System.IO.FileMode.OpenOrCreate, System.IO.FileAccess.ReadWrite);
|
|
var buffer = new byte[options.Blocksize];
|
|
long bytes_read = 0;
|
|
long bytes_written = 0;
|
|
|
|
int j = 0;
|
|
for (long i = 0; i < total_blocks; i++)
|
|
{
|
|
int read;
|
|
if (j < blocks.Count && blocks[j].BlockOffset == i)
|
|
{
|
|
// The current block is a missing block
|
|
try
|
|
{
|
|
f_original.Seek(i * (long)options.Blocksize, System.IO.SeekOrigin.Begin);
|
|
read = await f_original.ReadAsync(buffer, 0, (int) blocks[j].BlockSize);
|
|
}
|
|
catch (Exception)
|
|
{
|
|
lock (results)
|
|
{
|
|
results.BrokenLocalFiles.Add(file.Path);
|
|
}
|
|
throw;
|
|
}
|
|
|
|
if (read == blocks[j].BlockSize)
|
|
{
|
|
var blockhash = Convert.ToBase64String(blockhasher.ComputeHash(buffer, 0, read));
|
|
if (blockhash != blocks[j].BlockHash)
|
|
{
|
|
missing_blocks.Add(blocks[j]);
|
|
}
|
|
else
|
|
{
|
|
try
|
|
{
|
|
f_target.Seek(blocks[j].BlockOffset * (long)options.Blocksize, System.IO.SeekOrigin.Begin);
|
|
await f_target.WriteAsync(buffer, 0, read);
|
|
}
|
|
catch (Exception)
|
|
{
|
|
lock (results)
|
|
{
|
|
results.BrokenLocalFiles.Add(file.Path);
|
|
}
|
|
throw;
|
|
}
|
|
bytes_read += read;
|
|
bytes_written += read;
|
|
blocks[j].CacheDecrEvict = true;
|
|
await block_request.WriteAsync(blocks[j]);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
missing_blocks.Add(blocks[j]);
|
|
if (f_original.Position == f_original.Length)
|
|
{
|
|
missing_blocks.AddRange(blocks.Skip(j + 1));
|
|
break;
|
|
}
|
|
}
|
|
|
|
j++;
|
|
}
|
|
else
|
|
{
|
|
// The current block is not a missing block - read from the target file.
|
|
try
|
|
{
|
|
f_target.Seek(i * (long)options.Blocksize, System.IO.SeekOrigin.Begin);
|
|
read = await f_target.ReadAsync(buffer, 0, options.Blocksize);
|
|
}
|
|
catch (Exception)
|
|
{
|
|
lock (results)
|
|
{
|
|
results.BrokenLocalFiles.Add(file.Path);
|
|
}
|
|
throw;
|
|
}
|
|
}
|
|
filehasher.TransformBlock(buffer, 0, read, buffer, 0);
|
|
}
|
|
|
|
if (missing_blocks.Count == 0)
|
|
{
|
|
// No more blocks are missing, so check the file hash.
|
|
filehasher.TransformFinalBlock([], 0, 0);
|
|
if (Convert.ToBase64String(filehasher.Hash) == file.Hash)
|
|
{
|
|
// Truncate the file if it is larger than the expected size.
|
|
if (file.Length < f_target.Length)
|
|
{
|
|
if (options.Dryrun)
|
|
{
|
|
Logging.Log.WriteDryrunMessage(LOGTAG, "DryrunRestore", @$"Would have truncated ""{file.Path}"" from {f_target.Length} to {file.Length}");
|
|
}
|
|
else
|
|
{
|
|
try
|
|
{
|
|
f_target.SetLength(file.Length);
|
|
}
|
|
catch (Exception)
|
|
{
|
|
lock (results)
|
|
{
|
|
results.BrokenLocalFiles.Add(file.Path);
|
|
}
|
|
throw;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
Logging.Log.WriteErrorMessage(LOGTAG, "FileHashMismatch", null, $"File hash mismatch for {file.Path} - expected: {file.Hash}, actual: {Convert.ToBase64String(filehasher.Hash)}");
|
|
lock (results)
|
|
{
|
|
results.BrokenLocalFiles.Add(file.Path);
|
|
}
|
|
}
|
|
}
|
|
|
|
return (bytes_written, missing_blocks);
|
|
}
|
|
else
|
|
{
|
|
// The original file is no longer present, so all blocks are missing.
|
|
return (0, blocks);
|
|
}
|
|
}
|
|
}
|
|
|
|
} |