Kenneth Skovhede
f03cf2f7a3
When creating a new controller, the options dictionary was used by-reference. This meant that code could update the options dictionary *after* creating the controller, which would not be thread-safe. It does not have much effect on regular operations as the normal method is to create the dictionary, pass it, and not modify it. But in the tests, we frequently re-use the dictionary and update it with new values to test various cases. Due to the dictionary being a reference, the controller would update the dictionary to fit the operations, but this would reflect back into the tests, meaning that some tests were being run with options that were not supposed to be there. This meant that at least two edge cases that we test for were never found to have errors, even though they do. The first issue is the listing of files using a non-default blocksize. The second issue is recreating the database with a non-default file-hash algorithm. This PR removes the reference passing, and fixes the issues that were uncovered.
1046 lines
62 KiB
C#
1046 lines
62 KiB
C#
// Copyright (C) 2026, The Duplicati Team
|
|
// https://duplicati.com, hello@duplicati.com
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a
|
|
// copy of this software and associated documentation files (the "Software"),
|
|
// to deal in the Software without restriction, including without limitation
|
|
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
// and/or sell copies of the Software, and to permit persons to whom the
|
|
// Software is furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in
|
|
// all copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
|
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
// DEALINGS IN THE SOFTWARE.
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using Duplicati.Library.Interface;
|
|
using Duplicati.Library.Main.Database;
|
|
using Duplicati.Library.Main.Volumes;
|
|
using Duplicati.Library.Utility;
|
|
|
|
namespace Duplicati.Library.Main.Operation
|
|
{
|
|
internal class RecreateDatabaseHandler : IDisposable
|
|
{
|
|
/// <summary>
|
|
/// The tag used for logging
|
|
/// </summary>
|
|
private static readonly string LOGTAG = Logging.Log.LogTagFromType<RecreateDatabaseHandler>();
|
|
|
|
private readonly Options m_options;
|
|
private readonly RecreateDatabaseResults m_result;
|
|
|
|
public delegate IEnumerable<KeyValuePair<long, IParsedVolume>> NumberedFilterFilelistDelegate(IEnumerable<IParsedVolume> filelist);
|
|
public delegate void BlockVolumePostProcessor(string volumename, BlockVolumeReader reader);
|
|
|
|
public RecreateDatabaseHandler(Options options, RecreateDatabaseResults result)
|
|
{
|
|
m_options = options;
|
|
m_result = result;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Run the recreate procedure.
|
|
/// </summary>
|
|
/// <param name="path">Path to the database that will be created.</param>
|
|
/// <param name="backendManager">The backend manager to use for downloading files.</param>
|
|
/// <param name="filter">Filters the files in a filelist to prevent downloading unwanted data.</param>
|
|
/// <param name="filelistfilter">A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist.</param>
|
|
/// <param name="blockprocessor">A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists.</param>
|
|
public async Task RunAsync(string path, IBackendManager backendManager, IFilter filter, NumberedFilterFilelistDelegate filelistfilter, BlockVolumePostProcessor blockprocessor)
|
|
{
|
|
if (File.Exists(path))
|
|
throw new UserInformationException(string.Format("Cannot recreate database because file already exists: {0}", path), "RecreateTargetDatabaseExists");
|
|
|
|
await using var db =
|
|
await LocalDatabase.CreateLocalDatabaseAsync(path, "Recreate", true, null, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
|
|
await DoRunAsync(backendManager, db, false, filter, filelistfilter, blockprocessor).ConfigureAwait(false);
|
|
|
|
// Ensure database is consistent after the recreate
|
|
await db
|
|
.VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Updates a database with new path information from a remote fileset.
|
|
/// </summary>
|
|
/// <param name="backendManager">The backend manager to use for downloading files.</param>
|
|
/// <param name="filter">Filters the files in a filelist to prevent downloading unwanted data.</param>
|
|
/// <param name="filelistfilter">A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist.</param>
|
|
/// <param name="blockprocessor">A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists.</param>
|
|
public async Task RunUpdateAsync(IBackendManager backendManager, Library.Utility.IFilter filter, NumberedFilterFilelistDelegate filelistfilter, BlockVolumePostProcessor blockprocessor)
|
|
{
|
|
if (!m_options.RepairOnlyPaths)
|
|
throw new UserInformationException(string.Format("Can only update with paths, try setting --{0}", "repair-only-paths"), "RepairUpdateRequiresPathsOnly");
|
|
|
|
await using var db =
|
|
await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Recreate", true, null, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
if ((await db.FindMatchingFilesetsAsync(m_options.Time, m_options.Version, m_result.TaskControl.ProgressToken).ConfigureAwait(false)).Any())
|
|
{
|
|
if (m_options.IgnoreUpdateIfVersionExists)
|
|
{
|
|
Logging.Log.WriteInformationMessage(LOGTAG, "UpdateVersionAlreadyExists", "The version(s) being updated to already exists, ignoring update request");
|
|
return;
|
|
}
|
|
|
|
throw new UserInformationException("The version(s) being updated to, already exists", "UpdateVersionAlreadyExists");
|
|
}
|
|
|
|
// Mark as incomplete
|
|
await db.PartiallyRecreatedAsync(m_result.TaskControl.ProgressToken, true).ConfigureAwait(false);
|
|
|
|
var preexistingOptionsInDatabase =
|
|
await Utility.ContainsOptionsForVerificationAsync(db, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
await Utility.UpdateOptionsFromDbAsync(db, m_options, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
|
|
// Make sure the options have not changed between calls, unless there are no previous options
|
|
if (preexistingOptionsInDatabase)
|
|
await Utility.VerifyOptionsAndUpdateDatabaseAsync(db, m_options, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
|
|
await DoRunAsync(backendManager, db, true, filter, filelistfilter, blockprocessor).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Run the recreate procedure.
|
|
/// </summary>
|
|
/// <param name="backendManager">The backend manager to use for downloading files.</param>
|
|
/// <param name="dbparent">The database to restore into.</param>
|
|
/// <param name="updating">True if this is an update call, false otherwise.</param>
|
|
/// <param name="filter">A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist.</param>
|
|
/// <param name="filelistfilter">Filters the files in a filelist to prevent downloading unwanted data.</param>
|
|
/// <param name="blockprocessor">A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists.</param>
|
|
/// <returns>A task that completes when the operation is done.</returns>
|
|
internal async Task DoRunAsync(IBackendManager backendManager, LocalDatabase dbparent, bool updating, IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null)
|
|
{
|
|
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Recreate_Running);
|
|
|
|
//We build a local database in steps.
|
|
await using var restoredb =
|
|
await LocalRecreateDatabase.CreateAsync(dbparent, m_options, null, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
await restoredb.RepairInProgressAsync(m_result.TaskControl.ProgressToken, true).ConfigureAwait(false);
|
|
|
|
// If a filelist filter is applied, we're only recreating a partial database (e.g., for a specific version restore).
|
|
// Mark it as partially recreated so it cannot be used for backups or other operations that require a complete database.
|
|
if (filelistfilter != null)
|
|
await restoredb.PartiallyRecreatedAsync(m_result.TaskControl.ProgressToken, true).ConfigureAwait(false);
|
|
|
|
var expRecreateDb = false; // experimental recreate db code flag
|
|
var volumeIds = new Dictionary<string, long>();
|
|
|
|
if (string.Equals(Environment.GetEnvironmentVariable("EXPERIMENTAL_RECREATEDB_DUPLICATI") ?? string.Empty, "1"))
|
|
{
|
|
expRecreateDb = true;
|
|
}
|
|
|
|
var rawlist = await backendManager.ListAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
|
|
|
//First step is to examine the remote storage to see what
|
|
// kind of data we can find
|
|
var remotefiles =
|
|
(from x in rawlist
|
|
let n = VolumeBase.ParseFilename(x)
|
|
where
|
|
n != null
|
|
&&
|
|
n.Prefix == m_options.Prefix
|
|
select n).ToArray(); //ToArray() ensures that we do not remote-request it multiple times
|
|
|
|
if (remotefiles.Length == 0)
|
|
{
|
|
if (!rawlist.Any())
|
|
throw new UserInformationException("No files were found at the remote location, perhaps the target url is incorrect?", "EmptyRemoteLocation");
|
|
else
|
|
{
|
|
var tmp =
|
|
(from x in rawlist
|
|
let n = VolumeBase.ParseFilename(x)
|
|
where
|
|
n != null
|
|
select n.Prefix).ToArray();
|
|
|
|
var types = tmp.Distinct().ToArray();
|
|
if (tmp.Length == 0)
|
|
throw new UserInformationException(string.Format("Found {0} files at the remote storage, but none that could be parsed", rawlist.Count()), "EmptyRemoteLocation");
|
|
else if (types.Length == 1)
|
|
throw new UserInformationException(string.Format("Found {0} parse-able files with the prefix {1}, did you forget to set the backup prefix?", tmp.Length, types[0]), "EmptyRemoteLocationWithPrefix");
|
|
else
|
|
throw new UserInformationException(string.Format("Found {0} parse-able files (of {1} files) with different prefixes: {2}, did you forget to set the backup prefix?", tmp.Length, rawlist.Count(), string.Join(", ", types)), "EmptyRemoteLocationWithPrefix");
|
|
}
|
|
}
|
|
|
|
if (string.IsNullOrWhiteSpace(m_options.Passphrase) && remotefiles.Any(x => !string.IsNullOrWhiteSpace(x.EncryptionModule)))
|
|
throw new UserInformationException("The remote files are encrypted, but no passphrase was provided", "MissingPassphrase");
|
|
|
|
//Then we select the filelist we should work with,
|
|
// and create the filelist table to fit
|
|
IEnumerable<IParsedVolume> filelists =
|
|
from n in remotefiles
|
|
where n.FileType == RemoteVolumeType.Files
|
|
orderby n.Time descending
|
|
select n;
|
|
|
|
if (!filelists.Any())
|
|
throw new UserInformationException("No filelists found on the remote destination", "EmptyRemoteLocation");
|
|
|
|
if (filelistfilter != null)
|
|
filelists = filelistfilter(filelists).Select(x => x.Value).ToArray();
|
|
|
|
if (!filelists.Any())
|
|
throw new UserInformationException("No filelists", "NoMatchingRemoteFilelists");
|
|
|
|
// If we are updating, all files should be accounted for
|
|
foreach (var fl in remotefiles)
|
|
volumeIds[fl.File.Name] = updating
|
|
? await restoredb.GetRemoteVolumeIDAsync(fl.File.Name, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false)
|
|
: await restoredb.RegisterRemoteVolumeAsync(fl.File.Name, fl.FileType, fl.File.Size, RemoteVolumeState.Uploaded, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
|
|
var hasUpdatedOptions = false;
|
|
|
|
// Record all blocksets and files needed
|
|
var filelistWork = (from n in filelists orderby n.Time select new RemoteVolume(n.File) as IRemoteVolume).ToList();
|
|
Logging.Log.WriteInformationMessage(LOGTAG, "RebuildStarted", "Rebuild database started, downloading {0} filelists", filelistWork.Count);
|
|
|
|
var progress = 0;
|
|
|
|
// Register the files we are working with, if not already updated
|
|
if (updating)
|
|
{
|
|
foreach (var n in filelists)
|
|
if (volumeIds[n.File.Name] == -1)
|
|
volumeIds[n.File.Name] = await restoredb
|
|
.RegisterRemoteVolumeAsync(n.File.Name, n.FileType, RemoteVolumeState.Uploaded, n.File.Size, new TimeSpan(0), m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
}
|
|
|
|
var isFirstFilelist = true;
|
|
// At this point, we do not know the hashing used to verify the files, so we need to use direct download, and manually decrypt the files
|
|
// so we can calculate the hash AFTER we have read the manifest content and updated the options
|
|
await foreach (var (tmpencfile, name) in backendManager.GetFilesOverlappedDirectAsync(filelistWork, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
|
{
|
|
try
|
|
{
|
|
if (!await m_result.TaskControl.ProgressRendevouzAsync().ConfigureAwait(false))
|
|
{
|
|
await backendManager.WaitForEmptyAsync(restoredb, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
|
m_result.EndTime = DateTime.UtcNow;
|
|
// Implicit rollback
|
|
await restoredb.Transaction
|
|
.RollBackAsync(m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
return;
|
|
}
|
|
|
|
progress++;
|
|
if (filelistWork.Count == 1 && m_options.RepairOnlyPaths)
|
|
{
|
|
m_result.OperationProgressUpdater.UpdateProgress(0.5f);
|
|
}
|
|
else
|
|
{
|
|
m_result.OperationProgressUpdater.UpdateProgress(((float)progress / filelistWork.Count()) * (m_options.RepairOnlyPaths ? 1f : 0.2f));
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingFilelistVolumes", "Processing filelist volume {0} of {1}", progress, filelistWork.Count);
|
|
}
|
|
|
|
using (var tmpfile = backendManager.DecryptFile(tmpencfile, name, m_options, dispose: false))
|
|
{
|
|
isFirstFilelist = false;
|
|
var parsed = VolumeBase.ParseFilename(name);
|
|
|
|
using var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read);
|
|
using var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions);
|
|
if (compressor == null)
|
|
throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule");
|
|
|
|
if (!hasUpdatedOptions)
|
|
{
|
|
VolumeReaderBase.UpdateOptionsFromManifest(compressor, m_options);
|
|
hasUpdatedOptions = true;
|
|
}
|
|
|
|
var size = new FileInfo(tmpencfile).Length;
|
|
string hash;
|
|
using (var fs = File.OpenRead(tmpencfile))
|
|
using (var hasher = HashFactory.CreateHasher(m_options.FileHashAlgorithm))
|
|
hash = Convert.ToBase64String(hasher.ComputeHash(fs));
|
|
|
|
if (!string.IsNullOrWhiteSpace(hash) && size > 0)
|
|
await restoredb
|
|
.UpdateRemoteVolumeAsync(name, RemoteVolumeState.Verified, size, hash, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
|
|
|
|
// Create timestamped operations based on the file timestamp
|
|
var filesetid = await restoredb
|
|
.CreateFilesetAsync(volumeIds[name], parsed.Time, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
|
|
await RecreateFilesetFromRemoteListAsync(restoredb, compressor, filesetid, m_options, filter, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "FileProcessingFailed", ex, "Failed to process file: {0}", name);
|
|
if (ex.IsAbortException())
|
|
{
|
|
m_result.EndTime = DateTime.UtcNow;
|
|
// Implicit rollback
|
|
await restoredb.Transaction
|
|
.RollBackAsync(m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
throw;
|
|
}
|
|
|
|
if (isFirstFilelist && ex is System.Security.Cryptography.CryptographicException)
|
|
{
|
|
m_result.EndTime = DateTime.UtcNow;
|
|
// Implicit rollback
|
|
await restoredb.Transaction
|
|
.RollBackAsync(m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
throw;
|
|
}
|
|
|
|
if (m_options.UnittestMode)
|
|
{
|
|
// Implicit rollback
|
|
await restoredb.Transaction
|
|
.RollBackAsync(m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
throw;
|
|
}
|
|
}
|
|
}
|
|
|
|
//Make sure we write the config if it has been read from a manifest
|
|
if (hasUpdatedOptions)
|
|
await Utility.VerifyOptionsAndUpdateDatabaseAsync(restoredb, m_options, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
|
|
using (new Logging.Timer(LOGTAG, "CommitUpdateFilesetFromRemote", "CommitUpdateFilesetFromRemote"))
|
|
await restoredb.Transaction
|
|
.CommitAsync(m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
|
|
// do we stop after just handling the dlist files ?
|
|
// (if yes, we never will be able to do a backup !)
|
|
if (!m_options.RepairOnlyPaths)
|
|
{
|
|
var hashsize = 0;
|
|
//Grab all index files, and update the block table
|
|
|
|
using (var hashalg = HashFactory.CreateHasher(m_options.BlockHashAlgorithm))
|
|
{
|
|
hashsize = hashalg.HashSize / 8;
|
|
|
|
var indexfiles = (
|
|
from n in remotefiles
|
|
where n.FileType == RemoteVolumeType.Index
|
|
select new RemoteVolume(n.File) as IRemoteVolume).ToList();
|
|
|
|
Logging.Log.WriteInformationMessage(LOGTAG, "FilelistsRestored", "Filelists restored, downloading {0} index files", indexfiles.Count);
|
|
|
|
progress = 0;
|
|
|
|
await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(indexfiles, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
|
{
|
|
try
|
|
{
|
|
if (!await m_result.TaskControl.ProgressRendevouzAsync().ConfigureAwait(false))
|
|
{
|
|
await backendManager.WaitForEmptyAsync(restoredb, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
|
m_result.EndTime = DateTime.UtcNow;
|
|
// Implicit rollback
|
|
await restoredb.Transaction
|
|
.RollBackAsync(m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
return;
|
|
}
|
|
|
|
progress++;
|
|
m_result.OperationProgressUpdater.UpdateProgress((((float)progress / indexfiles.Count) * 0.5f) + 0.2f);
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingIndexlistVolumes", "Processing indexlist volume {0} of {1}", progress, indexfiles.Count);
|
|
|
|
using (tmpfile)
|
|
{
|
|
if (!string.IsNullOrWhiteSpace(hash) && size > 0)
|
|
await restoredb
|
|
.UpdateRemoteVolumeAsync(name, RemoteVolumeState.Verified, size, hash, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
|
|
using (var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options, hashsize))
|
|
{
|
|
foreach (var a in svr.Volumes)
|
|
{
|
|
var filename = a.Filename;
|
|
var volumeID = await restoredb
|
|
.GetRemoteVolumeIDAsync(filename, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
|
|
// No such file
|
|
if (volumeID < 0)
|
|
(volumeID, filename) =
|
|
await ProbeForMatchingFilenameAsync(filename, restoredb, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
|
|
var missing = false;
|
|
// Still broken, register a missing item
|
|
if (volumeID < 0)
|
|
{
|
|
var p = VolumeBase.ParseFilename(filename);
|
|
if (p == null)
|
|
throw new Exception(string.Format("Unable to parse filename: {0}", filename));
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "MissingFileDetected", null, "Remote file referenced as {0} by {1}, but not found in list, registering a missing remote file", filename, name);
|
|
missing = true;
|
|
volumeID = await restoredb
|
|
.RegisterRemoteVolumeAsync(filename, p.FileType, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
}
|
|
|
|
bool anyChange = false;
|
|
//Add all block/volume mappings
|
|
foreach (var b in a.Blocks)
|
|
{
|
|
anyChange |= (await restoredb.UpdateBlockAsync(b.Key, b.Value, volumeID, m_result.TaskControl.ProgressToken).ConfigureAwait(false)).Item1;
|
|
}
|
|
|
|
await restoredb
|
|
.UpdateRemoteVolumeAsync(filename, missing ? RemoteVolumeState.Temporary : RemoteVolumeState.Verified, a.Length, a.Hash, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
await restoredb
|
|
.AddIndexBlockLinkAsync(
|
|
await restoredb.GetRemoteVolumeIDAsync(name, m_result.TaskControl.ProgressToken).ConfigureAwait(false),
|
|
volumeID,
|
|
m_result.TaskControl.ProgressToken
|
|
)
|
|
.ConfigureAwait(false);
|
|
}
|
|
|
|
//If there are blocklists in the index file, add them to the temp blocklist hashes table
|
|
int wrongHashes = 0;
|
|
foreach (var b in svr.BlockLists)
|
|
{
|
|
// Compact might have created undetected invalid blocklist entries in index files due to broken LocalDatabase.GetBlocklists
|
|
// If the hash is wrong, recreate will download the dblock volume with the correct file
|
|
try
|
|
{
|
|
// We need to instantiate the list to ensure the verification is
|
|
// done before we add it to the database, since we do not have nested transactions
|
|
var list = b.Blocklist.ToList();
|
|
await restoredb
|
|
.AddTempBlockListHashAsync(b.Hash, list, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
}
|
|
catch (System.IO.InvalidDataException e)
|
|
{
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "InvalidDataBlocklist", e, "Exception while processing blocklists in {0}", name);
|
|
++wrongHashes;
|
|
}
|
|
}
|
|
if (wrongHashes != 0)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "WrongBlocklistHashes", null, "{0} had invalid blocklists which could not be used. Consider deleting this index file and run repair to recreate it.", name);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
//Not fatal
|
|
Logging.Log.WriteErrorMessage(LOGTAG, "IndexFileProcessingFailed", ex, "Failed to process index file: {0}", name);
|
|
if (ex.IsAbortException())
|
|
{
|
|
m_result.EndTime = DateTime.UtcNow;
|
|
// Implicit rollback
|
|
await restoredb.Transaction
|
|
.RollBackAsync(m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
throw;
|
|
}
|
|
|
|
if (m_options.UnittestMode)
|
|
{
|
|
// Implicit rollback
|
|
await restoredb.Transaction
|
|
.RollBackAsync(m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
throw;
|
|
}
|
|
}
|
|
}
|
|
|
|
using (new Logging.Timer(LOGTAG, "CommitRecreateDb", "CommitRecreatedDb"))
|
|
await restoredb.Transaction
|
|
.CommitAsync(m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
|
|
// TODO: In some cases, we can avoid downloading all index files,
|
|
// if we are lucky and pick the right ones
|
|
}
|
|
|
|
await restoredb.CleanupMissingVolumesAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
|
|
|
// Update the real tables from the temp tables
|
|
if (expRecreateDb)
|
|
// add missing blocks and blocksetentry data (at this point
|
|
// we have not yet anything in the blocksetentry table)
|
|
await restoredb
|
|
.AddBlockAndBlockSetEntryFromTempAsync(hashsize, m_options.Blocksize, false, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
else
|
|
await restoredb
|
|
.FindMissingBlocklistHashesAsync(hashsize, m_options.Blocksize, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
|
|
// We have now grabbed as much information as possible,
|
|
// if we are still missing data, we must now fetch block files
|
|
//We do this in three passes
|
|
for (var i = 0; i < 3; i++)
|
|
{
|
|
// Grab the list matching the pass type
|
|
var lst = await restoredb
|
|
.GetMissingBlockListVolumesAsync(i, m_options.Blocksize, hashsize, m_options.RepairForceBlockUse, m_result.TaskControl.ProgressToken)
|
|
.ToListAsync()
|
|
.ConfigureAwait(false);
|
|
|
|
if (lst.Count == 0)
|
|
continue;
|
|
|
|
var fullist = ": " + string.Join(", ", lst.Select(x => x.Name));
|
|
switch (i)
|
|
{
|
|
case 0:
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingRequiredBlocklistVolumes", "Processing required {0} blocklist volumes{1}", lst.Count, fullist);
|
|
Logging.Log.WriteInformationMessage(LOGTAG, "ProcessingRequiredBlocklistVolumes", "Processing required {0} blocklist volumes{1}", lst.Count, m_options.FullResult ? fullist : string.Empty);
|
|
break;
|
|
case 1:
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "ProbingCandicateBlocklistVolumes", "Probing {0} candidate blocklist volumes{1}", lst.Count, fullist);
|
|
Logging.Log.WriteInformationMessage(LOGTAG, "ProbingCandicateBlocklistVolumes", "Probing {0} candidate blocklist volumes{1}", lst.Count, m_options.FullResult ? fullist : string.Empty);
|
|
break;
|
|
default:
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingAllBlocklistVolumes", "Processing all of the {0} volumes for blocklists{1}", lst.Count, fullist);
|
|
Logging.Log.WriteInformationMessage(LOGTAG, "ProcessingAllBlocklistVolumes", "Processing all of the {0} volumes for blocklists{1}", lst.Count, m_options.FullResult ? fullist : string.Empty);
|
|
break;
|
|
}
|
|
|
|
progress = 0;
|
|
await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(lst, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
|
{
|
|
try
|
|
{
|
|
using (tmpfile)
|
|
using (var rd = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options))
|
|
{
|
|
if (!await m_result.TaskControl.ProgressRendevouzAsync().ConfigureAwait(false))
|
|
{
|
|
await backendManager.WaitForEmptyAsync(restoredb, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
|
m_result.EndTime = DateTime.UtcNow;
|
|
// Implicit rollback
|
|
await restoredb.Transaction
|
|
.RollBackAsync(m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
return;
|
|
}
|
|
|
|
progress++;
|
|
m_result.OperationProgressUpdater.UpdateProgress((((float)progress / lst.Count) * 0.1f) + 0.7f + (i * 0.1f));
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingBlocklistVolumes", "Pass {0} of 3, processing blocklist volume {1} of {2}", (i + 1), progress, lst.Count);
|
|
|
|
var volumeid = await restoredb
|
|
.GetRemoteVolumeIDAsync(name, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
|
|
await restoredb
|
|
.UpdateRemoteVolumeAsync(name, RemoteVolumeState.Uploaded, size, hash, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
|
|
bool anyChange = false;
|
|
// Update the block table so we know about the block/volume map
|
|
foreach (var h in rd.Blocks)
|
|
{
|
|
anyChange |= (await restoredb.UpdateBlockAsync(h.Key, h.Value, volumeid, m_result.TaskControl.ProgressToken).ConfigureAwait(false)).Item1;
|
|
}
|
|
|
|
// now that we have the blocks/volume relationships, we can go from the (already known from dlist step) blocklisthashes
|
|
// to the needed list blocks in the volume, so grab them from the database
|
|
// read the blocks list hashes from the volume data (the handled file) and insert them into the temp blocklisthash table
|
|
await foreach (var blocklisthash in restoredb.GetBlockListsAsync(volumeid, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
|
{
|
|
if (await restoredb.AddTempBlockListHashAsync(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize), m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
|
{
|
|
anyChange = true;
|
|
}
|
|
}
|
|
|
|
// Update tables if necessary (if no block or hash have been changed by a data volume
|
|
// there is no need to run expensive queries - most data volumes have been
|
|
// managed successfully by correct index volumes), so we know if we are done
|
|
// if any change, add to the block and blocksetentry tables the references found in
|
|
// the block lists of the volume saved in the temp blocklisthash table by AddTempBLockListHash
|
|
if (anyChange)
|
|
{
|
|
if (i == 2)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "UpdatingTables", null, "Unexpected changes caused by block {0}", name);
|
|
}
|
|
if (expRecreateDb)
|
|
await restoredb
|
|
.AddBlockAndBlockSetEntryFromTempAsync(hashsize, m_options.Blocksize, false, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
else
|
|
await restoredb
|
|
.FindMissingBlocklistHashesAsync(hashsize, m_options.Blocksize, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
}
|
|
|
|
using (new Logging.Timer(LOGTAG, "CommitRestoredBlocklist", "CommitRestoredBlocklist"))
|
|
await restoredb.Transaction
|
|
.CommitAsync(m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
|
|
//At this point we can patch files with data from the block volume
|
|
if (blockprocessor != null)
|
|
blockprocessor(name, rd);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "FailedRebuildingWithFile", ex, "Failed to use information from {0} to rebuild database: {1}", name, ex.Message);
|
|
if (m_options.UnittestMode)
|
|
{
|
|
// Implicit rollback
|
|
await restoredb.Transaction
|
|
.RollBackAsync(m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
throw;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
await backendManager.WaitForEmptyAsync(restoredb, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
|
|
|
if (!m_options.RepairOnlyPaths)
|
|
{
|
|
// All blocks are collected and added into the Block table
|
|
// Find out which blocks are deleted and move them into DeletedBlock,
|
|
// so that compact can calculate the unused space
|
|
await restoredb.CleanupDeletedBlocksAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
|
}
|
|
|
|
await restoredb.CleanupMissingVolumesAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
|
|
|
if (!m_options.RepairOnlyPaths && m_options.StoreMetadataContentInDatabase)
|
|
{
|
|
await RestoreMetadataContentAsync(backendManager, restoredb, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
|
|
// The metadata content restore is executed after the main recreate commits.
|
|
// Ensure the restored Metadataset.Content values are persisted.
|
|
await restoredb.Transaction
|
|
.CommitAsync(m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
}
|
|
|
|
if (m_options.RepairOnlyPaths)
|
|
{
|
|
Logging.Log.WriteInformationMessage(LOGTAG, "RecreateOrUpdateOnly", "Recreate/path-update completed, not running consistency checks");
|
|
}
|
|
else
|
|
{
|
|
Logging.Log.WriteInformationMessage(LOGTAG, "RecreateCompletedCheckingDatabase", "Recreate completed, verifying the database consistency");
|
|
|
|
//All done, we must verify that we have all blocklist fully intact
|
|
// if this fails, the db will not be deleted, so it can be used,
|
|
// except to continue a backup
|
|
m_result.EndTime = DateTime.UtcNow;
|
|
|
|
await using (var lbfdb = await LocalListBrokenFilesDatabase.CreateAsync(restoredb, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
|
{
|
|
var broken = await lbfdb
|
|
.GetBrokenFilesetsAsync(new DateTime(0), null, m_result.TaskControl.ProgressToken)
|
|
.CountAsync(cancellationToken: m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
|
|
if (broken != 0)
|
|
throw new UserInformationException(string.Format("Recreated database has missing blocks and {0} broken filelists. Consider using \"{1}\" and \"{2}\" to purge broken data from the remote store and the database.", broken, "list-broken-files", "purge-broken-files"), "DatabaseIsBrokenConsiderPurge");
|
|
}
|
|
|
|
// When a filelist filter is applied, we are intentionally only recreating a subset
|
|
// of the database (e.g., restoring a specific version with --no-local-db).
|
|
// In this case, we need to use the lax consistency check that allows volumes
|
|
// without filesets, as only the selected filesets are being recreated.
|
|
if (filelistfilter != null)
|
|
{
|
|
await restoredb
|
|
.VerifyConsistencyForRepairAsync(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
}
|
|
else
|
|
{
|
|
await restoredb
|
|
.VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
}
|
|
|
|
Logging.Log.WriteInformationMessage(LOGTAG, "RecreateCompleted", "Recreate completed, and consistency checks completed, marking database as complete");
|
|
|
|
await restoredb.RepairInProgressAsync(m_result.TaskControl.ProgressToken, false).ConfigureAwait(false);
|
|
}
|
|
|
|
m_result.EndTime = DateTime.UtcNow;
|
|
}
|
|
|
|
private sealed record MetadataBlockRef(long MetadataId, long BlockIndex, string BlockHash, long BlockSize);
|
|
|
|
private sealed class MetadataRecreateState
|
|
{
|
|
public long MetadataId { get; }
|
|
public long MetaLength { get; }
|
|
public int ExpectedBlockCount { get; set; }
|
|
|
|
// Index -> block bytes
|
|
public SortedDictionary<long, byte[]> Blocks { get; } = new();
|
|
|
|
public MetadataRecreateState(long metadataId, long metaLength)
|
|
{
|
|
MetadataId = metadataId;
|
|
MetaLength = metaLength;
|
|
}
|
|
}
|
|
|
|
private async Task RestoreMetadataContentAsync(IBackendManager backendManager, LocalRecreateDatabase restoredb, CancellationToken cancellationToken)
|
|
{
|
|
// Metadata content restoration requires the block structure to be present.
|
|
if (m_options.RepairOnlyPaths)
|
|
return;
|
|
|
|
// Gather block references for each metadataset that is missing content.
|
|
var volumeBlocks = new Dictionary<string, List<MetadataBlockRef>>(StringComparer.Ordinal);
|
|
var volumeInfo = new Dictionary<string, RemoteVolume>(StringComparer.Ordinal);
|
|
var metaStates = new Dictionary<long, MetadataRecreateState>();
|
|
|
|
await foreach (var info in restoredb.GetMissingMetadataBlocksAsync(cancellationToken).ConfigureAwait(false))
|
|
{
|
|
var metadataId = info.MetadataId;
|
|
var metaLength = info.MetaLength;
|
|
var blockIndex = info.BlockIndex;
|
|
var blockHash = info.BlockHash;
|
|
var blockSize = info.BlockSize;
|
|
var volumeName = info.VolumeName;
|
|
var volumeHash = info.VolumeHash;
|
|
var volumeSize = info.VolumeSize;
|
|
|
|
if (string.IsNullOrEmpty(volumeName) || string.IsNullOrEmpty(blockHash) || blockSize <= 0)
|
|
continue;
|
|
|
|
if (!metaStates.TryGetValue(metadataId, out var state))
|
|
metaStates[metadataId] = state = new MetadataRecreateState(metadataId, metaLength);
|
|
state.ExpectedBlockCount++;
|
|
|
|
if (!volumeBlocks.TryGetValue(volumeName, out var list))
|
|
volumeBlocks[volumeName] = list = new List<MetadataBlockRef>();
|
|
list.Add(new MetadataBlockRef(metadataId, blockIndex, blockHash, blockSize));
|
|
|
|
if (!volumeInfo.ContainsKey(volumeName))
|
|
volumeInfo[volumeName] = new RemoteVolume(volumeName, volumeHash, volumeSize);
|
|
}
|
|
|
|
if (volumeInfo.Count == 0)
|
|
return;
|
|
|
|
Logging.Log.WriteInformationMessage(LOGTAG, "RestoringMetadataContent", "Restoring metadata content into the database by downloading {0} block volume(s)", volumeInfo.Count);
|
|
|
|
// Download all required block volumes and extract metadata blocks.
|
|
var volumesToDownload = volumeInfo.Values.Cast<IRemoteVolume>().ToList();
|
|
await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(volumesToDownload, cancellationToken).ConfigureAwait(false))
|
|
{
|
|
using (tmpfile)
|
|
{
|
|
if (!await m_result.TaskControl.ProgressRendevouzAsync().ConfigureAwait(false))
|
|
return;
|
|
|
|
if (!volumeBlocks.TryGetValue(name, out var blocksForVolume) || blocksForVolume.Count == 0)
|
|
continue;
|
|
|
|
try
|
|
{
|
|
using var reader = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options);
|
|
|
|
foreach (var bref in blocksForVolume)
|
|
{
|
|
if (!metaStates.TryGetValue(bref.MetadataId, out var state))
|
|
continue;
|
|
|
|
// Already captured
|
|
if (state.Blocks.ContainsKey(bref.BlockIndex))
|
|
continue;
|
|
|
|
if (bref.BlockSize > int.MaxValue)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "MetadataBlockTooLarge", null, "Skipping metadata block {0} for metadataset {1} because block size {2} exceeds supported range", bref.BlockHash, bref.MetadataId, bref.BlockSize);
|
|
continue;
|
|
}
|
|
|
|
var buf = new byte[(int)bref.BlockSize];
|
|
var read = reader.ReadBlock(bref.BlockHash, buf);
|
|
if (read != buf.Length)
|
|
{
|
|
// Trim buffer to actual read length.
|
|
Array.Resize(ref buf, read);
|
|
}
|
|
|
|
state.Blocks[bref.BlockIndex] = buf;
|
|
|
|
// If we have all blocks for this metadataset, assemble and store content now.
|
|
if (state.Blocks.Count == state.ExpectedBlockCount)
|
|
{
|
|
try
|
|
{
|
|
var ms = new MemoryStream(checked((int)Math.Min(state.MetaLength, int.MaxValue)));
|
|
foreach (var kvp in state.Blocks.OrderBy(x => x.Key))
|
|
{
|
|
if (ms.Length >= state.MetaLength)
|
|
break;
|
|
|
|
var remaining = state.MetaLength - ms.Length;
|
|
var take = (int)Math.Min(remaining, kvp.Value.LongLength);
|
|
await ms.WriteAsync(kvp.Value, 0, take);
|
|
}
|
|
|
|
if (ms.Length != state.MetaLength)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "MetadataContentIncomplete", null, "Failed to restore metadataset content for {0}: expected {1} bytes, got {2} bytes", state.MetadataId, state.MetaLength, ms.Length);
|
|
}
|
|
else
|
|
{
|
|
var content = Library.Utility.Utility.GetStringWithoutBOM(ms, false);
|
|
await restoredb.SetMetadataContentAsync(state.MetadataId, content, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "MetadataContentRestoreFailed", ex, "Failed to restore metadataset content for {0}", state.MetadataId);
|
|
}
|
|
finally
|
|
{
|
|
metaStates.Remove(state.MetadataId);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "MetadataVolumeProcessingFailed", ex, "Failed to process block volume {0} for metadata content restore", name);
|
|
if (ex.IsAbortException())
|
|
throw;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Recreate a fileset from the remote filelist.
|
|
/// </summary>
|
|
/// <param name="restoredb">The database to restore to.</param>
|
|
/// <param name="compressor">The compression module to use.</param>
|
|
/// <param name="filesetid">The ID of the fileset to recreate.</param>
|
|
/// <param name="options">The options to use for the operation.</param>
|
|
/// <param name="filter">The filter to apply to the files.</param>
|
|
/// <param name="cancellationToken">The cancellation token to monitor for cancellation requests.</param>
|
|
/// <returns>A task that completes when the fileset has been recreated.</returns>
|
|
public static async Task RecreateFilesetFromRemoteListAsync(LocalRecreateDatabase restoredb, ICompression compressor, long filesetid, Options options, IFilter filter, CancellationToken cancellationToken)
|
|
{
|
|
var blocksize = options.Blocksize;
|
|
var hashes_pr_block = blocksize / options.BlockhashSize;
|
|
|
|
// retrieve fileset data from dlist
|
|
var filesetData = VolumeReaderBase.GetFilesetData(compressor, options);
|
|
|
|
// update fileset using filesetData
|
|
await restoredb
|
|
.UpdateFullBackupStateInFilesetAsync(filesetid, filesetData.IsFullBackup, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
|
|
// clear any existing fileset entries
|
|
await restoredb.ClearFilesetEntriesAsync(filesetid, cancellationToken).ConfigureAwait(false);
|
|
|
|
using (var filelistreader = new FilesetVolumeReader(compressor, options))
|
|
foreach (var fe in filelistreader.Files.Where(x => Library.Utility.FilterExpression.Matches(filter, x.Path)))
|
|
{
|
|
try
|
|
{
|
|
var expectedmetablocks = (fe.Metasize + blocksize - 1) / blocksize;
|
|
var expectedmetablocklisthashes = (expectedmetablocks + hashes_pr_block - 1) / hashes_pr_block;
|
|
if (expectedmetablocks <= 1) expectedmetablocklisthashes = 0;
|
|
|
|
if (!options.AllowRestoreOutsideTargetDirectory)
|
|
{
|
|
// Check for cross-platform rooted paths
|
|
var isAbsolute =
|
|
(fe.Path.Length > 0 && fe.Path[0] == '/') ||
|
|
(fe.Path.Length > 2 && fe.Path[1] == ':' && (fe.Path[2] == '\\' || fe.Path[2] == '/')) ||
|
|
fe.Path.StartsWith("\\\\");
|
|
|
|
// Check for relative path segments (accounting for mixed path separators)
|
|
var normalizedPath = fe.Path.Replace('\\', '/');
|
|
var hasRelativeSegments = normalizedPath.Contains("/../")
|
|
|| normalizedPath.EndsWith("/..")
|
|
|| normalizedPath.StartsWith("../");
|
|
|
|
if (hasRelativeSegments)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "PathTraversalDetected", null, "Path traversal detected in path: {0}", fe.Path);
|
|
continue;
|
|
}
|
|
|
|
if (!Path.IsPathFullyQualified(fe.Path) && !isAbsolute)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "InvalidPath", null, "Invalid path: {0}", fe.Path);
|
|
continue;
|
|
}
|
|
}
|
|
|
|
var metadataid = long.MinValue;
|
|
var split = LocalDatabase.SplitIntoPrefixAndName(fe.Path);
|
|
var prefixid = await restoredb
|
|
.GetOrCreatePathPrefixAsync(split.Key, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
|
|
switch (fe.Type)
|
|
{
|
|
case FilelistEntryType.Folder:
|
|
metadataid = await restoredb
|
|
.AddMetadatasetAsync(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
await restoredb
|
|
.AddDirectoryEntryAsync(filesetid, prefixid, split.Value, fe.Time, metadataid, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
|
|
break;
|
|
case FilelistEntryType.File:
|
|
var expectedblocks = (fe.Size + blocksize - 1) / blocksize;
|
|
var expectedblocklisthashes = (expectedblocks + hashes_pr_block - 1) / hashes_pr_block;
|
|
if (expectedblocks <= 1) expectedblocklisthashes = 0;
|
|
|
|
var blocksetid = await restoredb
|
|
.AddBlocksetAsync(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
metadataid = await restoredb
|
|
.AddMetadatasetAsync(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
await restoredb
|
|
.AddFileEntryAsync(filesetid, prefixid, split.Value, fe.Time, blocksetid, metadataid, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
|
|
if (fe.Size <= blocksize && fe.Size > 0)
|
|
{
|
|
if (!string.IsNullOrWhiteSpace(fe.Blockhash))
|
|
await restoredb
|
|
.AddSmallBlocksetLinkAsync(fe.Hash, fe.Blockhash, fe.Blocksize, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
else if (options.BlockHashAlgorithm == options.FileHashAlgorithm)
|
|
await restoredb
|
|
.AddSmallBlocksetLinkAsync(fe.Hash, fe.Hash, fe.Size, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
else
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "MissingBlockHash", null, "No block hash found for file: {0}", fe.Path);
|
|
}
|
|
|
|
break;
|
|
case FilelistEntryType.Symlink:
|
|
metadataid = await restoredb
|
|
.AddMetadatasetAsync(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
await restoredb
|
|
.AddSymlinkEntryAsync(filesetid, prefixid, split.Value, fe.Time, metadataid, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
break;
|
|
default:
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "SkippingUnknownFileEntry", null, "Skipping file-entry with unknown type {0}: {1} ", fe.Type, fe.Path);
|
|
break;
|
|
}
|
|
|
|
if (fe.Metasize <= blocksize && (fe.Type == FilelistEntryType.Folder || fe.Type == FilelistEntryType.File || fe.Type == FilelistEntryType.Symlink))
|
|
{
|
|
if (!string.IsNullOrWhiteSpace(fe.Metablockhash))
|
|
await restoredb
|
|
.AddSmallBlocksetLinkAsync(fe.Metahash, fe.Metablockhash, fe.Metasize, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
else if (options.BlockHashAlgorithm == options.FileHashAlgorithm)
|
|
await restoredb
|
|
.AddSmallBlocksetLinkAsync(fe.Metahash, fe.Metahash, fe.Metasize, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
else
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "MissingMetadataBlockHash", null, "No block hash found for file metadata: {0}", fe.Path);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "FileEntryProcessingFailed", ex, "Failed to process file-entry: {0}", fe.Path);
|
|
}
|
|
}
|
|
|
|
await restoredb.Transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Look in the database for filenames similar to the current filename, but with a different compression and encryption module.
|
|
/// </summary>
|
|
/// <param name="filename">The filename read and written.</param>
|
|
/// <param name="restoredb">The database to query.</param>
|
|
/// <param name="cancellationToken">A cancellation token to cancel the operation.</param>
|
|
/// <returns>The volume id of the item.</returns>
|
|
public static async Task<(long, string)> ProbeForMatchingFilenameAsync(string filename, LocalRecreateDatabase restoredb, CancellationToken cancellationToken)
|
|
{
|
|
var p = VolumeBase.ParseFilename(filename);
|
|
if (p != null)
|
|
{
|
|
foreach (var compmodule in Library.DynamicLoader.CompressionLoader.Keys)
|
|
foreach (var encmodule in Library.DynamicLoader.EncryptionLoader.Keys.Union([""]))
|
|
{
|
|
var testfilename = VolumeBase.GenerateFilename(p.FileType, p.Prefix, p.Guid, p.Time, compmodule, encmodule);
|
|
var tvid = await restoredb
|
|
.GetRemoteVolumeIDAsync(testfilename, cancellationToken)
|
|
.ConfigureAwait(false);
|
|
|
|
if (tvid >= 0)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "RewritingFilenameMapping", null, "Unable to find volume {0}, but mapping to matching file {1}", filename, testfilename);
|
|
filename = testfilename;
|
|
return (tvid, filename);
|
|
}
|
|
}
|
|
}
|
|
|
|
return (-1, filename);
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
}
|
|
}
|
|
}
|