Rewrote the way the database is recreated to ensure greater stability when recovering from a damaged backup.
The restore now correctly works with missing index files and full index files.
This commit is contained in:
Kenneth Skovhede
@@ -138,6 +138,15 @@ CREATE TABLE "DeletedBlock" (
|
||||
"VolumeID" INTEGER NOT NULL
|
||||
);
|
||||
|
||||
/*
|
||||
If extra copies of blocks are detected,
|
||||
they are recorded here
|
||||
*/
|
||||
CREATE TABLE "DuplicateBlock" (
|
||||
"BlockID" INTEGER NOT NULL,
|
||||
"VolumeID" INTEGER NOT NULL
|
||||
);
|
||||
|
||||
|
||||
/*
|
||||
A metadata set, essentially a placeholder
|
||||
|
||||
@@ -12,7 +12,7 @@ namespace Duplicati.Library.Main.Database
|
||||
/// <summary>
|
||||
/// An approximate size of a hash-string in memory (44 chars * 2 for unicode + 8 bytes for pointer = 104)
|
||||
/// </summary>
|
||||
private const uint HASH_GUESS_SIZE = 128;
|
||||
internal const uint HASH_GUESS_SIZE = 128;
|
||||
|
||||
/// <summary>
|
||||
/// An approximate size of a path string in bytes.
|
||||
@@ -23,7 +23,7 @@ namespace Duplicati.Library.Main.Database
|
||||
/// in slightly more memory being used that what the user
|
||||
/// specifies.
|
||||
/// </summary>
|
||||
private const uint PATH_STRING_GUESS_SIZE = 256;
|
||||
internal const uint PATH_STRING_GUESS_SIZE = 256;
|
||||
|
||||
/// <summary>
|
||||
/// The usage threshold for a lookup table that triggers a warning
|
||||
@@ -58,7 +58,6 @@ namespace Duplicati.Library.Main.Database
|
||||
private readonly System.Data.IDbCommand m_updateblockCommand;
|
||||
|
||||
private readonly System.Data.IDbCommand m_insertfileOperationCommand;
|
||||
private readonly System.Data.IDbCommand m_insertIndexBlockLink;
|
||||
|
||||
private HashDatabaseProtector<string> m_blockHashLookup;
|
||||
private HashDatabaseProtector<string, long> m_fileHashLookup;
|
||||
@@ -96,7 +95,6 @@ namespace Duplicati.Library.Main.Database
|
||||
m_selectfileSimpleCommand = m_connection.CreateCommand();
|
||||
m_selectfileHashCommand = m_connection.CreateCommand();
|
||||
m_findmetadatasetProbeCommand = m_connection.CreateCommand();
|
||||
m_insertIndexBlockLink = m_connection.CreateCommand();
|
||||
|
||||
m_findblockCommand.CommandText = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?";
|
||||
m_findblockCommand.AddParameters(2);
|
||||
@@ -153,10 +151,7 @@ namespace Duplicati.Library.Main.Database
|
||||
|
||||
m_updateblockCommand.CommandText = @"UPDATE ""Block"" SET ""VolumeID"" = ? WHERE ""Hash"" = ? AND ""Size"" = ? ";
|
||||
m_updateblockCommand.AddParameters(3);
|
||||
|
||||
m_insertIndexBlockLink.CommandText = @"INSERT INTO ""IndexBlockLink"" (""IndexVolumeID"", ""BlockVolumeID"") VALUES (?, ?)";
|
||||
m_insertIndexBlockLink.AddParameters(2);
|
||||
|
||||
|
||||
if (options.BlockHashLookupMemory > 0)
|
||||
m_blockHashLookup = new HashDatabaseProtector<string>(HASH_GUESS_SIZE, (ulong)options.BlockHashLookupMemory);
|
||||
if (options.FileHashLookupMemory > 0)
|
||||
@@ -812,33 +807,18 @@ namespace Duplicati.Library.Main.Database
|
||||
tr.Commit();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a timestamped backup operation to correctly associate the fileset with the time it was created.
|
||||
/// </summary>
|
||||
/// <param name="volumeid">The ID of the fileset volume to update</param>
|
||||
/// <param name="timestamp">The timestamp of the operation to create</param>
|
||||
/// <param name="transaction">An optional external transaction</param>
|
||||
public long CreateFileset(long volumeid, DateTime timestamp, System.Data.IDbTransaction transaction = null)
|
||||
{
|
||||
using (var cmd = m_connection.CreateCommand())
|
||||
using (var tr = new TemporaryTransactionWrapper(m_connection, transaction))
|
||||
{
|
||||
cmd.Transaction = tr.Parent;
|
||||
m_filesetId = Convert.ToInt64(cmd.ExecuteScalar(@"INSERT INTO ""Fileset"" (""OperationID"", ""Timestamp"", ""VolumeID"") VALUES (?, ?, ?); SELECT last_insert_rowid();", m_operationid, NormalizeDateTime(timestamp), volumeid));
|
||||
tr.Commit();
|
||||
return m_filesetId;
|
||||
}
|
||||
}
|
||||
|
||||
public void AddIndexBlockLink(long indexVolumeID, long blockVolumeID, System.Data.IDbTransaction transaction)
|
||||
{
|
||||
m_insertIndexBlockLink.Transaction = transaction;
|
||||
m_insertIndexBlockLink.SetParameterValue(0, indexVolumeID);
|
||||
m_insertIndexBlockLink.SetParameterValue(1, blockVolumeID);
|
||||
m_insertIndexBlockLink.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Creates a timestamped backup operation to correctly associate the fileset with the time it was created.
|
||||
/// </summary>
|
||||
/// <param name="volumeid">The ID of the fileset volume to update</param>
|
||||
/// <param name="timestamp">The timestamp of the operation to create</param>
|
||||
/// <param name="transaction">An optional external transaction</param>
|
||||
public override long CreateFileset(long volumeid, DateTime timestamp, System.Data.IDbTransaction transaction = null)
|
||||
{
|
||||
return m_filesetId = base.CreateFileset(volumeid, timestamp, transaction);
|
||||
}
|
||||
|
||||
public IEnumerable<KeyValuePair<long, DateTime>> GetIncompleteFilesets(System.Data.IDbTransaction transaction)
|
||||
{
|
||||
using(var cmd = m_connection.CreateCommand())
|
||||
|
||||
@@ -1,116 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
|
||||
namespace Duplicati.Library.Main.Database
|
||||
{
|
||||
internal partial class LocalBlocklistUpdateDatabase : LocalRestoreDatabase
|
||||
{
|
||||
protected string m_tempblockvolumetable;
|
||||
protected string m_temphashtable;
|
||||
|
||||
public LocalBlocklistUpdateDatabase(string path, long blocksize)
|
||||
: base(new LocalDatabase(path, "Rebuild"), blocksize)
|
||||
{
|
||||
}
|
||||
|
||||
public LocalBlocklistUpdateDatabase(LocalDatabase parentdb, long blocksize)
|
||||
: base(parentdb, blocksize)
|
||||
{
|
||||
}
|
||||
|
||||
public void FindMissingBlocklistHashes()
|
||||
{
|
||||
m_tempblockvolumetable = "MissingBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
|
||||
m_temphashtable = "Hashlist-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
|
||||
using (var cmd = m_connection.CreateCommand())
|
||||
{
|
||||
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT DISTINCT ""Block"".""VolumeID"" AS ""VolumeID"", ""BlocklistHash"".""Hash"" AS ""Hash"", 0 AS ""Restored"" FROM ""Block"", ""BlocklistHash"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" ", m_tempblockvolumetable));
|
||||
cmd.ExecuteNonQuery(string.Format(@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") SELECT DISTINCT ""Blockset"".""ID"", 0, ""Block"".""ID"" FROM ""Blockset"", ""Block"" WHERE ""Blockset"".""Fullhash"" = ""Block"".""Hash"" AND ""Blockset"".""Length"" < {0} AND ""Blockset"".""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"") ", m_blocksize));
|
||||
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Hash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)", m_temphashtable));
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public IEnumerable<string> GetBlockLists(long volumeid)
|
||||
{
|
||||
return new BlocklistsEnumerable(m_connection, m_tempblockvolumetable, volumeid);
|
||||
}
|
||||
|
||||
public void UpdateBlocklist(string hash, IEnumerable<string> hashes, long hashsize, System.Data.IDbTransaction transaction)
|
||||
{
|
||||
using (var cmd = m_connection.CreateCommand())
|
||||
{
|
||||
cmd.Transaction = transaction;
|
||||
|
||||
cmd.CommandText = string.Format(@"UPDATE ""{0}"" SET ""Restored"" = 1 WHERE ""Restored"" = 0 AND ""Hash"" = ? ", m_tempblockvolumetable);
|
||||
cmd.AddParameter(hash);
|
||||
var c = cmd.ExecuteNonQuery();
|
||||
if (c != 1)
|
||||
throw new Exception(string.Format("Blocklist not found {0}", hash));
|
||||
|
||||
cmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""{0}""", m_temphashtable));
|
||||
cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Hash"", ""Index"") VALUES (?, ?)", m_temphashtable);
|
||||
long ix = 0;
|
||||
foreach (var h in hashes)
|
||||
{
|
||||
c = cmd.ExecuteNonQuery(null, h, ix);
|
||||
if (c != 1)
|
||||
throw new Exception(string.Format("Insert row error {0}", h));
|
||||
ix++;
|
||||
}
|
||||
|
||||
cmd.CommandText = string.Format(@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") SELECT ""A"".""BlocksetID"", (""A"".""Index"" * ?) + ""B"".""Index"", ""C"".""ID"" FROM ""BlocklistHash"" A, ""{0}"" B, ""Block"" C WHERE ""A"".""Hash"" = ? AND ""C"".""Hash"" = ""B"".""Hash"" ", m_temphashtable);
|
||||
c = cmd.ExecuteNonQuery(null, (m_blocksize / hashsize), hash);
|
||||
if (c == 0 || c % ix != 0)
|
||||
throw new Exception(string.Format("Wrong number of inserts, got {0} records from {1} hashes!", c, ix));
|
||||
}
|
||||
}
|
||||
|
||||
public IEnumerable<IRemoteVolume> GetMissingBlockListVolumes()
|
||||
{
|
||||
using (var cmd = m_connection.CreateCommand())
|
||||
{
|
||||
cmd.CommandText = string.Format(@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT DISTINCT ""VolumeID"" FROM ""{0}"")", m_tempblockvolumetable);
|
||||
using (var rd = cmd.ExecuteReader())
|
||||
{
|
||||
object[] r = new object[3];
|
||||
while (rd.Read())
|
||||
{
|
||||
rd.GetValues(r);
|
||||
yield return new RemoteVolume(
|
||||
r[0] == null ? null : r[0].ToString(),
|
||||
r[1] == null ? null : r[1].ToString(),
|
||||
r[2] == null ? -1 : Convert.ToInt64(r[2])
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
base.Dispose();
|
||||
using (var cmd = m_connection.CreateCommand())
|
||||
{
|
||||
if (m_tempblockvolumetable != null)
|
||||
try
|
||||
{
|
||||
cmd.CommandText = string.Format(@"DROP TABLE ""{0}""", m_tempblockvolumetable);
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
finally { m_tempblockvolumetable = null; }
|
||||
|
||||
if (m_temphashtable != null)
|
||||
try
|
||||
{
|
||||
cmd.CommandText = string.Format(@"DROP TABLE ""{0}""", m_temphashtable);
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
finally { m_temphashtable = null; }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ namespace Duplicati.Library.Main.Database
|
||||
|
||||
private readonly System.Data.IDbCommand m_insertlogCommand;
|
||||
private readonly System.Data.IDbCommand m_insertremotelogCommand;
|
||||
private readonly System.Data.IDbCommand m_insertIndexBlockLink;
|
||||
|
||||
private BasicResults m_result;
|
||||
|
||||
@@ -95,6 +96,7 @@ namespace Duplicati.Library.Main.Database
|
||||
m_removeremotevolumeCommand = connection.CreateCommand();
|
||||
m_selectremotevolumeIdCommand = connection.CreateCommand();
|
||||
m_createremotevolumeCommand = connection.CreateCommand();
|
||||
m_insertIndexBlockLink = connection.CreateCommand();
|
||||
|
||||
m_insertlogCommand.CommandText = @"INSERT INTO ""LogData"" (""OperationID"", ""Timestamp"", ""Type"", ""Message"", ""Exception"") VALUES (?, ?, ?, ?, ?)";
|
||||
m_insertlogCommand.AddParameters(5);
|
||||
@@ -116,7 +118,10 @@ namespace Duplicati.Library.Main.Database
|
||||
m_selectremotevolumeIdCommand.CommandText = @"SELECT ""ID"" FROM ""Remotevolume"" WHERE ""Name"" = ?";
|
||||
|
||||
m_createremotevolumeCommand.CommandText = @"INSERT INTO ""Remotevolume"" (""OperationID"", ""Name"", ""Type"", ""State"") VALUES (?, ?, ?, ?); SELECT last_insert_rowid();";
|
||||
m_createremotevolumeCommand.AddParameters(4);
|
||||
m_createremotevolumeCommand.AddParameters(4);
|
||||
|
||||
m_insertIndexBlockLink.CommandText = @"INSERT INTO ""IndexBlockLink"" (""IndexVolumeID"", ""BlockVolumeID"") VALUES (?, ?)";
|
||||
m_insertIndexBlockLink.AddParameters(2);
|
||||
}
|
||||
|
||||
internal void SetResult(BasicResults result)
|
||||
@@ -798,6 +803,32 @@ namespace Duplicati.Library.Main.Database
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a timestamped backup operation to correctly associate the fileset with the time it was created.
|
||||
/// </summary>
|
||||
/// <param name="volumeid">The ID of the fileset volume to update</param>
|
||||
/// <param name="timestamp">The timestamp of the operation to create</param>
|
||||
/// <param name="transaction">An optional external transaction</param>
|
||||
public virtual long CreateFileset(long volumeid, DateTime timestamp, System.Data.IDbTransaction transaction = null)
|
||||
{
|
||||
using (var cmd = m_connection.CreateCommand())
|
||||
using (var tr = new TemporaryTransactionWrapper(m_connection, transaction))
|
||||
{
|
||||
cmd.Transaction = tr.Parent;
|
||||
var id = Convert.ToInt64(cmd.ExecuteScalar(@"INSERT INTO ""Fileset"" (""OperationID"", ""Timestamp"", ""VolumeID"") VALUES (?, ?, ?); SELECT last_insert_rowid();", m_operationid, NormalizeDateTime(timestamp), volumeid));
|
||||
tr.Commit();
|
||||
return id;
|
||||
}
|
||||
}
|
||||
|
||||
public void AddIndexBlockLink(long indexVolumeID, long blockVolumeID, System.Data.IDbTransaction transaction)
|
||||
{
|
||||
m_insertIndexBlockLink.Transaction = transaction;
|
||||
m_insertIndexBlockLink.SetParameterValue(0, indexVolumeID);
|
||||
m_insertIndexBlockLink.SetParameterValue(1, blockVolumeID);
|
||||
m_insertIndexBlockLink.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
public virtual void Dispose()
|
||||
{
|
||||
if (IsDisposed)
|
||||
|
||||
@@ -0,0 +1,688 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
|
||||
namespace Duplicati.Library.Main.Database
|
||||
{
|
||||
internal partial class LocalRecreateDatabase : LocalRestoreDatabase
|
||||
{
|
||||
private System.Data.IDbCommand m_insertFileCommand;
|
||||
private System.Data.IDbCommand m_insertFilesetEntryCommand;
|
||||
private System.Data.IDbCommand m_insertMetadatasetCommand;
|
||||
private System.Data.IDbCommand m_insertBlocksetCommand;
|
||||
private System.Data.IDbCommand m_insertBlocklistHashCommand;
|
||||
private System.Data.IDbCommand m_updateBlockVolumeCommand;
|
||||
private System.Data.IDbCommand m_insertBlockset;
|
||||
private System.Data.IDbCommand m_findBlocksetCommand;
|
||||
private System.Data.IDbCommand m_findMetadatasetCommand;
|
||||
private System.Data.IDbCommand m_findFilesetCommand;
|
||||
private System.Data.IDbCommand m_findblocklisthashCommand;
|
||||
private System.Data.IDbCommand m_findHashBlockCommand;
|
||||
private System.Data.IDbCommand m_insertBlockCommand;
|
||||
private System.Data.IDbCommand m_insertDuplicateBlockCommand;
|
||||
|
||||
private HashDatabaseProtector<string> m_blockListHashLookup;
|
||||
private HashDatabaseProtector<string, long> m_blockHashLookup;
|
||||
private HashDatabaseProtector<string, long> m_fileHashLookup;
|
||||
private HashDatabaseProtector<string, long> m_metadataLookup;
|
||||
private HashDatabaseProtector<Tuple<string, long,long>, long> m_filesetLookup;
|
||||
|
||||
private string m_tempblocklist;
|
||||
|
||||
/// <summary>
|
||||
/// A lookup table that prevents multiple downloads of the same volume
|
||||
/// </summary>
|
||||
private Dictionary<long, long> m_proccessedVolumes;
|
||||
|
||||
// SQL that finds index and block size for all blocklist hashes, based on the temporary hash list
|
||||
private const string SELECT_BLOCKLIST_ENTRIES =
|
||||
@"SELECT ""B"".""Length"", ""A"".""BlocklistHash"", ""A"".""BlockHash"", ""C"".""Index"", ""A"".""Index"" AS ""BlocIndex"", ""B"".""Length""/{0} AS ""LastBlock"", ""B"".""Length"" - (""B"".""Length""/{0})*{0} AS ""LastBlockSize"", " +
|
||||
@" CASE WHEN ""A"".""Index"" + (""C"".""Index"" * ({0}/{1})) = ""B"".""Length""/{0} THEN ""B"".""Length"" - (""B"".""Length""/{0})*{0} ELSE {0} END AS ""BlockSize"", " +
|
||||
@" ""A"".""Index"" + (""C"".""Index"" * ({0}/{1})) AS ""FullIndex"" " +
|
||||
@" FROM ""Blockset"" B, ""BlocklistHash"" C, ""{2}"" A " +
|
||||
@" WHERE ""B"".""ID"" = ""C"".""BlocksetID"" AND ""A"".""BlockListHash"" = ""C"".""Hash"" " +
|
||||
@" ORDER BY ""A"".""BlockListHash"", ""A"".""Index"" ";
|
||||
|
||||
public LocalRecreateDatabase(LocalDatabase parentdb, Options options)
|
||||
: base(parentdb, options.Blocksize)
|
||||
{
|
||||
m_tempblocklist = "TempBlocklist-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
|
||||
|
||||
using (var cmd = m_connection.CreateCommand())
|
||||
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)", m_tempblocklist));
|
||||
|
||||
m_insertFileCommand = m_connection.CreateCommand();
|
||||
m_insertFilesetEntryCommand = m_connection.CreateCommand();
|
||||
m_insertMetadatasetCommand = m_connection.CreateCommand();
|
||||
m_insertBlocksetCommand = m_connection.CreateCommand();
|
||||
m_insertBlocklistHashCommand = m_connection.CreateCommand();
|
||||
m_updateBlockVolumeCommand = m_connection.CreateCommand();
|
||||
m_insertBlockset = m_connection.CreateCommand();
|
||||
m_findBlocksetCommand = m_connection.CreateCommand();
|
||||
m_findMetadatasetCommand = m_connection.CreateCommand();
|
||||
m_findFilesetCommand = m_connection.CreateCommand();
|
||||
m_findblocklisthashCommand = m_connection.CreateCommand();
|
||||
m_findHashBlockCommand = m_connection.CreateCommand();
|
||||
m_insertBlockCommand = m_connection.CreateCommand();
|
||||
m_insertDuplicateBlockCommand = m_connection.CreateCommand();
|
||||
|
||||
m_insertFileCommand.CommandText = @"INSERT INTO ""File"" (""Path"", ""BlocksetID"", ""MetadataID"") VALUES (?,?,?); SELECT last_insert_rowid();";
|
||||
m_insertFileCommand.AddParameters(3);
|
||||
|
||||
m_insertFilesetEntryCommand.CommandText = @"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Scantime"") VALUES (?,?,?)";
|
||||
m_insertFilesetEntryCommand.AddParameters(3);
|
||||
|
||||
m_insertMetadatasetCommand.CommandText = @"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (?); SELECT last_insert_rowid();";
|
||||
m_insertMetadatasetCommand.AddParameters(1);
|
||||
|
||||
m_insertBlocksetCommand.CommandText = @"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (?,?); SELECT last_insert_rowid();";
|
||||
m_insertBlocksetCommand.AddParameters(2);
|
||||
|
||||
m_insertBlocklistHashCommand.CommandText = @"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (?,?,?)";
|
||||
m_insertBlocklistHashCommand.AddParameters(3);
|
||||
|
||||
m_updateBlockVolumeCommand.CommandText = @"UPDATE ""Block"" SET ""VolumeID"" = ? WHERE ""Hash"" = ? AND ""Size"" = ?";
|
||||
m_updateBlockVolumeCommand.AddParameters(3);
|
||||
|
||||
m_insertBlockset.CommandText = string.Format(@"INSERT INTO ""{0}"" (""BlocklistHash"", ""BlockHash"", ""Index"") VALUES (?,?,?) ", m_tempblocklist);
|
||||
m_insertBlockset.AddParameters(3);
|
||||
|
||||
m_findBlocksetCommand.CommandText = @"SELECT ""ID"" FROM ""Blockset"" WHERE ""Size"" = ? AND ""FullHash"" = ? ";
|
||||
m_findBlocksetCommand.AddParameters(2);
|
||||
|
||||
m_findMetadatasetCommand.CommandText = @"SELECT ""Metadataset"".""ID"" FROM ""Metadataset"",""BlocksetEntry"",""Block"" WHERE ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""Block"".""Hash"" = ? AND ""Block"".""Size"" = ? ";
|
||||
m_findMetadatasetCommand.AddParameters(2);
|
||||
|
||||
m_findFilesetCommand.CommandText = @"SELECT ""ID"" FROM ""File"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? ";
|
||||
m_findFilesetCommand.AddParameters(3);
|
||||
|
||||
m_findblocklisthashCommand.CommandText = string.Format(@"SELECT DISTINCT ""BlockListHash"" FROM ""{0}"" WHERE ""BlockListHash"" = ? ", m_tempblocklist);
|
||||
m_findblocklisthashCommand.AddParameters(1);
|
||||
|
||||
m_findHashBlockCommand.CommandText = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ? ";
|
||||
m_findHashBlockCommand.AddParameters(2);
|
||||
|
||||
m_insertBlockCommand.CommandText = @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") VALUES (?,?,?)";
|
||||
m_insertBlockCommand.AddParameters(3);
|
||||
|
||||
m_insertDuplicateBlockCommand.CommandText = @"INSERT INTO ""DuplicateBlock"" (""Hash"", ""Size"", ""VolumeID"") VALUE (?,?,?)";
|
||||
m_insertDuplicateBlockCommand.AddParameters(3);
|
||||
|
||||
if (options.BlockHashLookupMemory > 0)
|
||||
{
|
||||
m_blockHashLookup = new HashDatabaseProtector<string, long>(LocalBackupDatabase.HASH_GUESS_SIZE, (ulong)options.BlockHashLookupMemory/2);
|
||||
m_blockListHashLookup = new HashDatabaseProtector<string>(LocalBackupDatabase.HASH_GUESS_SIZE, (ulong)options.BlockHashLookupMemory/2);
|
||||
}
|
||||
if (options.FileHashLookupMemory > 0)
|
||||
m_fileHashLookup = new HashDatabaseProtector<string, long>(LocalBackupDatabase.HASH_GUESS_SIZE, (ulong)options.FileHashLookupMemory);
|
||||
if (options.MetadataHashMemory > 0)
|
||||
m_metadataLookup = new HashDatabaseProtector<string, long>(LocalBackupDatabase.HASH_GUESS_SIZE, (ulong)options.MetadataHashMemory);
|
||||
if (options.FilePathMemory > 0)
|
||||
m_filesetLookup = new HashDatabaseProtector<Tuple<string, long, long>, long>(LocalBackupDatabase.PATH_STRING_GUESS_SIZE, (ulong)options.FilePathMemory);
|
||||
}
|
||||
|
||||
public void FindMissingBlocklistHashes(long hashsize, System.Data.IDbTransaction transaction)
|
||||
{
|
||||
using(var cmd = m_connection.CreateCommand())
|
||||
{
|
||||
cmd.Transaction = transaction;
|
||||
|
||||
//Update all small blocklists and matching blocks
|
||||
var selectSmallBlocks = string.Format(@"SELECT ""Blockset"".""Fullhash"", ""Blockset"".""Length"" FROM ""Blockset"" WHERE ""Blockset"".""Length"" <= {0}", m_blocksize);
|
||||
|
||||
var selectBlockHashes = string.Format(
|
||||
@"SELECT ""BlockHash"" AS ""FullHash"", ""BlockSize"" AS ""Length"" FROM ( " +
|
||||
SELECT_BLOCKLIST_ENTRIES +
|
||||
@" )",
|
||||
m_blocksize,
|
||||
hashsize,
|
||||
m_tempblocklist
|
||||
);
|
||||
|
||||
var selectAllBlocks = @"SELECT DISTINCT ""FullHash"", ""Length"" FROM (" + selectBlockHashes + " UNION " + selectSmallBlocks + " )";
|
||||
|
||||
var selectNewBlocks = string.Format(
|
||||
@"SELECT ""FullHash"" AS ""Hash"", ""Length"" AS ""Size"", -1 AS ""VolumeID"" " +
|
||||
@" FROM (SELECT ""A"".""FullHash"", ""A"".""Length"", CASE WHEN ""B"".""Hash"" IS NULL THEN '' ELSE ""B"".""Hash"" END AS ""Hash"", CASE WHEN ""B"".""Size"" is NULL THEN -1 ELSE ""B"".""Size"" END AS ""Size"" FROM ({0}) A" +
|
||||
@" LEFT OUTER JOIN ""Block"" B ON (""B"".""Hash"" || ':' || ""B"".""Size"") = (""A"".""FullHash"" || ':' || ""A"".""Length"") )" +
|
||||
@" WHERE ""FullHash"" != ""Hash"" AND ""Length"" != ""Size"" ",
|
||||
selectAllBlocks
|
||||
);
|
||||
|
||||
var insertBlocksCommand =
|
||||
@"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") " +
|
||||
selectNewBlocks;
|
||||
|
||||
// Insert all known blocks into block table with volumeid = -1
|
||||
cmd.ExecuteNonQuery(insertBlocksCommand);
|
||||
|
||||
// Update the cache with new wblocks
|
||||
if (m_blockHashLookup != null)
|
||||
{
|
||||
using(var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Hash"" FROM ""Block"" WHERE ""VolumeID"" = -1 "))
|
||||
while(rd.Read())
|
||||
{
|
||||
var hash = rd.GetValue(0).ToString();
|
||||
m_blockHashLookup.Add(HashPrefixLookup.DecodeBase64Hash(hash), hash, -1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
var selectBlocklistBlocksetEntries = string.Format(
|
||||
@"SELECT ""E"".""BlocksetID"" AS ""BlocksetID"", ""D"".""FullIndex"" AS ""Index"", ""F"".""ID"" AS ""BlockID"" FROM ( " +
|
||||
SELECT_BLOCKLIST_ENTRIES +
|
||||
@") D, ""BlocklistHash"" E, ""Block"" F WHERE ""D"".""BlocklistHash"" = ""E"".""Hash"" AND ""D"".""Blockhash"" = ""F"".""Hash"" AND ""D"".""BlockSize"" = ""F"".""Size"" ",
|
||||
m_blocksize,
|
||||
hashsize,
|
||||
m_tempblocklist
|
||||
);
|
||||
|
||||
var selectBlocksetEntries = string.Format(
|
||||
@"SELECT ""Blockset"".""ID"" AS ""BlocksetID"", 0 AS ""Index"", ""Block"".""ID"" AS ""BlockID"" FROM ""Blockset"", ""Block"" WHERE ""Blockset"".""Fullhash"" = ""Block"".""Hash"" AND ""Blockset"".""Length"" <= {0} ",
|
||||
m_blocksize
|
||||
);
|
||||
|
||||
var selectAllBlocksetEntries =
|
||||
selectBlocklistBlocksetEntries +
|
||||
@" UNION " +
|
||||
selectBlocksetEntries;
|
||||
|
||||
var selectFiltered =
|
||||
@"SELECT DISTINCT ""BlocksetID"", ""Index"", ""BlockID"" FROM (" +
|
||||
selectAllBlocksetEntries +
|
||||
@") A WHERE (""A"".""BlocksetID"" || ':' || ""A"".""Index"") NOT IN (SELECT (""BlocksetID"" || ':' || ""Index"") FROM ""BlocksetEntry"" )";
|
||||
|
||||
var insertBlocksetEntriesCommand =
|
||||
@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") " + selectFiltered;
|
||||
|
||||
cmd.ExecuteNonQuery(insertBlocksetEntriesCommand);
|
||||
}
|
||||
}
|
||||
|
||||
public void AddDirectoryEntry(long filesetid, string path, DateTime time, string metahash, long metahashsize, System.Data.IDbTransaction transaction)
|
||||
{
|
||||
AddEntry(FilelistEntryType.Folder, filesetid, path, time, FOLDER_BLOCKSET_ID, metahash, metahashsize, transaction);
|
||||
}
|
||||
|
||||
public void AddSymlinkEntry(long filesetid, string path, DateTime time, string metahash, long metahashsize, System.Data.IDbTransaction transaction)
|
||||
{
|
||||
AddEntry(FilelistEntryType.Symlink, filesetid, path, time, SYMLINK_BLOCKSET_ID, metahash, metahashsize, transaction);
|
||||
}
|
||||
|
||||
public void AddFileEntry(long filesetid, string path, DateTime time, long blocksetid, string metahash, long metahashsize, System.Data.IDbTransaction transaction)
|
||||
{
|
||||
AddEntry(FilelistEntryType.File , filesetid, path, time, blocksetid, metahash, metahashsize, transaction);
|
||||
}
|
||||
|
||||
private void AddEntry(FilelistEntryType type, long filesetid, string path, DateTime time, long blocksetid, string metahash, long metahashsize, System.Data.IDbTransaction transaction)
|
||||
{
|
||||
var fileid = -1L;
|
||||
var metadataid = AddMetadataset(metahash, metahashsize, transaction);
|
||||
|
||||
var hashdata = (ulong)blocksetid ^ (ulong)metadataid ^ (ulong)path.GetHashCode();
|
||||
var tp = new Tuple<string, long, long>(path, blocksetid, metadataid);
|
||||
|
||||
if (m_filesetLookup != null)
|
||||
{
|
||||
switch (m_filesetLookup.HasValue(hashdata, tp, out fileid))
|
||||
{
|
||||
case HashLookupResult.Found:
|
||||
break;
|
||||
case HashLookupResult.NotFound:
|
||||
fileid = -1;
|
||||
break;
|
||||
case HashLookupResult.Uncertain:
|
||||
m_findFilesetCommand.Transaction = transaction;
|
||||
m_findFilesetCommand.SetParameterValue(0, path);
|
||||
m_findFilesetCommand.SetParameterValue(1, blocksetid);
|
||||
m_findFilesetCommand.SetParameterValue(2, metadataid);
|
||||
var r = m_findFilesetCommand.ExecuteScalar();
|
||||
if (r == null || r == DBNull.Value)
|
||||
{
|
||||
m_filesetLookup.PositiveMisses++;
|
||||
fileid = -1;
|
||||
}
|
||||
else
|
||||
{
|
||||
fileid = Convert.ToInt64(r);
|
||||
m_filesetLookup.NegativeMisses++;
|
||||
m_filesetLookup.Add(hashdata, tp, fileid);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
m_findFilesetCommand.Transaction = transaction;
|
||||
m_findFilesetCommand.SetParameterValue(0, path);
|
||||
m_findFilesetCommand.SetParameterValue(1, blocksetid);
|
||||
m_findFilesetCommand.SetParameterValue(2, metadataid);
|
||||
var r = m_findFilesetCommand.ExecuteScalar();
|
||||
if (r != null && r != DBNull.Value)
|
||||
fileid = Convert.ToInt64(r);
|
||||
}
|
||||
|
||||
if (fileid < 0)
|
||||
{
|
||||
m_insertFileCommand.Transaction = transaction;
|
||||
m_insertFileCommand.SetParameterValue(0, path);
|
||||
m_insertFileCommand.SetParameterValue(1, blocksetid);
|
||||
m_insertFileCommand.SetParameterValue(2, metadataid);
|
||||
fileid = Convert.ToInt64(m_insertFileCommand.ExecuteScalar());
|
||||
if (m_filesetLookup != null)
|
||||
m_filesetLookup.Add(hashdata, tp, fileid);
|
||||
}
|
||||
|
||||
m_insertFilesetEntryCommand.Transaction = transaction;
|
||||
m_insertFilesetEntryCommand.SetParameterValue(0, filesetid);
|
||||
m_insertFilesetEntryCommand.SetParameterValue(1, fileid);
|
||||
m_insertFilesetEntryCommand.SetParameterValue(2, time);
|
||||
m_insertFilesetEntryCommand.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
private long AddMetadataset(string metahash, long metahashsize, System.Data.IDbTransaction transaction)
|
||||
{
|
||||
var metadataid = -1L;
|
||||
if (metahash == null)
|
||||
return metadataid;
|
||||
|
||||
var hashdata = HashPrefixLookup.DecodeBase64Hash(metahash);
|
||||
if (m_metadataLookup != null)
|
||||
{
|
||||
switch (m_metadataLookup.HasValue(hashdata, metahash, out metadataid))
|
||||
{
|
||||
case HashLookupResult.Found:
|
||||
return metadataid;
|
||||
case HashLookupResult.NotFound:
|
||||
metadataid = -1;
|
||||
break;
|
||||
case HashLookupResult.Uncertain:
|
||||
m_findMetadatasetCommand.Transaction = transaction;
|
||||
m_findMetadatasetCommand.SetParameterValue(0, metahash);
|
||||
m_findMetadatasetCommand.SetParameterValue(1, metahashsize);
|
||||
var r = m_findMetadatasetCommand.ExecuteScalar();
|
||||
if (r == null || r == DBNull.Value)
|
||||
{
|
||||
metadataid = -1;
|
||||
m_metadataLookup.PositiveMisses++;
|
||||
}
|
||||
else
|
||||
{
|
||||
metadataid = Convert.ToInt64(r);
|
||||
m_metadataLookup.NegativeMisses++;
|
||||
m_metadataLookup.Add(hashdata, metahash, metadataid);
|
||||
return metadataid;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
m_findMetadatasetCommand.Transaction = transaction;
|
||||
m_findMetadatasetCommand.SetParameterValue(0, metahash);
|
||||
m_findMetadatasetCommand.SetParameterValue(1, metahashsize);
|
||||
var r = m_findMetadatasetCommand.ExecuteScalar();
|
||||
if (r != null && r != DBNull.Value)
|
||||
return Convert.ToInt64(r);
|
||||
}
|
||||
|
||||
var blocksetid = AddBlockset(metahash, metahashsize, null, transaction);
|
||||
|
||||
m_insertMetadatasetCommand.Transaction = transaction;
|
||||
m_insertMetadatasetCommand.SetParameterValue(0, blocksetid);
|
||||
metadataid = Convert.ToInt64(m_insertMetadatasetCommand.ExecuteScalar());
|
||||
|
||||
if (m_metadataLookup != null)
|
||||
m_metadataLookup.Add(hashdata, metahash, metadataid);
|
||||
|
||||
return metadataid;
|
||||
}
|
||||
|
||||
public long AddBlockset(string fullhash, long size, IEnumerable<string> blocklisthashes, System.Data.IDbTransaction transaction)
|
||||
{
|
||||
var blocksetid = -1L;
|
||||
var hashdata = HashPrefixLookup.DecodeBase64Hash(fullhash);
|
||||
if (m_fileHashLookup != null)
|
||||
{
|
||||
switch (m_fileHashLookup.HasValue(hashdata, fullhash, out blocksetid))
|
||||
{
|
||||
case HashLookupResult.Found:
|
||||
return blocksetid;
|
||||
case HashLookupResult.NotFound:
|
||||
blocksetid = -1;
|
||||
break;
|
||||
case HashLookupResult.Uncertain:
|
||||
m_findBlocksetCommand.Transaction = transaction;
|
||||
m_findBlocksetCommand.SetParameterValue(0, size);
|
||||
m_findBlocksetCommand.SetParameterValue(1, fullhash);
|
||||
var r = m_findBlocksetCommand.ExecuteScalar();
|
||||
if (r == null || r == DBNull.Value)
|
||||
{
|
||||
m_fileHashLookup.PositiveMisses++;
|
||||
blocksetid = -1;
|
||||
}
|
||||
else
|
||||
{
|
||||
blocksetid = Convert.ToInt64(r);
|
||||
m_fileHashLookup.NegativeMisses++;
|
||||
m_fileHashLookup.Add(hashdata, fullhash, blocksetid);
|
||||
return blocksetid;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
m_findBlocksetCommand.Transaction = transaction;
|
||||
m_findBlocksetCommand.SetParameterValue(0, size);
|
||||
m_findBlocksetCommand.SetParameterValue(1, fullhash);
|
||||
var r = m_findBlocksetCommand.ExecuteScalar();
|
||||
if (r != null && r != DBNull.Value)
|
||||
return Convert.ToInt64(r);
|
||||
}
|
||||
|
||||
m_insertBlocksetCommand.Transaction = transaction;
|
||||
m_insertBlocksetCommand.SetParameterValue(0, size);
|
||||
m_insertBlocksetCommand.SetParameterValue(1, fullhash);
|
||||
blocksetid = Convert.ToInt64(m_insertBlocksetCommand.ExecuteScalar());
|
||||
|
||||
if (m_fileHashLookup != null)
|
||||
m_fileHashLookup.Add(hashdata, fullhash, blocksetid);
|
||||
|
||||
if (blocklisthashes != null)
|
||||
{
|
||||
var index = 0L;
|
||||
m_insertBlocklistHashCommand.Transaction = transaction;
|
||||
m_insertBlocklistHashCommand.SetParameterValue(0, blocksetid);
|
||||
foreach(var hash in blocklisthashes)
|
||||
{
|
||||
if (!string.IsNullOrEmpty(hash))
|
||||
{
|
||||
m_insertBlocklistHashCommand.SetParameterValue(1, index++);
|
||||
m_insertBlocklistHashCommand.SetParameterValue(2, hash);
|
||||
m_insertBlocklistHashCommand.ExecuteNonQuery();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return blocksetid;
|
||||
}
|
||||
|
||||
public void UpdateBlock(string hash, long size, long volumeID, System.Data.IDbTransaction transaction)
|
||||
{
|
||||
var hashdata = HashPrefixLookup.DecodeBase64Hash(hash);
|
||||
var currentVolumeId = -2L;
|
||||
if (m_blockHashLookup != null)
|
||||
{
|
||||
switch (m_blockHashLookup.HasValue(hashdata, hash, out currentVolumeId))
|
||||
{
|
||||
case HashLookupResult.Found:
|
||||
break;
|
||||
case HashLookupResult.NotFound:
|
||||
currentVolumeId = -2;
|
||||
break;
|
||||
case HashLookupResult.Uncertain:
|
||||
m_findHashBlockCommand.Transaction = transaction;
|
||||
m_findHashBlockCommand.SetParameterValue(0, hash);
|
||||
m_findHashBlockCommand.SetParameterValue(1, size);
|
||||
var r = m_findHashBlockCommand.ExecuteScalar();
|
||||
if (r == null || r == DBNull.Value)
|
||||
{
|
||||
m_blockHashLookup.PositiveMisses++;
|
||||
currentVolumeId = -2;
|
||||
}
|
||||
else
|
||||
{
|
||||
m_blockHashLookup.NegativeMisses++;
|
||||
currentVolumeId = Convert.ToInt64(r);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
m_findHashBlockCommand.Transaction = transaction;
|
||||
m_findHashBlockCommand.SetParameterValue(0, hash);
|
||||
m_findHashBlockCommand.SetParameterValue(1, size);
|
||||
var r = m_findHashBlockCommand.ExecuteScalar();
|
||||
if (r != null && r != DBNull.Value)
|
||||
currentVolumeId = Convert.ToInt64(r);
|
||||
}
|
||||
|
||||
if (currentVolumeId == volumeID)
|
||||
return;
|
||||
|
||||
if (currentVolumeId == -2)
|
||||
{
|
||||
//Insert
|
||||
m_insertBlockCommand.Transaction = transaction;
|
||||
m_insertBlockCommand.SetParameterValue(0, hash);
|
||||
m_insertBlockCommand.SetParameterValue(1, size);
|
||||
m_insertBlockCommand.SetParameterValue(2, volumeID);
|
||||
m_insertBlockCommand.ExecuteNonQuery();
|
||||
|
||||
if (m_blockHashLookup != null)
|
||||
m_blockHashLookup.Add(hashdata, hash, volumeID);
|
||||
}
|
||||
else if (currentVolumeId == -1)
|
||||
{
|
||||
//Update
|
||||
m_updateBlockVolumeCommand.Transaction = transaction;
|
||||
m_updateBlockVolumeCommand.SetParameterValue(0, volumeID);
|
||||
m_updateBlockVolumeCommand.SetParameterValue(1, hash);
|
||||
m_updateBlockVolumeCommand.SetParameterValue(2, size);
|
||||
var c = m_updateBlockVolumeCommand.ExecuteNonQuery();
|
||||
if (c != 1)
|
||||
throw new Exception(string.Format("Failed to update table, found {0} entries for key {1} with size {2}", c ,hash, size));
|
||||
|
||||
if (m_blockHashLookup != null)
|
||||
m_blockHashLookup.Add(hashdata, hash, volumeID);
|
||||
}
|
||||
else
|
||||
{
|
||||
m_insertDuplicateBlockCommand.Transaction = transaction;
|
||||
m_insertDuplicateBlockCommand.SetParameterValue(0, hash);
|
||||
m_insertDuplicateBlockCommand.SetParameterValue(1, size);
|
||||
m_insertDuplicateBlockCommand.SetParameterValue(2, volumeID);
|
||||
m_insertDuplicateBlockCommand.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public void UpdateBlockset(string hash, IEnumerable<string> blocklisthashes, long hashsize, System.Data.IDbTransaction transaction)
|
||||
{
|
||||
var hashdata = HashPrefixLookup.DecodeBase64Hash(hash);
|
||||
if (m_blockListHashLookup != null)
|
||||
{
|
||||
switch (m_blockListHashLookup.HasValue(hashdata, hash))
|
||||
{
|
||||
case HashLookupResult.Found:
|
||||
return;
|
||||
case HashLookupResult.NotFound:
|
||||
break;
|
||||
case HashLookupResult.Uncertain:
|
||||
m_findblocklisthashCommand.Transaction = transaction;
|
||||
m_findblocklisthashCommand.SetParameterValue(0, hash);
|
||||
var r = m_findblocklisthashCommand.ExecuteScalar();
|
||||
if (r != null && r != DBNull.Value)
|
||||
{
|
||||
m_blockListHashLookup.Add(hashdata, hash);
|
||||
return;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
m_findblocklisthashCommand.Transaction = transaction;
|
||||
m_findblocklisthashCommand.SetParameterValue(0, hash);
|
||||
var r = m_findblocklisthashCommand.ExecuteScalar();
|
||||
if (r != null && r != DBNull.Value)
|
||||
return;
|
||||
}
|
||||
|
||||
if (m_blockListHashLookup != null)
|
||||
m_blockListHashLookup.Add(hashdata, hash);
|
||||
|
||||
m_insertBlockset.Transaction = transaction;
|
||||
m_insertBlockset.SetParameterValue(0, hash);
|
||||
|
||||
var index = 0L;
|
||||
|
||||
foreach(var s in blocklisthashes)
|
||||
{
|
||||
m_insertBlockset.SetParameterValue(1, s);
|
||||
m_insertBlockset.SetParameterValue(2, index++);
|
||||
m_insertBlockset.ExecuteNonQuery();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public IEnumerable<string> GetBlockLists(long volumeid)
|
||||
{
|
||||
using(var cmd = m_connection.CreateCommand())
|
||||
{
|
||||
cmd.CommandText = string.Format(@"SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"", ""Block"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" AND ""Block"".""VolumeID"" = ?");
|
||||
cmd.AddParameter(volumeid);
|
||||
|
||||
using(var rd = cmd.ExecuteReader())
|
||||
while (rd.Read())
|
||||
yield return rd.GetValue(0).ToString();
|
||||
}
|
||||
}
|
||||
|
||||
public IEnumerable<IRemoteVolume> GetMissingBlockListVolumes(int passNo)
|
||||
{
|
||||
using(var cmd = m_connection.CreateCommand())
|
||||
{
|
||||
var selectCommand = @"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"", ""RemoteVolume"".""ID"" FROM ""RemoteVolume""";
|
||||
|
||||
var missingBlocklistEntries =
|
||||
@"SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"" LEFT OUTER JOIN ""BlocksetEntry"" ON ""BlocksetEntry"".""Index"" = ""BlocklistHash"".""Index"" AND ""BlocksetEntry"".""BlocksetID"" = ""BlocklistHash"".""BlocksetID"" WHERE ""BlocksetEntry"".""BlocksetID"" IS NULL";
|
||||
|
||||
var missingBlockInfo =
|
||||
@"SELECT ""VolumeID"" FROM ""Block"" WHERE ""VolumeID"" < 0 ";
|
||||
|
||||
var missingBlocklistVolumes = string.Format(
|
||||
@"SELECT ""VolumeID"" FROM ""Block"", (" +
|
||||
missingBlocklistEntries +
|
||||
@") A WHERE ""A"".""Hash"" = ""Block"".""Hash"" "
|
||||
);
|
||||
|
||||
var countMissingInformation = string.Format(
|
||||
@"SELECT COUNT(*) FROM (SELECT DISTINCT ""VolumeID"" FROM ({0} UNION {1}))",
|
||||
missingBlockInfo,
|
||||
missingBlocklistEntries);
|
||||
|
||||
if (passNo == 0)
|
||||
{
|
||||
// On the first pass, we select all the volumes we know we need,
|
||||
// which may be an empty list
|
||||
cmd.CommandText = string.Format(selectCommand + @" WHERE ""ID"" IN ({0})", missingBlocklistVolumes);
|
||||
|
||||
// Reset the list
|
||||
m_proccessedVolumes = new Dictionary<long, long>();
|
||||
}
|
||||
else
|
||||
{
|
||||
//On anything but the first pass, we check if we are done
|
||||
var r = cmd.ExecuteScalar(countMissingInformation);
|
||||
if (r == null || r == DBNull.Value || Convert.ToInt64(r) == 0)
|
||||
yield break;
|
||||
|
||||
if (passNo == 1)
|
||||
{
|
||||
// On the second pass, we select all volumes that are not mentioned in the db
|
||||
|
||||
var mentionedVolumes =
|
||||
@"SELECT DISTINCT ""VolumeID"" FROM ""Block"" ";
|
||||
|
||||
cmd.CommandText = string.Format(selectCommand + @" WHERE ""ID"" NOT IN ({0}) AND ""Type"" = ? ", mentionedVolumes);
|
||||
cmd.AddParameter(RemoteVolumeType.Blocks.ToString());
|
||||
}
|
||||
else
|
||||
{
|
||||
// On the final pass, we select all volumes
|
||||
// the filter will ensure that we do not download anything twice
|
||||
cmd.CommandText = selectCommand + @" WHERE ""Type"" = ?";
|
||||
cmd.AddParameter(RemoteVolumeType.Blocks.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
using(var rd = cmd.ExecuteReader())
|
||||
{
|
||||
object[] r = new object[4];
|
||||
while (rd.Read())
|
||||
{
|
||||
rd.GetValues(r);
|
||||
|
||||
var volumeID = Convert.ToInt64(r[3]);
|
||||
|
||||
// Guard against multiple downloads of the same file
|
||||
if (!m_proccessedVolumes.ContainsKey(volumeID))
|
||||
{
|
||||
m_proccessedVolumes.Add(volumeID, volumeID);
|
||||
|
||||
yield return new RemoteVolume(
|
||||
(r[0] == null || r[0] == DBNull.Value) ? null : r[0].ToString(),
|
||||
(r[1] == null || r[1] == DBNull.Value) ? null : r[1].ToString(),
|
||||
(r[2] == null || r[2] == DBNull.Value) ? -1 : Convert.ToInt64(r[2])
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
using (var cmd = m_connection.CreateCommand())
|
||||
{
|
||||
if (m_tempblocklist != null)
|
||||
try
|
||||
{
|
||||
cmd.CommandText = string.Format(@"DROP TABLE ""{0}""", m_tempblocklist);
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
finally { m_tempblocklist = null; }
|
||||
|
||||
}
|
||||
|
||||
foreach(var cmd in new IDisposable [] {
|
||||
m_insertFileCommand,
|
||||
m_insertFilesetEntryCommand,
|
||||
m_insertMetadatasetCommand,
|
||||
m_insertBlocksetCommand,
|
||||
m_insertBlocklistHashCommand,
|
||||
m_updateBlockVolumeCommand,
|
||||
m_insertBlockset,
|
||||
m_findBlocksetCommand,
|
||||
m_findMetadatasetCommand,
|
||||
m_findFilesetCommand,
|
||||
m_blockHashLookup,
|
||||
m_fileHashLookup,
|
||||
m_metadataLookup,
|
||||
m_filesetLookup,
|
||||
m_findblocklisthashCommand,
|
||||
m_findHashBlockCommand,
|
||||
m_insertBlockCommand,
|
||||
m_insertDuplicateBlockCommand
|
||||
})
|
||||
try
|
||||
{
|
||||
if (cmd != null)
|
||||
cmd.Dispose();
|
||||
}
|
||||
catch
|
||||
{
|
||||
}
|
||||
|
||||
base.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -967,83 +967,4 @@ namespace Duplicati.Library.Main.Database
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal partial class LocalBlocklistUpdateDatabase
|
||||
{
|
||||
private class BlocklistsEnumerable : IEnumerable<string>
|
||||
{
|
||||
private class BlocklistEnumerator : IEnumerator<string>
|
||||
{
|
||||
private System.Data.IDbConnection m_connection;
|
||||
private string m_temptable;
|
||||
private long m_volumeid;
|
||||
private System.Data.IDataReader m_reader;
|
||||
private System.Data.IDbCommand m_command;
|
||||
private string m_current;
|
||||
|
||||
public BlocklistEnumerator(System.Data.IDbConnection connection, string temptable, long volumeid)
|
||||
{
|
||||
m_connection = connection;
|
||||
m_temptable = temptable;
|
||||
m_volumeid = volumeid;
|
||||
this.Reset();
|
||||
}
|
||||
|
||||
public string Current { get { return m_current; } }
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (m_reader != null)
|
||||
try { m_reader.Dispose(); }
|
||||
finally { m_reader = null; }
|
||||
|
||||
if (m_command != null)
|
||||
try { m_command.Dispose(); }
|
||||
finally { m_command = null; }
|
||||
}
|
||||
|
||||
object System.Collections.IEnumerator.Current { get { return this.Current; } }
|
||||
|
||||
public bool MoveNext()
|
||||
{
|
||||
if (!m_reader.Read())
|
||||
{
|
||||
m_current = null;
|
||||
return false;
|
||||
}
|
||||
|
||||
var c = m_reader.GetValue(0);
|
||||
if (c == null || c == DBNull.Value)
|
||||
m_current = null;
|
||||
else
|
||||
m_current = c.ToString();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public void Reset()
|
||||
{
|
||||
this.Dispose();
|
||||
m_current = null;
|
||||
m_command = m_connection.CreateCommand();
|
||||
m_command.CommandText = string.Format(@"SELECT ""Hash"" FROM ""{0}"" WHERE ""VolumeID"" = ""{1}"" AND ""Restored"" = 0 ", m_temptable, m_volumeid);
|
||||
m_reader = m_command.ExecuteReader();
|
||||
}
|
||||
}
|
||||
|
||||
private System.Data.IDbConnection m_connection;
|
||||
private string m_temptable;
|
||||
private long m_volumeid;
|
||||
|
||||
public BlocklistsEnumerable(System.Data.IDbConnection connection, string temptable, long volumeid)
|
||||
{
|
||||
m_connection = connection;
|
||||
m_temptable = temptable;
|
||||
m_volumeid = volumeid;
|
||||
}
|
||||
|
||||
public IEnumerator<string> GetEnumerator() { return new BlocklistEnumerator(m_connection, m_temptable, m_volumeid); }
|
||||
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return this.GetEnumerator(); }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -66,7 +66,6 @@
|
||||
<Compile Include="Database\ILocalFileEntry.cs" />
|
||||
<Compile Include="Database\IRemoteVolume.cs" />
|
||||
<Compile Include="Database\LocalBackupDatabase.cs" />
|
||||
<Compile Include="Database\LocalBlocklistUpdateDatabase.cs" />
|
||||
<Compile Include="Database\LocalDatabase.cs" />
|
||||
<Compile Include="Database\LocalRestoreDatabase.cs" />
|
||||
<Compile Include="Database\RemoteVolume.cs" />
|
||||
@@ -115,6 +114,7 @@
|
||||
<Compile Include="Database\LocalListChangesDatabase.cs" />
|
||||
<Compile Include="Database\LocalTestDatabase.cs" />
|
||||
<Compile Include="Operation\TestHandler.cs" />
|
||||
<Compile Include="Database\LocalRecreateDatabase.cs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Utility\Duplicati.Library.Utility.csproj">
|
||||
|
||||
@@ -633,6 +633,11 @@ namespace Duplicati.Library.Main.Operation
|
||||
if (m_database.AddBlock(key, len, m_blockvolume.VolumeID, m_transaction))
|
||||
{
|
||||
m_blockvolume.AddBlock(key, data, len, hint);
|
||||
|
||||
//TODO: In theory a normal data block and blocklist block could be equal.
|
||||
// this would cause the index file to not contain all data,
|
||||
// if the data file is added before the blocklist data
|
||||
// ... highly theoretical ...
|
||||
if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full && isBlocklistData)
|
||||
m_indexvolume.WriteBlocklist(key, data, len);
|
||||
|
||||
|
||||
@@ -14,89 +14,89 @@ namespace Duplicati.Library.Main.Operation
|
||||
private RecreateDatabaseResults m_result;
|
||||
|
||||
public delegate IEnumerable<KeyValuePair<long, IParsedVolume>> NumberedFilterFilelistDelegate(IEnumerable<IParsedVolume> filelist);
|
||||
public delegate void BlockVolumePostProcessor(string volumename,BlockVolumeReader reader);
|
||||
public delegate void BlockVolumePostProcessor(string volumename,BlockVolumeReader reader);
|
||||
|
||||
public RecreateDatabaseHandler(string backendurl, Options options, RecreateDatabaseResults result)
|
||||
{
|
||||
m_options = options;
|
||||
m_backendurl = backendurl;
|
||||
public RecreateDatabaseHandler(string backendurl, Options options, RecreateDatabaseResults result)
|
||||
{
|
||||
m_options = options;
|
||||
m_backendurl = backendurl;
|
||||
m_result = result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Run the recreate procedure
|
||||
/// </summary>
|
||||
/// <param name="path">Path to the database that will be created</param>
|
||||
/// <param name="filelistfilter">A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist</param>
|
||||
/// <param name="filenamefilter">Filters the files in a filelist to prevent downloading unwanted data</param>
|
||||
/// <param name="blockprocessor">A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists</param>
|
||||
public void Run(string path, Library.Utility.IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null)
|
||||
{
|
||||
if (System.IO.File.Exists(path))
|
||||
throw new Exception(string.Format("Cannot recreate database because file already exists: {0}", path));
|
||||
}
|
||||
|
||||
using(var db = new LocalBlocklistUpdateDatabase(path, m_options.Blocksize))
|
||||
{
|
||||
/// <summary>
|
||||
/// Run the recreate procedure
|
||||
/// </summary>
|
||||
/// <param name="path">Path to the database that will be created</param>
|
||||
/// <param name="filelistfilter">A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist</param>
|
||||
/// <param name="filenamefilter">Filters the files in a filelist to prevent downloading unwanted data</param>
|
||||
/// <param name="blockprocessor">A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists</param>
|
||||
public void Run(string path, Library.Utility.IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null)
|
||||
{
|
||||
if (System.IO.File.Exists(path))
|
||||
throw new Exception(string.Format("Cannot recreate database because file already exists: {0}", path));
|
||||
|
||||
using(var db = new LocalDatabase(path, "Recreate"))
|
||||
{
|
||||
m_result.SetDatabase(db);
|
||||
DoRun(db, filter, filelistfilter, blockprocessor);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Run the recreate procedure
|
||||
/// </summary>
|
||||
/// <param name="path">Path to the database that will be created</param>
|
||||
/// <param name="filelistfilter">A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist</param>
|
||||
/// <param name="filenamefilter">Filters the files in a filelist to prevent downloading unwanted data</param>
|
||||
/// <param name="blockprocessor">A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists</param>
|
||||
DoRun(db, filter, filelistfilter, blockprocessor);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Run the recreate procedure
|
||||
/// </summary>
|
||||
/// <param name="path">Path to the database that will be created</param>
|
||||
/// <param name="filelistfilter">A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist</param>
|
||||
/// <param name="filenamefilter">Filters the files in a filelist to prevent downloading unwanted data</param>
|
||||
/// <param name="blockprocessor">A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists</param>
|
||||
internal void DoRun(LocalDatabase dbparent, Library.Utility.IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null)
|
||||
{
|
||||
var hashalg = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
|
||||
if (hashalg == null)
|
||||
throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.BlockHashAlgorithm));
|
||||
var hashalg = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
|
||||
if (hashalg == null)
|
||||
throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.BlockHashAlgorithm));
|
||||
var hashsize = hashalg.HashSize / 8;
|
||||
|
||||
//We build a local database in steps.
|
||||
using (var restoredb = new LocalBlocklistUpdateDatabase(dbparent, m_options.Blocksize))
|
||||
using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, restoredb))
|
||||
using(var restoredb = new LocalRecreateDatabase(dbparent, m_options))
|
||||
using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, restoredb))
|
||||
{
|
||||
var volumeIds = new Dictionary<string, long>();
|
||||
var volumeIds = new Dictionary<string, long>();
|
||||
|
||||
var rawlist = backend.List();
|
||||
|
||||
var rawlist = backend.List();
|
||||
|
||||
//First step is to examine the remote storage to see what
|
||||
// kind of data we can find
|
||||
var remotefiles =
|
||||
(from x in rawlist
|
||||
let n = VolumeBase.ParseFilename(x)
|
||||
where
|
||||
n != null
|
||||
&&
|
||||
n.Prefix == m_options.Prefix
|
||||
select n).ToArray(); //ToArray() ensures that we do not remote-request it multiple times
|
||||
(from x in rawlist
|
||||
let n = VolumeBase.ParseFilename(x)
|
||||
where
|
||||
n != null
|
||||
&&
|
||||
n.Prefix == m_options.Prefix
|
||||
select n).ToArray(); //ToArray() ensures that we do not remote-request it multiple times
|
||||
|
||||
if (remotefiles.Length == 0)
|
||||
{
|
||||
if (rawlist.Count == 0)
|
||||
throw new Exception("No files were found at the remote location, perhaps the target url is incorrect?");
|
||||
else
|
||||
{
|
||||
var tmp =
|
||||
(from x in rawlist
|
||||
let n = VolumeBase.ParseFilename(x)
|
||||
where
|
||||
n != null
|
||||
select n.Prefix).ToArray();
|
||||
|
||||
var types = tmp.Distinct().ToArray();
|
||||
if (tmp.Length == 0)
|
||||
throw new Exception(string.Format("Found {0} files at the remote storage, but none that could be parsed", rawlist.Count));
|
||||
else if (types.Length == 1)
|
||||
throw new Exception(string.Format("Found {0} parse-able files with the prefix {1}, did you forget to set the backup-prefix?", tmp.Length, types[0]));
|
||||
else
|
||||
throw new Exception(string.Format("Found {0} parse-able files (of {1} files) with different prefixes: {2}, did you forget to set the backup-prefix?", tmp.Length, rawlist.Count, string.Join(", ", types)));
|
||||
}
|
||||
}
|
||||
if (remotefiles.Length == 0)
|
||||
{
|
||||
if (rawlist.Count == 0)
|
||||
throw new Exception("No files were found at the remote location, perhaps the target url is incorrect?");
|
||||
else
|
||||
{
|
||||
var tmp =
|
||||
(from x in rawlist
|
||||
let n = VolumeBase.ParseFilename(x)
|
||||
where
|
||||
n != null
|
||||
select n.Prefix).ToArray();
|
||||
|
||||
var types = tmp.Distinct().ToArray();
|
||||
if (tmp.Length == 0)
|
||||
throw new Exception(string.Format("Found {0} files at the remote storage, but none that could be parsed", rawlist.Count));
|
||||
else if (types.Length == 1)
|
||||
throw new Exception(string.Format("Found {0} parse-able files with the prefix {1}, did you forget to set the backup-prefix?", tmp.Length, types[0]));
|
||||
else
|
||||
throw new Exception(string.Format("Found {0} parse-able files (of {1} files) with different prefixes: {2}, did you forget to set the backup-prefix?", tmp.Length, rawlist.Count, string.Join(", ", types)));
|
||||
}
|
||||
}
|
||||
|
||||
//Then we select the filelist we should work with,
|
||||
// and create the filelist table to fit
|
||||
@@ -109,125 +109,155 @@ namespace Duplicati.Library.Main.Operation
|
||||
if (filelistfilter != null)
|
||||
filelists = filelistfilter(filelists).Select(x => x.Value).ToArray();
|
||||
|
||||
using (var backupdb = new LocalBackupDatabase(restoredb, m_options))
|
||||
foreach(var fl in remotefiles)
|
||||
volumeIds[fl.File.Name] = restoredb.RegisterRemoteVolume(fl.File.Name, fl.FileType, RemoteVolumeState.Uploaded);
|
||||
|
||||
//Record all blocksets and files needed
|
||||
using(var tr = restoredb.BeginTransaction())
|
||||
{
|
||||
foreach (var fl in remotefiles)
|
||||
volumeIds[fl.File.Name] = backupdb.RegisterRemoteVolume(fl.File.Name, fl.FileType, RemoteVolumeState.Uploaded);
|
||||
|
||||
//We grab all index files, and update the block table
|
||||
using (var tr = restoredb.BeginTransaction())
|
||||
{
|
||||
var indexfiles =
|
||||
from n in remotefiles
|
||||
where n.FileType == RemoteVolumeType.Index
|
||||
select new RemoteVolume(n.File) as IRemoteVolume;
|
||||
|
||||
foreach (var sf in new AsyncDownloader(indexfiles.ToList(), backend))
|
||||
using (var tmpfile = sf.TempFile)
|
||||
var filelistWork = (from n in filelists orderby n.Time select new RemoteVolume(n.File) as IRemoteVolume).ToList();
|
||||
foreach(var entry in new AsyncDownloader(filelistWork, backend))
|
||||
try
|
||||
{
|
||||
if (sf.Hash != null && sf.Size > 0)
|
||||
backupdb.UpdateRemoteVolume(sf.Name, RemoteVolumeState.Verified, sf.Size, sf.Hash, tr);
|
||||
|
||||
using (var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(sf.Name), tmpfile, m_options, hashsize))
|
||||
using(var tmpfile = entry.TempFile)
|
||||
{
|
||||
Utility.VerifyParameters(restoredb, m_options);
|
||||
if (entry.Hash != null && entry.Size > 0)
|
||||
restoredb.UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, entry.Size, entry.Hash, tr);
|
||||
|
||||
//If there are blocklists in the index file, update the blocklists
|
||||
foreach (var b in svr.BlockLists)
|
||||
restoredb.UpdateBlocklist(b.Hash, b.Blocklist, hashsize, tr);
|
||||
|
||||
foreach (var a in svr.Volumes)
|
||||
var parsed = VolumeBase.ParseFilename(entry.Name);
|
||||
// Create timestamped operations based on the file timestamp
|
||||
var filesetid = restoredb.CreateFileset(volumeIds[entry.Name], parsed.Time, tr);
|
||||
using(var filelistreader = new FilesetVolumeReader(parsed.CompressionModule, tmpfile, m_options))
|
||||
foreach(var fe in filelistreader.Files.Where(x => Library.Utility.FilterExpression.Matches(filter, x.Path)))
|
||||
{
|
||||
try
|
||||
{
|
||||
if (fe.Type == FilelistEntryType.Folder)
|
||||
{
|
||||
restoredb.AddDirectoryEntry(filesetid, fe.Path, fe.Time, fe.Metahash, fe.Metahash == null ? -1 : fe.Metasize, tr);
|
||||
}
|
||||
else if (fe.Type == FilelistEntryType.File)
|
||||
{
|
||||
var blocksetid = restoredb.AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, tr);
|
||||
restoredb.AddFileEntry(filesetid, fe.Path, fe.Time, blocksetid, fe.Metahash, fe.Metahash == null ? -1 : fe.Metasize, tr);
|
||||
}
|
||||
else if (fe.Type == FilelistEntryType.Symlink)
|
||||
{
|
||||
restoredb.AddSymlinkEntry(filesetid, fe.Path, fe.Time, fe.Metahash, fe.Metahash == null ? -1 : fe.Metasize, tr);
|
||||
}
|
||||
else
|
||||
{
|
||||
m_result.AddWarning(string.Format("Skipping file-entry with unknown type {0}: {1} ", fe.Type, fe.Path), null);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
m_result.AddWarning(string.Format("Failed to process file-entry: {0}", fe.Path), ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
m_result.AddWarning(string.Format("Failed to process file: {0}", entry.Name), ex);
|
||||
}
|
||||
|
||||
using(new Logging.Timer("CommitUpdateFilesetFromRemote"))
|
||||
tr.Commit();
|
||||
}
|
||||
|
||||
//Grab all index files, and update the block table
|
||||
using(var tr = restoredb.BeginTransaction())
|
||||
{
|
||||
var indexfiles =
|
||||
from n in remotefiles
|
||||
where n.FileType == RemoteVolumeType.Index
|
||||
select new RemoteVolume(n.File) as IRemoteVolume;
|
||||
|
||||
foreach(var sf in new AsyncDownloader(indexfiles.ToList(), backend))
|
||||
try
|
||||
{
|
||||
using(var tmpfile = sf.TempFile)
|
||||
{
|
||||
if (sf.Hash != null && sf.Size > 0)
|
||||
restoredb.UpdateRemoteVolume(sf.Name, RemoteVolumeState.Verified, sf.Size, sf.Hash, tr);
|
||||
|
||||
using(var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(sf.Name), tmpfile, m_options, hashsize))
|
||||
{
|
||||
var volumeID = restoredb.GetRemoteVolumeID(a.Filename);
|
||||
//Add all block/volume mappings
|
||||
foreach (var b in a.Blocks)
|
||||
backupdb.AddBlock(b.Key, b.Value, volumeID, tr);
|
||||
Utility.VerifyParameters(restoredb, m_options);
|
||||
|
||||
backupdb.UpdateRemoteVolume(a.Filename, RemoteVolumeState.Verified, a.Length, a.Hash, tr);
|
||||
backupdb.AddIndexBlockLink(restoredb.GetRemoteVolumeID(sf.Name), volumeID, tr);
|
||||
foreach(var a in svr.Volumes)
|
||||
{
|
||||
var volumeID = restoredb.GetRemoteVolumeID(a.Filename);
|
||||
//Add all block/volume mappings
|
||||
foreach(var b in a.Blocks)
|
||||
restoredb.UpdateBlock(b.Key, b.Value, volumeID, tr);
|
||||
|
||||
restoredb.UpdateRemoteVolume(a.Filename, RemoteVolumeState.Verified, a.Length, a.Hash, tr);
|
||||
restoredb.AddIndexBlockLink(restoredb.GetRemoteVolumeID(sf.Name), volumeID, tr);
|
||||
}
|
||||
|
||||
//If there are blocklists in the index file, update the blocklists
|
||||
foreach(var b in svr.BlockLists)
|
||||
restoredb.UpdateBlockset(b.Hash, b.Blocklist, hashsize, tr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
using(new Logging.Timer("CommitRecreatedDb"))
|
||||
tr.Commit();
|
||||
}
|
||||
|
||||
//We need this to prepare for the block-lists
|
||||
var dummylist = new string[0];
|
||||
|
||||
//Now record all blocksets and files needed
|
||||
using (var tr = backupdb.BeginTransaction())
|
||||
{
|
||||
var filelistWork = (from n in filelists select new RemoteVolume(n.File) as IRemoteVolume).ToList();
|
||||
foreach (var entry in new AsyncDownloader(filelistWork, backend))
|
||||
using (var tmpfile = entry.TempFile)
|
||||
catch (Exception ex)
|
||||
{
|
||||
if (entry.Hash != null && entry.Size > 0)
|
||||
backupdb.UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, entry.Size, entry.Hash, tr);
|
||||
|
||||
var parsed = VolumeBase.ParseFilename(entry.Name);
|
||||
// Create timestamped operations based on the file timestamp
|
||||
backupdb.CreateFileset(volumeIds[entry.Name], parsed.Time, tr);
|
||||
using (var filelistreader = new FilesetVolumeReader(parsed.CompressionModule, tmpfile, m_options))
|
||||
foreach (var fe in filelistreader.Files.Where(x => Library.Utility.FilterExpression.Matches(filter, x.Path)))
|
||||
{
|
||||
if (fe.Type == FilelistEntryType.Folder)
|
||||
{
|
||||
long metaid = -1;
|
||||
if (fe.Metahash != null)
|
||||
backupdb.AddMetadataset(fe.Metahash, fe.Metasize, out metaid, tr);
|
||||
backupdb.AddDirectoryEntry(fe.Path, metaid, fe.Time, tr);
|
||||
|
||||
}
|
||||
else if (fe.Type == FilelistEntryType.File)
|
||||
{
|
||||
long metaid = -1;
|
||||
long blocksetid;
|
||||
if (fe.Metahash != null)
|
||||
backupdb.AddMetadataset(fe.Metahash, fe.Metasize, out metaid, tr);
|
||||
|
||||
backupdb.AddBlockset(fe.Hash, fe.Size, m_options.Blocksize, dummylist, fe.BlocklistHashes, out blocksetid, tr);
|
||||
backupdb.AddFile(fe.Path, fe.Time, blocksetid, metaid, tr);
|
||||
}
|
||||
}
|
||||
//Not fatal
|
||||
m_result.AddWarning(string.Format("Failed to process index file: {0}", sf.Name), ex);
|
||||
}
|
||||
|
||||
using(new Logging.Timer("CommitUpdateFilesetFromRemote"))
|
||||
tr.Commit();
|
||||
}
|
||||
|
||||
using(new Logging.Timer("CommitRecreatedDb"))
|
||||
tr.Commit();
|
||||
|
||||
// TODO: In some cases, we can avoid downloading all index files,
|
||||
// if we are lucky and pick the right ones
|
||||
}
|
||||
|
||||
//We now need some blocklists, so we start by grabbing a
|
||||
// volume with one of the blocklists
|
||||
//For each volume we then update the blocklist table
|
||||
// and then restore the blocks we know
|
||||
restoredb.FindMissingBlocklistHashes();
|
||||
|
||||
foreach (var sf in new AsyncDownloader(restoredb.GetMissingBlockListVolumes().ToList(), backend))
|
||||
using (var tmpfile = sf.TempFile)
|
||||
using (var rd = new BlockVolumeReader(RestoreHandler.GetCompressionModule(sf.Name), tmpfile, m_options))
|
||||
using (var tr = restoredb.BeginTransaction())
|
||||
{
|
||||
var volumeid = restoredb.GetRemoteVolumeID(sf.Name);
|
||||
foreach (var blocklisthash in restoredb.GetBlockLists(volumeid))
|
||||
restoredb.UpdateBlocklist(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize), hashsize, tr);
|
||||
|
||||
using(new Logging.Timer("CommitRestoredBlocklist"))
|
||||
tr.Commit();
|
||||
|
||||
//At this point we can patch files with data from the block volume
|
||||
if (blockprocessor != null)
|
||||
blockprocessor(sf.Name, rd);
|
||||
}
|
||||
|
||||
// We have now grabbed as much information as possible,
|
||||
// if we are still missing data, we must now fetch block files
|
||||
restoredb.FindMissingBlocklistHashes(hashsize, null);
|
||||
|
||||
//We do this in three passes
|
||||
for(var i = 0; i < 3; i++)
|
||||
{
|
||||
// Grab the list matching the pass type
|
||||
var lst = restoredb.GetMissingBlockListVolumes(i).ToList();
|
||||
foreach (var sf in new AsyncDownloader(lst, backend))
|
||||
using (var tmpfile = sf.TempFile)
|
||||
using (var rd = new BlockVolumeReader(RestoreHandler.GetCompressionModule(sf.Name), tmpfile, m_options))
|
||||
using (var tr = restoredb.BeginTransaction())
|
||||
{
|
||||
var volumeid = restoredb.GetRemoteVolumeID(sf.Name);
|
||||
|
||||
// Update the block table so we know about the block/volume map
|
||||
foreach(var h in rd.Blocks)
|
||||
restoredb.UpdateBlock(h.Key, h.Value, volumeid, tr);
|
||||
|
||||
// Grab all known blocklists from the volume
|
||||
foreach (var blocklisthash in restoredb.GetBlockLists(volumeid))
|
||||
restoredb.UpdateBlockset(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize), hashsize, tr);
|
||||
|
||||
// Update tables so we know if we are done
|
||||
restoredb.FindMissingBlocklistHashes(hashsize, tr);
|
||||
|
||||
using(new Logging.Timer("CommitRestoredBlocklist"))
|
||||
tr.Commit();
|
||||
|
||||
//At this point we can patch files with data from the block volume
|
||||
if (blockprocessor != null)
|
||||
blockprocessor(sf.Name, rd);
|
||||
}
|
||||
}
|
||||
|
||||
backend.WaitForComplete(restoredb, null);
|
||||
|
||||
//All done, we must verify that we have all blocklist fully intact
|
||||
// if this fails, the db will not be deleted, so it can be used,
|
||||
// except to continue a backup
|
||||
restoredb.VerifyConsistency(null);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user