Added double lookup for block and file hash, this reduces the execution time to half.
This commit is contained in:
Kenneth Skovhede
@@ -70,6 +70,8 @@
|
||||
<Compile Include="CompressionWrapper.cs" />
|
||||
<Compile Include="FilenameStrategy.cs" />
|
||||
<Compile Include="ForestHash\Database\AsyncDownloader.cs" />
|
||||
<Compile Include="ForestHash\Database\HashLookup.cs" />
|
||||
<Compile Include="ForestHash\Database\HashLookupWithData.cs" />
|
||||
<Compile Include="ForestHash\Database\HashPrefixLookup.cs" />
|
||||
<Compile Include="ForestHash\Database\ILocalFileEntry.cs" />
|
||||
<Compile Include="ForestHash\Database\IRemoteVolume.cs" />
|
||||
|
||||
@@ -1,63 +1,63 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
|
||||
namespace Duplicati.Library.Main.ForestHash.Database
|
||||
{
|
||||
public static class ExtensionMethods
|
||||
{
|
||||
public static void AddParameters(this System.Data.IDbCommand self, int count)
|
||||
{
|
||||
for(var i = 0; i < count; i++)
|
||||
self.Parameters.Add(self.CreateParameter());
|
||||
}
|
||||
|
||||
public static void AddParameter(this System.Data.IDbCommand self)
|
||||
{
|
||||
self.Parameters.Add(self.CreateParameter());
|
||||
}
|
||||
|
||||
public static void AddParameter<T>(this System.Data.IDbCommand self, T value)
|
||||
{
|
||||
var p = self.CreateParameter();
|
||||
p.Value = value;
|
||||
self.Parameters.Add(p);
|
||||
}
|
||||
|
||||
public static void SetParameterValue<T>(this System.Data.IDbCommand self, int index, T value)
|
||||
{
|
||||
((System.Data.IDataParameter)self.Parameters[index]).Value = value;
|
||||
}
|
||||
|
||||
public static int ExecuteNonQuery(this System.Data.IDbCommand self, string cmd, params object[] values)
|
||||
{
|
||||
if (cmd != null)
|
||||
self.CommandText = cmd;
|
||||
|
||||
if (values != null && values.Length > 0)
|
||||
{
|
||||
self.Parameters.Clear();
|
||||
foreach (var n in values)
|
||||
self.AddParameter(n);
|
||||
}
|
||||
|
||||
return self.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
public static object ExecuteScalar(this System.Data.IDbCommand self, string cmd, params object[] values)
|
||||
{
|
||||
if (cmd != null)
|
||||
self.CommandText = cmd;
|
||||
|
||||
if (values != null && values.Length > 0)
|
||||
{
|
||||
self.Parameters.Clear();
|
||||
foreach (var n in values)
|
||||
self.AddParameter(n);
|
||||
}
|
||||
|
||||
return self.ExecuteScalar();
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
|
||||
namespace Duplicati.Library.Main.ForestHash.Database
|
||||
{
|
||||
public static class ExtensionMethods
|
||||
{
|
||||
public static void AddParameters(this System.Data.IDbCommand self, int count)
|
||||
{
|
||||
for(var i = 0; i < count; i++)
|
||||
self.Parameters.Add(self.CreateParameter());
|
||||
}
|
||||
|
||||
public static void AddParameter(this System.Data.IDbCommand self)
|
||||
{
|
||||
self.Parameters.Add(self.CreateParameter());
|
||||
}
|
||||
|
||||
public static void AddParameter<T>(this System.Data.IDbCommand self, T value)
|
||||
{
|
||||
var p = self.CreateParameter();
|
||||
p.Value = value;
|
||||
self.Parameters.Add(p);
|
||||
}
|
||||
|
||||
public static void SetParameterValue<T>(this System.Data.IDbCommand self, int index, T value)
|
||||
{
|
||||
((System.Data.IDataParameter)self.Parameters[index]).Value = value;
|
||||
}
|
||||
|
||||
public static int ExecuteNonQuery(this System.Data.IDbCommand self, string cmd, params object[] values)
|
||||
{
|
||||
if (cmd != null)
|
||||
self.CommandText = cmd;
|
||||
|
||||
if (values != null && values.Length > 0)
|
||||
{
|
||||
self.Parameters.Clear();
|
||||
foreach (var n in values)
|
||||
self.AddParameter(n);
|
||||
}
|
||||
|
||||
return self.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
public static object ExecuteScalar(this System.Data.IDbCommand self, string cmd, params object[] values)
|
||||
{
|
||||
if (cmd != null)
|
||||
self.CommandText = cmd;
|
||||
|
||||
if (values != null && values.Length > 0)
|
||||
{
|
||||
self.Parameters.Clear();
|
||||
foreach (var n in values)
|
||||
self.AddParameter(n);
|
||||
}
|
||||
|
||||
return self.ExecuteScalar();
|
||||
}
|
||||
|
||||
public static System.Data.IDataReader ExecuteReader(this System.Data.IDbCommand self, string cmd, params object[] values)
|
||||
@@ -73,35 +73,35 @@ namespace Duplicati.Library.Main.ForestHash.Database
|
||||
}
|
||||
|
||||
return self.ExecuteReader();
|
||||
}
|
||||
|
||||
public static void DumpSQL(this System.Data.IDbConnection self, string sql, params object[] parameters)
|
||||
{
|
||||
using (var c = self.CreateCommand())
|
||||
{
|
||||
c.CommandText = sql;
|
||||
if (parameters != null)
|
||||
foreach (var p in parameters)
|
||||
c.AddParameter(p);
|
||||
|
||||
using (var rd = c.ExecuteReader())
|
||||
{
|
||||
for (int i = 0; i < rd.FieldCount; i++)
|
||||
Console.Write((i == 0 ? "" : "\t") + rd.GetName(i));
|
||||
Console.WriteLine();
|
||||
|
||||
long n = 0;
|
||||
while (rd.Read())
|
||||
{
|
||||
for (int i = 0; i < rd.FieldCount; i++)
|
||||
Console.Write(string.Format((i == 0 ? "{0}" : "\t{0}"), rd.GetValue(i)));
|
||||
Console.WriteLine();
|
||||
n++;
|
||||
}
|
||||
Console.WriteLine("{0} records", n);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void DumpSQL(this System.Data.IDbConnection self, string sql, params object[] parameters)
|
||||
{
|
||||
using (var c = self.CreateCommand())
|
||||
{
|
||||
c.CommandText = sql;
|
||||
if (parameters != null)
|
||||
foreach (var p in parameters)
|
||||
c.AddParameter(p);
|
||||
|
||||
using (var rd = c.ExecuteReader())
|
||||
{
|
||||
for (int i = 0; i < rd.FieldCount; i++)
|
||||
Console.Write((i == 0 ? "" : "\t") + rd.GetName(i));
|
||||
Console.WriteLine();
|
||||
|
||||
long n = 0;
|
||||
while (rd.Read())
|
||||
{
|
||||
for (int i = 0; i < rd.FieldCount; i++)
|
||||
Console.Write(string.Format((i == 0 ? "{0}" : "\t{0}"), rd.GetValue(i)));
|
||||
Console.WriteLine();
|
||||
n++;
|
||||
}
|
||||
Console.WriteLine("{0} records", n);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
|
||||
namespace Duplicati.Library.Main.ForestHash.Database
|
||||
{
|
||||
/// <summary>
|
||||
/// Provides a simple hash-table like lookup,
|
||||
/// similar to a MRU cache
|
||||
/// </summary>
|
||||
internal class HashLookup<T>
|
||||
{
|
||||
private const long MAX_MEMORY = 32 * 1024 * 1024; //32mb
|
||||
|
||||
/// <summary>
|
||||
/// The lookup table with all values
|
||||
/// </summary>
|
||||
private T[] m_lookup;
|
||||
|
||||
/// <summary>
|
||||
/// Typecasted size of table
|
||||
/// </summary>
|
||||
private ulong m_modulo;
|
||||
|
||||
public HashLookup(uint elementsize, ulong memorylimit = MAX_MEMORY)
|
||||
{
|
||||
//Allocate as a large boolean/bit array
|
||||
m_lookup = new T[Math.Max(32, Math.Max(1024, memorylimit * 8) / elementsize)];
|
||||
m_modulo = (ulong)m_lookup.Length;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds a hash to the table
|
||||
/// </summary>
|
||||
/// <param name="key">The hash key</param>
|
||||
/// <param name="value">The actual hash data</param>
|
||||
public void AddHash(ulong key, T value)
|
||||
{
|
||||
//Textbox hashing, the value modulo number of entries
|
||||
// works because the input is high quality random data
|
||||
var bit = key % m_modulo;
|
||||
m_lookup[bit] = value;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queries the table to see if a hash exists.
|
||||
/// A true result value is definitive, a false result value should be interpreted as maybe.
|
||||
/// </summary>
|
||||
/// <param name="key">The hash key</param>
|
||||
/// <param name="data">The actual hash data</param>
|
||||
/// <returns>True if the hash was found, false otherwise</returns>
|
||||
public bool HashExists(ulong key, T value)
|
||||
{
|
||||
//Textbox hashing, the value modulo number of entries
|
||||
// works because the input is high quality random data
|
||||
var bit = key % m_modulo;
|
||||
return object.Equals(m_lookup[bit], value);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Calculates number of used entries in the table
|
||||
/// </summary>
|
||||
public ulong Entries
|
||||
{
|
||||
get
|
||||
{
|
||||
ulong c = 0;
|
||||
foreach (var x in m_lookup)
|
||||
if (!object.Equals(x, default(T)))
|
||||
c++;
|
||||
|
||||
return c;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets a value between 0 and 1 descring how
|
||||
/// many entries are used
|
||||
/// </summary>
|
||||
public double TableUsageRatio
|
||||
{
|
||||
get
|
||||
{
|
||||
return Entries / (double)m_lookup.Length;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
|
||||
namespace Duplicati.Library.Main.ForestHash.Database
|
||||
{
|
||||
/// <summary>
|
||||
/// Provides a simple hash-table like lookup,
|
||||
/// similar to a MRU cache
|
||||
/// </summary>
|
||||
internal class HashLookupWithData<TKey, TValue>
|
||||
{
|
||||
private const long MAX_MEMORY = 32 * 1024 * 1024; //32mb
|
||||
|
||||
/// <summary>
|
||||
/// The lookup table with all values
|
||||
/// </summary>
|
||||
private KeyValuePair<TKey, TValue>[] m_lookup;
|
||||
|
||||
/// <summary>
|
||||
/// Typecasted size of table
|
||||
/// </summary>
|
||||
private ulong m_modulo;
|
||||
|
||||
public HashLookupWithData(uint elementsize, ulong memorylimit = MAX_MEMORY)
|
||||
{
|
||||
//Allocate as a large boolean/bit array
|
||||
m_lookup = new KeyValuePair<TKey, TValue>[Math.Max(32, Math.Max(1024, memorylimit * 8) / elementsize)];
|
||||
m_modulo = (ulong)m_lookup.Length;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds a hash to the table
|
||||
/// </summary>
|
||||
/// <param name="key">The hash key</param>
|
||||
/// <param name="value">The actual hash data</param>
|
||||
public void AddHash(ulong key, TKey value, TValue data)
|
||||
{
|
||||
//Textbox hashing, the value modulo number of entries
|
||||
// works because the input is high quality random data
|
||||
var bit = key % m_modulo;
|
||||
m_lookup[bit] = new KeyValuePair<TKey,TValue>(value, data);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queries the table to see if a hash exists.
|
||||
/// A true result value is definitive, a false result value should be interpreted as maybe.
|
||||
/// </summary>
|
||||
/// <param name="key">The hash key</param>
|
||||
/// <param name="data">The actual hash data</param>
|
||||
/// <returns>True if the hash was found, false otherwise</returns>
|
||||
public bool HashExists(ulong key, TKey value, out TValue data)
|
||||
{
|
||||
//Textbox hashing, the value modulo number of entries
|
||||
// works because the input is high quality random data
|
||||
var bit = key % m_modulo;
|
||||
var el = m_lookup[bit];
|
||||
if (object.Equals(el.Key, value))
|
||||
{
|
||||
data = el.Value;
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
data = default(TValue);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Calculates number of used entries in the table
|
||||
/// </summary>
|
||||
public ulong Entries
|
||||
{
|
||||
get
|
||||
{
|
||||
ulong c = 0;
|
||||
foreach (var x in m_lookup)
|
||||
if (!object.Equals(x.Key, default(TKey)))
|
||||
c++;
|
||||
|
||||
return c;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets a value between 0 and 1 descring how
|
||||
/// many entries are used
|
||||
/// </summary>
|
||||
public double TableUsageRatio
|
||||
{
|
||||
get
|
||||
{
|
||||
return Entries / (double)m_lookup.Length;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -29,27 +29,23 @@ namespace Duplicati.Library.Main.ForestHash.Database
|
||||
/// </summary>
|
||||
/// <param name="memorylimit">The maximum number of bytes to allocate for storage table</param>
|
||||
/// <param name="data">Data to prepoulate the table with</param>
|
||||
public HashPrefixLookup(ulong memorylimit = MAX_MEMORY, IEnumerable<byte[]> data = null)
|
||||
public HashPrefixLookup(ulong memorylimit = MAX_MEMORY)
|
||||
{
|
||||
m_bits = Math.Max(1024, memorylimit * 8);
|
||||
|
||||
//Allocate as a large boolean/bit array
|
||||
m_lookup = new ulong[m_bits / 64];
|
||||
|
||||
if (data != null)
|
||||
foreach (var d in data)
|
||||
AddHash(d);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds a hash to the table
|
||||
/// </summary>
|
||||
/// <param name="data">The bytes representing the hash</param>
|
||||
public void AddHash(byte[] data)
|
||||
public void AddHash(ulong data)
|
||||
{
|
||||
//Textbox hashing, the value modulo number of entries
|
||||
// works because the input is high quality random data
|
||||
var bit = BitConverter.ToUInt64(data, 0) % m_bits;
|
||||
var bit = data % m_bits;
|
||||
m_lookup[bit >> 6] |= 1uL << (ushort)(bit & 0x3f);
|
||||
}
|
||||
|
||||
@@ -59,11 +55,11 @@ namespace Duplicati.Library.Main.ForestHash.Database
|
||||
/// </summary>
|
||||
/// <param name="data">The bytes representing the hash</param>
|
||||
/// <returns>True if the hash was found, false otherwise</returns>
|
||||
public bool HashExists(byte[] data)
|
||||
public bool HashExists(ulong data)
|
||||
{
|
||||
//Textbox hashing, the value modulo number of entries
|
||||
// works because the input is high quality random data
|
||||
var bit = BitConverter.ToUInt64(data, 0) % m_bits;
|
||||
var bit = data % m_bits;
|
||||
return (m_lookup[bit >> 6] & (1uL << (ushort)(bit & 0x3f))) != 0;
|
||||
}
|
||||
|
||||
@@ -95,5 +91,102 @@ namespace Duplicati.Library.Main.ForestHash.Database
|
||||
return (this.BitsUsed / (double)m_bits);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Hex digit ASCII lookup table
|
||||
/// </summary>
|
||||
private static readonly byte[] HEXTB = new byte[128];
|
||||
|
||||
/// <summary>
|
||||
/// Base64 digit ASCII lookup table
|
||||
/// </summary>
|
||||
private static readonly byte[] BAS64TB = new byte[128];
|
||||
|
||||
/// <summary>
|
||||
/// Initializes the lookup
|
||||
/// </summary>
|
||||
static HashPrefixLookup()
|
||||
{
|
||||
for(byte i = 0; i < 10; i++)
|
||||
HEXTB['0' + i] = i;
|
||||
|
||||
for (byte i = 0; i < 6; i++)
|
||||
{
|
||||
//Case insensitive support
|
||||
HEXTB['A' + i] = (byte)(i + 10);
|
||||
HEXTB['a' + i] = (byte)(i + 10);
|
||||
}
|
||||
|
||||
for (byte i = 0; i < 26; i++)
|
||||
BAS64TB['A' + i] = i;
|
||||
for (byte i = 0; i < 26; i++)
|
||||
BAS64TB['a' + i] = (byte)(i + 26);
|
||||
for (byte i = 0; i < 10; i++)
|
||||
BAS64TB['0' + i] = (byte)(i + 52);
|
||||
|
||||
BAS64TB['+'] = 62;
|
||||
BAS64TB['/'] = 63;
|
||||
|
||||
//Base64 URL support
|
||||
BAS64TB['-'] = 62;
|
||||
BAS64TB['_'] = 63;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Debug help function to compare values
|
||||
/// </summary>
|
||||
/// <param name="value">The value to convert</param>
|
||||
/// <returns>The converted value</returns>
|
||||
public static ulong ConvertEndianness(ulong value)
|
||||
{
|
||||
return
|
||||
((value << 56) & 0xff00000000000000uL) |
|
||||
((value << 40) & 0x00ff000000000000uL) |
|
||||
((value << 24) & 0x0000ff0000000000uL) |
|
||||
((value << 8) & 0x000000ff00000000uL) |
|
||||
((value >> 8) & 0x00000000ff000000uL) |
|
||||
((value >> 24) & 0x0000000000ff0000uL) |
|
||||
((value >> 40) & 0x000000000000ff00uL) |
|
||||
((value >> 56) & 0x00000000000000ffuL);
|
||||
}
|
||||
|
||||
//Faster internal hex decoder, assumes valid hex string of at least 16 chars and no extended chars
|
||||
public static ulong DecodeHexHash(string hash)
|
||||
{
|
||||
return
|
||||
(ulong)HEXTB[hash[0]] << 60 |
|
||||
(ulong)HEXTB[hash[1]] << 56 |
|
||||
(ulong)HEXTB[hash[2]] << 52 |
|
||||
(ulong)HEXTB[hash[3]] << 48 |
|
||||
(ulong)HEXTB[hash[4]] << 44 |
|
||||
(ulong)HEXTB[hash[5]] << 40 |
|
||||
(ulong)HEXTB[hash[6]] << 36 |
|
||||
(ulong)HEXTB[hash[7]] << 32 |
|
||||
(ulong)HEXTB[hash[8]] << 28 |
|
||||
(ulong)HEXTB[hash[9]] << 24 |
|
||||
(ulong)HEXTB[hash[10]] << 20 |
|
||||
(ulong)HEXTB[hash[11]] << 16 |
|
||||
(ulong)HEXTB[hash[12]] << 12 |
|
||||
(ulong)HEXTB[hash[13]] << 8 |
|
||||
(ulong)HEXTB[hash[14]] << 4 |
|
||||
(ulong)HEXTB[hash[15]] << 0;
|
||||
}
|
||||
|
||||
//Faster internal base64 decoder, assumes valid hex string of at least 11 chars and no extended chars
|
||||
public static ulong DecodeBase64Hash(string hash)
|
||||
{
|
||||
return
|
||||
(ulong)BAS64TB[hash[0]] << 58 |
|
||||
(ulong)BAS64TB[hash[1]] << 52 |
|
||||
(ulong)BAS64TB[hash[2]] << 46 |
|
||||
(ulong)BAS64TB[hash[3]] << 40 |
|
||||
(ulong)BAS64TB[hash[4]] << 34 |
|
||||
(ulong)BAS64TB[hash[5]] << 28 |
|
||||
(ulong)BAS64TB[hash[6]] << 22 |
|
||||
(ulong)BAS64TB[hash[7]] << 16 |
|
||||
(ulong)BAS64TB[hash[8]] << 10 |
|
||||
(ulong)BAS64TB[hash[9]] << 4 |
|
||||
(((ulong)BAS64TB[hash[10]] >> 2) & 0x0f);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,8 +8,18 @@ namespace Duplicati.Library.Main.ForestHash.Database
|
||||
{
|
||||
public class LocalBackupDatabase : Localdatabase
|
||||
{
|
||||
/// <summary>
|
||||
/// The amount of memory allocated to store block hashes in memory (will allocate twice this memory)
|
||||
/// </summary>
|
||||
private const ulong BLOCK_HASH_LOOKUP_SIZE = 64 * 1024 * 1024;
|
||||
/// <summary>
|
||||
/// The amount of memory allocated to store file in memory (will allocate twice this memory)
|
||||
/// </summary>
|
||||
private const ulong FILE_HASH_LOOKUP_SIZE = 16 * 1024 * 1024;
|
||||
/// <summary>
|
||||
/// An approximate size of a hash-string in memory (44 chars * 2 for unicode + 8 bytes for pointer = 104)
|
||||
/// </summary>
|
||||
private const uint HASH_GUESS_SIZE = 128;
|
||||
|
||||
private readonly System.Data.IDbCommand m_findblockCommand;
|
||||
private readonly System.Data.IDbCommand m_findblocksetCommand;
|
||||
@@ -37,13 +47,18 @@ namespace Duplicati.Library.Main.ForestHash.Database
|
||||
private readonly System.Data.IDbCommand m_createremotevolumeCommand;
|
||||
private readonly System.Data.IDbCommand m_insertfileOperationCommand;
|
||||
|
||||
private HashPrefixLookup m_blockHashLookup;
|
||||
private HashPrefixLookup m_fileHashLookup;
|
||||
private HashPrefixLookup m_blockHashPrefixLookup;
|
||||
private HashPrefixLookup m_fileHashPrefixLookup;
|
||||
private HashLookup<string> m_blockHashFullLookup;
|
||||
private HashLookupWithData<string, long> m_fileHashFullLookup;
|
||||
|
||||
private long m_falseBlockPositives = 0;
|
||||
private long m_falseFilePositives = 0;
|
||||
private long m_falseBlockNegatives = 0;
|
||||
private long m_falseFileNegatives = 0;
|
||||
|
||||
private long m_missingBlockHashes;
|
||||
private string m_scantimelookupTablename;
|
||||
|
||||
public LocalBackupDatabase(string path)
|
||||
: this(CreateConnection(path))
|
||||
@@ -62,7 +77,6 @@ namespace Duplicati.Library.Main.ForestHash.Database
|
||||
m_insertblockCommand = m_connection.CreateCommand();
|
||||
m_insertfileCommand = m_connection.CreateCommand();
|
||||
m_insertblocksetCommand = m_connection.CreateCommand();
|
||||
//m_selectfileCommand = m_connection.CreateCommand();
|
||||
m_insertmetadatasetCommand = m_connection.CreateCommand();
|
||||
m_findblocksetCommand = m_connection.CreateCommand();
|
||||
m_findmetadatasetCommand = m_connection.CreateCommand();
|
||||
@@ -115,7 +129,13 @@ namespace Duplicati.Library.Main.ForestHash.Database
|
||||
m_insertmetadatasetCommand.CommandText = @"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (?); SELECT last_insert_rowid();";
|
||||
m_insertmetadatasetCommand.AddParameter();
|
||||
|
||||
m_selectfileSimpleCommand.CommandText = @"SELECT ""OperationFileset"".""FilesetID"", ""OperationFileset"".""Scantime"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""Fileset"".""Path"" = ? ORDER BY ""OperationFileset"".""Scantime"" DESC";
|
||||
//Need a temporary table with path/scantime lookups
|
||||
m_scantimelookupTablename = "ScanTime-" + Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
|
||||
using (var cmd = m_connection.CreateCommand())
|
||||
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT ""OperationFileset"".""FilesetID"" AS ""FilesetID"", MAX(""OperationFileset"".""Scantime"") AS ""Scantime"", ""Fileset"".""Path"" AS ""Path"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" GROUP BY ""FilesetID"", ""Path"" ", m_scantimelookupTablename));
|
||||
|
||||
m_selectfileSimpleCommand.CommandText = string.Format(@"SELECT ""FilesetID"", ""Scantime"" FROM ""{0}"" WHERE ""Path"" = ?", m_scantimelookupTablename);
|
||||
//m_selectfileSimpleCommand.CommandText = @"SELECT ""OperationFileset"".""FilesetID"", ""OperationFileset"".""Scantime"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""Fileset"".""Path"" = ? ORDER BY ""OperationFileset"".""Scantime"" DESC";
|
||||
m_selectfileSimpleCommand.AddParameters(1);
|
||||
|
||||
m_selectfileHashCommand.CommandText = @"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"" INNER JOIN ""Fileset"" ON ""Blockset"".""ID"" = ""Fileset"".""BlocksetID"" WHERE ""Fileset"".""ID"" = ? ";
|
||||
@@ -134,18 +154,32 @@ namespace Duplicati.Library.Main.ForestHash.Database
|
||||
m_createremotevolumeCommand.AddParameter(m_operationid);
|
||||
m_createremotevolumeCommand.AddParameters(3);
|
||||
|
||||
m_blockHashLookup = new HashPrefixLookup(BLOCK_HASH_LOOKUP_SIZE);
|
||||
m_fileHashLookup = new HashPrefixLookup(FILE_HASH_LOOKUP_SIZE);
|
||||
m_blockHashPrefixLookup = new HashPrefixLookup(BLOCK_HASH_LOOKUP_SIZE);
|
||||
m_fileHashPrefixLookup = new HashPrefixLookup(FILE_HASH_LOOKUP_SIZE);
|
||||
|
||||
m_blockHashFullLookup = new HashLookup<string>(HASH_GUESS_SIZE, BLOCK_HASH_LOOKUP_SIZE);
|
||||
m_fileHashFullLookup = new HashLookupWithData<string, long>(HASH_GUESS_SIZE, FILE_HASH_LOOKUP_SIZE);
|
||||
|
||||
using (var cmd = m_connection.CreateCommand())
|
||||
using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Block"".""Hash"" FROM ""Block"", ""RemoteVolume"" WHERE ""RemoteVolume"".""Name"" = ""Block"".""File"" AND ""RemoteVolume"".""State"" IN (?,?,?,?) ", RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()))
|
||||
while (rd.Read())
|
||||
m_blockHashLookup.AddHash(Convert.FromBase64String(rd.GetValue(0).ToString()));
|
||||
{
|
||||
var str = rd.GetValue(0).ToString();
|
||||
var key = HashPrefixLookup.DecodeBase64Hash(str);
|
||||
m_blockHashPrefixLookup.AddHash(key);
|
||||
m_blockHashFullLookup.AddHash(key, str);
|
||||
}
|
||||
|
||||
using (var cmd = m_connection.CreateCommand())
|
||||
using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""FullHash"" FROM ""BlockSet"""))
|
||||
using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""FullHash"", ""ID"" FROM ""BlockSet"""))
|
||||
while (rd.Read())
|
||||
m_fileHashLookup.AddHash(Convert.FromBase64String(rd.GetValue(0).ToString()));
|
||||
{
|
||||
var str = rd.GetValue(0).ToString();
|
||||
var id = Convert.ToInt64(rd.GetValue(1));
|
||||
var key = HashPrefixLookup.DecodeBase64Hash(str);
|
||||
m_fileHashPrefixLookup.AddHash(key);
|
||||
m_fileHashFullLookup.AddHash(key, str, id);
|
||||
}
|
||||
|
||||
using (var cmd = m_connection.CreateCommand())
|
||||
m_missingBlockHashes = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT (*) FROM (SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"" FROM ""Block"", ""RemoteVolume"" WHERE ""RemoteVolume"".""Name"" = ""Block"".""File"" AND ""RemoteVolume"".""State"" NOT IN (?,?,?,?))", RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()));
|
||||
@@ -161,9 +195,12 @@ namespace Duplicati.Library.Main.ForestHash.Database
|
||||
public bool AddBlock(string key, long size, string archivename, System.Data.IDbTransaction transaction = null)
|
||||
{
|
||||
object r = null;
|
||||
var hashdata = Convert.FromBase64String(key);
|
||||
if (m_blockHashLookup.HashExists(hashdata))
|
||||
var hashdata = HashPrefixLookup.DecodeBase64Hash(key);
|
||||
if (m_blockHashPrefixLookup.HashExists(hashdata))
|
||||
{
|
||||
if (m_missingBlockHashes == 0 && m_blockHashFullLookup.HashExists(hashdata, key))
|
||||
return false;
|
||||
|
||||
m_findblockCommand.Transaction = transaction;
|
||||
r = m_findblockCommand.ExecuteScalar(null, key, size);
|
||||
if (r == null || r == DBNull.Value)
|
||||
@@ -172,8 +209,12 @@ namespace Duplicati.Library.Main.ForestHash.Database
|
||||
}
|
||||
else
|
||||
{
|
||||
m_falseBlockNegatives++;
|
||||
if (m_missingBlockHashes == 0)
|
||||
{
|
||||
m_blockHashFullLookup.AddHash(hashdata, key);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -184,11 +225,15 @@ namespace Duplicati.Library.Main.ForestHash.Database
|
||||
m_insertblockCommand.SetParameterValue(1, archivename);
|
||||
m_insertblockCommand.SetParameterValue(2, size);
|
||||
m_insertblockCommand.ExecuteNonQuery();
|
||||
m_blockHashLookup.AddHash(hashdata);
|
||||
m_blockHashPrefixLookup.AddHash(hashdata);
|
||||
m_blockHashFullLookup.AddHash(hashdata, key);
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
//We add/update it now
|
||||
m_blockHashFullLookup.AddHash(hashdata, key);
|
||||
|
||||
//If the block is found and the volume is broken somehow.
|
||||
m_findremotevolumestateCommand.Transaction = transaction;
|
||||
r = m_findremotevolumestateCommand.ExecuteScalar(null, r);
|
||||
@@ -220,20 +265,24 @@ namespace Duplicati.Library.Main.ForestHash.Database
|
||||
public bool AddBlockset(string filehash, long size, int blocksize, IEnumerable<string> hashes, IEnumerable<string> blocklistHashes, out long blocksetid, System.Data.IDbTransaction transaction = null)
|
||||
{
|
||||
object r = null;
|
||||
var hashdata = Convert.FromBase64String(filehash);
|
||||
|
||||
if (m_fileHashLookup.HashExists(hashdata))
|
||||
var hashdata = HashPrefixLookup.DecodeBase64Hash(filehash);
|
||||
if (m_fileHashPrefixLookup.HashExists(hashdata))
|
||||
{
|
||||
if (m_fileHashFullLookup.HashExists(hashdata, filehash, out blocksetid))
|
||||
return false;
|
||||
|
||||
m_findblocksetCommand.Transaction = transaction;
|
||||
r = m_findblocksetCommand.ExecuteScalar(null, filehash, size);
|
||||
if (r == null || r == DBNull.Value)
|
||||
m_falseFilePositives++;
|
||||
else
|
||||
m_falseFileNegatives++;
|
||||
}
|
||||
|
||||
if (r != null && r != DBNull.Value)
|
||||
{
|
||||
blocksetid = Convert.ToInt64(r);
|
||||
//TODO: Make a sweep to see if the blocks match the record?
|
||||
m_fileHashFullLookup.AddHash(hashdata, filehash, blocksetid);
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -241,7 +290,8 @@ namespace Duplicati.Library.Main.ForestHash.Database
|
||||
{
|
||||
m_insertblocksetCommand.Transaction = tr.Parent;
|
||||
blocksetid = Convert.ToInt64(m_insertblocksetCommand.ExecuteScalar(null, size, filehash));
|
||||
m_fileHashLookup.AddHash(hashdata);
|
||||
m_fileHashPrefixLookup.AddHash(hashdata);
|
||||
m_fileHashFullLookup.AddHash(hashdata, filehash, blocksetid);
|
||||
|
||||
long ix = 0;
|
||||
if (blocklistHashes != null)
|
||||
@@ -555,6 +605,22 @@ namespace Duplicati.Library.Main.ForestHash.Database
|
||||
}
|
||||
}
|
||||
|
||||
public override void Dispose ()
|
||||
{
|
||||
if (m_scantimelookupTablename != null)
|
||||
try
|
||||
{
|
||||
using(var cmd = m_connection.CreateCommand())
|
||||
cmd.ExecuteNonQuery(string.Format(@"DROP TABLE ""{0}"" ", m_scantimelookupTablename));
|
||||
}
|
||||
finally
|
||||
{
|
||||
m_scantimelookupTablename = null;
|
||||
}
|
||||
|
||||
base.Dispose();
|
||||
}
|
||||
|
||||
internal void UpdateChangeStatistics (BackupStatistics m_stat)
|
||||
{
|
||||
using (var cmd = m_connection.CreateCommand())
|
||||
@@ -576,22 +642,26 @@ namespace Duplicati.Library.Main.ForestHash.Database
|
||||
m_stat.ModifiedFolders = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" = ? AND ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, m_operationid));
|
||||
m_stat.ModifiedSymlinks = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" = ? AND ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, SYMLINK_BLOCKSET_ID, m_operationid));
|
||||
m_stat.ModifiedFiles = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" != ? AND ""Fileset"".""BlocksetID"" != ? AND ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, m_operationid));
|
||||
|
||||
/*m_stat.ModifiedFolders = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT * FROM ( (SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""Fileset"".""ID"" WHERE ""OperationID"" = ?) AS A, (SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""Fileset"".""ID"" WHERE ""OperationID"" = ?) AS B WHERE ""A"".""Path"" = ""B"".""Path"" AND ""A"".""BlocsetID"" != ? AND ""A"".""MetadataID"" != ""B"".""MetadataID"" )", m_operationid, lastOperationId, FOLDER_BLOCKSET_ID));
|
||||
m_stat.ModifiedSymlinks = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT * FROM ( (SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""Fileset"".""ID"" WHERE ""OperationID"" = ?) AS A, (SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""Fileset"".""ID"" WHERE ""OperationID"" = ?) AS B WHERE ""A"".""Path"" = ""B"".""Path"" AND ""A"".""BlocsetID"" != ? AND ""A"".""MetadataID"" != ""B"".""MetadataID"" )", m_operationid, lastOperationId, SYMLINK_BLOCKSET_ID));
|
||||
m_stat.ModifiedFiles = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT * FROM ( (SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""Fileset"".""ID"" WHERE ""OperationID"" = ?) AS A, (SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""Fileset"".""ID"" WHERE ""OperationID"" = ?) AS B WHERE ""A"".""Path"" = ""B"".""Path"" AND (""A"".""BlocsetID"" != ""B"".""BlocksetID"" OR ""A"".""MetadataID"" != ""B"".""MetadataID"") )", m_operationid, lastOperationId));
|
||||
* */
|
||||
}
|
||||
|
||||
if (m_falseBlockPositives > 200 || m_falseFilePositives > 20)
|
||||
{
|
||||
m_stat.LogWarning(string.Format("Lookup tables gave false positives, this may indicate too small tables. Block: {0}, File: {0}", m_falseBlockPositives, m_falseFilePositives), null);
|
||||
|
||||
if (m_blockHashLookup.TableUsageRatio > 0.5)
|
||||
m_stat.LogWarning(string.Format("Block hash lookup table is too small, usage is: {0}%", m_blockHashLookup.TableUsageRatio * 100), null);
|
||||
if (m_blockHashPrefixLookup.TableUsageRatio > 0.5)
|
||||
m_stat.LogWarning(string.Format("Block hash lookup table is too small, usage is: {0}%", m_blockHashPrefixLookup.TableUsageRatio * 100), null);
|
||||
|
||||
if (m_fileHashLookup.TableUsageRatio > 0.5)
|
||||
m_stat.LogWarning(string.Format("File hash lookup table is too small, usage is: {0}%", m_fileHashLookup.TableUsageRatio * 100), null);
|
||||
if (m_fileHashPrefixLookup.TableUsageRatio > 0.5)
|
||||
m_stat.LogWarning(string.Format("File hash lookup table is too small, usage is: {0}%", m_fileHashPrefixLookup.TableUsageRatio * 100), null);
|
||||
}
|
||||
|
||||
// Good for profiling, but takes some time to calculate these stats
|
||||
if (Logging.Log.LogLevel == Logging.LogMessageType.Profiling)
|
||||
{
|
||||
Logging.Log.WriteMessage(string.Format("Prefix BlockHash Lookup entries {0}, usage: {1}, falsePositives: {2}", m_blockHashPrefixLookup.BitsUsed, m_blockHashPrefixLookup.TableUsageRatio, m_falseBlockPositives), Logging.LogMessageType.Profiling);
|
||||
Logging.Log.WriteMessage(string.Format("Prefix BlockHash Lookup entries {0}, usage: {1}, falsePositives: {2}", m_fileHashPrefixLookup.BitsUsed, m_fileHashPrefixLookup.TableUsageRatio, m_falseFilePositives), Logging.LogMessageType.Profiling);
|
||||
Logging.Log.WriteMessage(string.Format("Full BlockHash Lookup entries {0}, usage: {1}, falseNegatives: {2}", m_blockHashFullLookup.Entries, m_blockHashFullLookup.TableUsageRatio, m_falseBlockNegatives), Logging.LogMessageType.Profiling);
|
||||
Logging.Log.WriteMessage(string.Format("Full FileHash Lookup entries {0}, usage: {1}, falseNegatives: {2}", m_fileHashFullLookup.Entries, m_fileHashFullLookup.TableUsageRatio, m_falseFileNegatives), Logging.LogMessageType.Profiling);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user