diff --git a/Duplicati/Library/Main/Duplicati.Library.Main.csproj b/Duplicati/Library/Main/Duplicati.Library.Main.csproj index bc5d0fa2b..bb15a26bd 100644 --- a/Duplicati/Library/Main/Duplicati.Library.Main.csproj +++ b/Duplicati/Library/Main/Duplicati.Library.Main.csproj @@ -70,6 +70,8 @@ + + diff --git a/Duplicati/Library/Main/ForestHash/Database/ExtensionMethods.cs b/Duplicati/Library/Main/ForestHash/Database/ExtensionMethods.cs index 9ce0e610f..a21901a9d 100644 --- a/Duplicati/Library/Main/ForestHash/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/ForestHash/Database/ExtensionMethods.cs @@ -1,63 +1,63 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -namespace Duplicati.Library.Main.ForestHash.Database -{ - public static class ExtensionMethods - { - public static void AddParameters(this System.Data.IDbCommand self, int count) - { - for(var i = 0; i < count; i++) - self.Parameters.Add(self.CreateParameter()); - } - - public static void AddParameter(this System.Data.IDbCommand self) - { - self.Parameters.Add(self.CreateParameter()); - } - - public static void AddParameter(this System.Data.IDbCommand self, T value) - { - var p = self.CreateParameter(); - p.Value = value; - self.Parameters.Add(p); - } - - public static void SetParameterValue(this System.Data.IDbCommand self, int index, T value) - { - ((System.Data.IDataParameter)self.Parameters[index]).Value = value; - } - - public static int ExecuteNonQuery(this System.Data.IDbCommand self, string cmd, params object[] values) - { - if (cmd != null) - self.CommandText = cmd; - - if (values != null && values.Length > 0) - { - self.Parameters.Clear(); - foreach (var n in values) - self.AddParameter(n); - } - - return self.ExecuteNonQuery(); - } - - public static object ExecuteScalar(this System.Data.IDbCommand self, string cmd, params object[] values) - { - if (cmd != null) - self.CommandText = cmd; - - if (values != null && values.Length > 0) - { - self.Parameters.Clear(); - foreach (var n in values) - self.AddParameter(n); - } - - return self.ExecuteScalar(); +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Duplicati.Library.Main.ForestHash.Database +{ + public static class ExtensionMethods + { + public static void AddParameters(this System.Data.IDbCommand self, int count) + { + for(var i = 0; i < count; i++) + self.Parameters.Add(self.CreateParameter()); + } + + public static void AddParameter(this System.Data.IDbCommand self) + { + self.Parameters.Add(self.CreateParameter()); + } + + public static void AddParameter(this System.Data.IDbCommand self, T value) + { + var p = self.CreateParameter(); + p.Value = value; + self.Parameters.Add(p); + } + + public static void SetParameterValue(this System.Data.IDbCommand self, int index, T value) + { + ((System.Data.IDataParameter)self.Parameters[index]).Value = value; + } + + public static int ExecuteNonQuery(this System.Data.IDbCommand self, string cmd, params object[] values) + { + if (cmd != null) + self.CommandText = cmd; + + if (values != null && values.Length > 0) + { + self.Parameters.Clear(); + foreach (var n in values) + self.AddParameter(n); + } + + return self.ExecuteNonQuery(); + } + + public static object ExecuteScalar(this System.Data.IDbCommand self, string cmd, params object[] values) + { + if (cmd != null) + self.CommandText = cmd; + + if (values != null && values.Length > 0) + { + self.Parameters.Clear(); + foreach (var n in values) + self.AddParameter(n); + } + + return self.ExecuteScalar(); } public static System.Data.IDataReader ExecuteReader(this System.Data.IDbCommand self, string cmd, params object[] values) @@ -73,35 +73,35 @@ namespace Duplicati.Library.Main.ForestHash.Database } return self.ExecuteReader(); - } - - public static void DumpSQL(this System.Data.IDbConnection self, string sql, params object[] parameters) - { - using (var c = self.CreateCommand()) - { - c.CommandText = sql; - if (parameters != null) - foreach (var p in parameters) - c.AddParameter(p); - - using (var rd = c.ExecuteReader()) - { - for (int i = 0; i < rd.FieldCount; i++) - Console.Write((i == 0 ? "" : "\t") + rd.GetName(i)); - Console.WriteLine(); - - long n = 0; - while (rd.Read()) - { - for (int i = 0; i < rd.FieldCount; i++) - Console.Write(string.Format((i == 0 ? "{0}" : "\t{0}"), rd.GetValue(i))); - Console.WriteLine(); - n++; - } - Console.WriteLine("{0} records", n); - } - } - } - - } -} + } + + public static void DumpSQL(this System.Data.IDbConnection self, string sql, params object[] parameters) + { + using (var c = self.CreateCommand()) + { + c.CommandText = sql; + if (parameters != null) + foreach (var p in parameters) + c.AddParameter(p); + + using (var rd = c.ExecuteReader()) + { + for (int i = 0; i < rd.FieldCount; i++) + Console.Write((i == 0 ? "" : "\t") + rd.GetName(i)); + Console.WriteLine(); + + long n = 0; + while (rd.Read()) + { + for (int i = 0; i < rd.FieldCount; i++) + Console.Write(string.Format((i == 0 ? "{0}" : "\t{0}"), rd.GetValue(i))); + Console.WriteLine(); + n++; + } + Console.WriteLine("{0} records", n); + } + } + } + + } +} diff --git a/Duplicati/Library/Main/ForestHash/Database/HashLookup.cs b/Duplicati/Library/Main/ForestHash/Database/HashLookup.cs new file mode 100644 index 000000000..3dda37419 --- /dev/null +++ b/Duplicati/Library/Main/ForestHash/Database/HashLookup.cs @@ -0,0 +1,89 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Duplicati.Library.Main.ForestHash.Database +{ + /// + /// Provides a simple hash-table like lookup, + /// similar to a MRU cache + /// + internal class HashLookup + { + private const long MAX_MEMORY = 32 * 1024 * 1024; //32mb + + /// + /// The lookup table with all values + /// + private T[] m_lookup; + + /// + /// Typecasted size of table + /// + private ulong m_modulo; + + public HashLookup(uint elementsize, ulong memorylimit = MAX_MEMORY) + { + //Allocate as a large boolean/bit array + m_lookup = new T[Math.Max(32, Math.Max(1024, memorylimit * 8) / elementsize)]; + m_modulo = (ulong)m_lookup.Length; + } + + /// + /// Adds a hash to the table + /// + /// The hash key + /// The actual hash data + public void AddHash(ulong key, T value) + { + //Textbox hashing, the value modulo number of entries + // works because the input is high quality random data + var bit = key % m_modulo; + m_lookup[bit] = value; + } + + /// + /// Queries the table to see if a hash exists. + /// A true result value is definitive, a false result value should be interpreted as maybe. + /// + /// The hash key + /// The actual hash data + /// True if the hash was found, false otherwise + public bool HashExists(ulong key, T value) + { + //Textbox hashing, the value modulo number of entries + // works because the input is high quality random data + var bit = key % m_modulo; + return object.Equals(m_lookup[bit], value); + } + + /// + /// Calculates number of used entries in the table + /// + public ulong Entries + { + get + { + ulong c = 0; + foreach (var x in m_lookup) + if (!object.Equals(x, default(T))) + c++; + + return c; + } + } + + /// + /// Gets a value between 0 and 1 descring how + /// many entries are used + /// + public double TableUsageRatio + { + get + { + return Entries / (double)m_lookup.Length; + } + } + } +} diff --git a/Duplicati/Library/Main/ForestHash/Database/HashLookupWithData.cs b/Duplicati/Library/Main/ForestHash/Database/HashLookupWithData.cs new file mode 100644 index 000000000..ce87b8e4c --- /dev/null +++ b/Duplicati/Library/Main/ForestHash/Database/HashLookupWithData.cs @@ -0,0 +1,99 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Duplicati.Library.Main.ForestHash.Database +{ + /// + /// Provides a simple hash-table like lookup, + /// similar to a MRU cache + /// + internal class HashLookupWithData + { + private const long MAX_MEMORY = 32 * 1024 * 1024; //32mb + + /// + /// The lookup table with all values + /// + private KeyValuePair[] m_lookup; + + /// + /// Typecasted size of table + /// + private ulong m_modulo; + + public HashLookupWithData(uint elementsize, ulong memorylimit = MAX_MEMORY) + { + //Allocate as a large boolean/bit array + m_lookup = new KeyValuePair[Math.Max(32, Math.Max(1024, memorylimit * 8) / elementsize)]; + m_modulo = (ulong)m_lookup.Length; + } + + /// + /// Adds a hash to the table + /// + /// The hash key + /// The actual hash data + public void AddHash(ulong key, TKey value, TValue data) + { + //Textbox hashing, the value modulo number of entries + // works because the input is high quality random data + var bit = key % m_modulo; + m_lookup[bit] = new KeyValuePair(value, data); + } + + /// + /// Queries the table to see if a hash exists. + /// A true result value is definitive, a false result value should be interpreted as maybe. + /// + /// The hash key + /// The actual hash data + /// True if the hash was found, false otherwise + public bool HashExists(ulong key, TKey value, out TValue data) + { + //Textbox hashing, the value modulo number of entries + // works because the input is high quality random data + var bit = key % m_modulo; + var el = m_lookup[bit]; + if (object.Equals(el.Key, value)) + { + data = el.Value; + return true; + } + else + { + data = default(TValue); + return false; + } + } + + /// + /// Calculates number of used entries in the table + /// + public ulong Entries + { + get + { + ulong c = 0; + foreach (var x in m_lookup) + if (!object.Equals(x.Key, default(TKey))) + c++; + + return c; + } + } + + /// + /// Gets a value between 0 and 1 descring how + /// many entries are used + /// + public double TableUsageRatio + { + get + { + return Entries / (double)m_lookup.Length; + } + } + } +} diff --git a/Duplicati/Library/Main/ForestHash/Database/HashPrefixLookup.cs b/Duplicati/Library/Main/ForestHash/Database/HashPrefixLookup.cs index 47d069551..c8d9878c0 100644 --- a/Duplicati/Library/Main/ForestHash/Database/HashPrefixLookup.cs +++ b/Duplicati/Library/Main/ForestHash/Database/HashPrefixLookup.cs @@ -29,27 +29,23 @@ namespace Duplicati.Library.Main.ForestHash.Database /// /// The maximum number of bytes to allocate for storage table /// Data to prepoulate the table with - public HashPrefixLookup(ulong memorylimit = MAX_MEMORY, IEnumerable data = null) + public HashPrefixLookup(ulong memorylimit = MAX_MEMORY) { m_bits = Math.Max(1024, memorylimit * 8); //Allocate as a large boolean/bit array m_lookup = new ulong[m_bits / 64]; - - if (data != null) - foreach (var d in data) - AddHash(d); } /// /// Adds a hash to the table /// /// The bytes representing the hash - public void AddHash(byte[] data) + public void AddHash(ulong data) { //Textbox hashing, the value modulo number of entries // works because the input is high quality random data - var bit = BitConverter.ToUInt64(data, 0) % m_bits; + var bit = data % m_bits; m_lookup[bit >> 6] |= 1uL << (ushort)(bit & 0x3f); } @@ -59,11 +55,11 @@ namespace Duplicati.Library.Main.ForestHash.Database /// /// The bytes representing the hash /// True if the hash was found, false otherwise - public bool HashExists(byte[] data) + public bool HashExists(ulong data) { //Textbox hashing, the value modulo number of entries // works because the input is high quality random data - var bit = BitConverter.ToUInt64(data, 0) % m_bits; + var bit = data % m_bits; return (m_lookup[bit >> 6] & (1uL << (ushort)(bit & 0x3f))) != 0; } @@ -95,5 +91,102 @@ namespace Duplicati.Library.Main.ForestHash.Database return (this.BitsUsed / (double)m_bits); } } + + /// + /// Hex digit ASCII lookup table + /// + private static readonly byte[] HEXTB = new byte[128]; + + /// + /// Base64 digit ASCII lookup table + /// + private static readonly byte[] BAS64TB = new byte[128]; + + /// + /// Initializes the lookup + /// + static HashPrefixLookup() + { + for(byte i = 0; i < 10; i++) + HEXTB['0' + i] = i; + + for (byte i = 0; i < 6; i++) + { + //Case insensitive support + HEXTB['A' + i] = (byte)(i + 10); + HEXTB['a' + i] = (byte)(i + 10); + } + + for (byte i = 0; i < 26; i++) + BAS64TB['A' + i] = i; + for (byte i = 0; i < 26; i++) + BAS64TB['a' + i] = (byte)(i + 26); + for (byte i = 0; i < 10; i++) + BAS64TB['0' + i] = (byte)(i + 52); + + BAS64TB['+'] = 62; + BAS64TB['/'] = 63; + + //Base64 URL support + BAS64TB['-'] = 62; + BAS64TB['_'] = 63; + } + + /// + /// Debug help function to compare values + /// + /// The value to convert + /// The converted value + public static ulong ConvertEndianness(ulong value) + { + return + ((value << 56) & 0xff00000000000000uL) | + ((value << 40) & 0x00ff000000000000uL) | + ((value << 24) & 0x0000ff0000000000uL) | + ((value << 8) & 0x000000ff00000000uL) | + ((value >> 8) & 0x00000000ff000000uL) | + ((value >> 24) & 0x0000000000ff0000uL) | + ((value >> 40) & 0x000000000000ff00uL) | + ((value >> 56) & 0x00000000000000ffuL); + } + + //Faster internal hex decoder, assumes valid hex string of at least 16 chars and no extended chars + public static ulong DecodeHexHash(string hash) + { + return + (ulong)HEXTB[hash[0]] << 60 | + (ulong)HEXTB[hash[1]] << 56 | + (ulong)HEXTB[hash[2]] << 52 | + (ulong)HEXTB[hash[3]] << 48 | + (ulong)HEXTB[hash[4]] << 44 | + (ulong)HEXTB[hash[5]] << 40 | + (ulong)HEXTB[hash[6]] << 36 | + (ulong)HEXTB[hash[7]] << 32 | + (ulong)HEXTB[hash[8]] << 28 | + (ulong)HEXTB[hash[9]] << 24 | + (ulong)HEXTB[hash[10]] << 20 | + (ulong)HEXTB[hash[11]] << 16 | + (ulong)HEXTB[hash[12]] << 12 | + (ulong)HEXTB[hash[13]] << 8 | + (ulong)HEXTB[hash[14]] << 4 | + (ulong)HEXTB[hash[15]] << 0; + } + + //Faster internal base64 decoder, assumes valid hex string of at least 11 chars and no extended chars + public static ulong DecodeBase64Hash(string hash) + { + return + (ulong)BAS64TB[hash[0]] << 58 | + (ulong)BAS64TB[hash[1]] << 52 | + (ulong)BAS64TB[hash[2]] << 46 | + (ulong)BAS64TB[hash[3]] << 40 | + (ulong)BAS64TB[hash[4]] << 34 | + (ulong)BAS64TB[hash[5]] << 28 | + (ulong)BAS64TB[hash[6]] << 22 | + (ulong)BAS64TB[hash[7]] << 16 | + (ulong)BAS64TB[hash[8]] << 10 | + (ulong)BAS64TB[hash[9]] << 4 | + (((ulong)BAS64TB[hash[10]] >> 2) & 0x0f); + } } } diff --git a/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs index 598d3e680..d563b0ebb 100644 --- a/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs @@ -8,8 +8,18 @@ namespace Duplicati.Library.Main.ForestHash.Database { public class LocalBackupDatabase : Localdatabase { + /// + /// The amount of memory allocated to store block hashes in memory (will allocate twice this memory) + /// private const ulong BLOCK_HASH_LOOKUP_SIZE = 64 * 1024 * 1024; + /// + /// The amount of memory allocated to store file in memory (will allocate twice this memory) + /// private const ulong FILE_HASH_LOOKUP_SIZE = 16 * 1024 * 1024; + /// + /// An approximate size of a hash-string in memory (44 chars * 2 for unicode + 8 bytes for pointer = 104) + /// + private const uint HASH_GUESS_SIZE = 128; private readonly System.Data.IDbCommand m_findblockCommand; private readonly System.Data.IDbCommand m_findblocksetCommand; @@ -37,13 +47,18 @@ namespace Duplicati.Library.Main.ForestHash.Database private readonly System.Data.IDbCommand m_createremotevolumeCommand; private readonly System.Data.IDbCommand m_insertfileOperationCommand; - private HashPrefixLookup m_blockHashLookup; - private HashPrefixLookup m_fileHashLookup; + private HashPrefixLookup m_blockHashPrefixLookup; + private HashPrefixLookup m_fileHashPrefixLookup; + private HashLookup m_blockHashFullLookup; + private HashLookupWithData m_fileHashFullLookup; private long m_falseBlockPositives = 0; private long m_falseFilePositives = 0; + private long m_falseBlockNegatives = 0; + private long m_falseFileNegatives = 0; private long m_missingBlockHashes; + private string m_scantimelookupTablename; public LocalBackupDatabase(string path) : this(CreateConnection(path)) @@ -62,7 +77,6 @@ namespace Duplicati.Library.Main.ForestHash.Database m_insertblockCommand = m_connection.CreateCommand(); m_insertfileCommand = m_connection.CreateCommand(); m_insertblocksetCommand = m_connection.CreateCommand(); - //m_selectfileCommand = m_connection.CreateCommand(); m_insertmetadatasetCommand = m_connection.CreateCommand(); m_findblocksetCommand = m_connection.CreateCommand(); m_findmetadatasetCommand = m_connection.CreateCommand(); @@ -115,7 +129,13 @@ namespace Duplicati.Library.Main.ForestHash.Database m_insertmetadatasetCommand.CommandText = @"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (?); SELECT last_insert_rowid();"; m_insertmetadatasetCommand.AddParameter(); - m_selectfileSimpleCommand.CommandText = @"SELECT ""OperationFileset"".""FilesetID"", ""OperationFileset"".""Scantime"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""Fileset"".""Path"" = ? ORDER BY ""OperationFileset"".""Scantime"" DESC"; + //Need a temporary table with path/scantime lookups + m_scantimelookupTablename = "ScanTime-" + Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + using (var cmd = m_connection.CreateCommand()) + cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT ""OperationFileset"".""FilesetID"" AS ""FilesetID"", MAX(""OperationFileset"".""Scantime"") AS ""Scantime"", ""Fileset"".""Path"" AS ""Path"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" GROUP BY ""FilesetID"", ""Path"" ", m_scantimelookupTablename)); + + m_selectfileSimpleCommand.CommandText = string.Format(@"SELECT ""FilesetID"", ""Scantime"" FROM ""{0}"" WHERE ""Path"" = ?", m_scantimelookupTablename); + //m_selectfileSimpleCommand.CommandText = @"SELECT ""OperationFileset"".""FilesetID"", ""OperationFileset"".""Scantime"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""Fileset"".""Path"" = ? ORDER BY ""OperationFileset"".""Scantime"" DESC"; m_selectfileSimpleCommand.AddParameters(1); m_selectfileHashCommand.CommandText = @"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"" INNER JOIN ""Fileset"" ON ""Blockset"".""ID"" = ""Fileset"".""BlocksetID"" WHERE ""Fileset"".""ID"" = ? "; @@ -134,18 +154,32 @@ namespace Duplicati.Library.Main.ForestHash.Database m_createremotevolumeCommand.AddParameter(m_operationid); m_createremotevolumeCommand.AddParameters(3); - m_blockHashLookup = new HashPrefixLookup(BLOCK_HASH_LOOKUP_SIZE); - m_fileHashLookup = new HashPrefixLookup(FILE_HASH_LOOKUP_SIZE); + m_blockHashPrefixLookup = new HashPrefixLookup(BLOCK_HASH_LOOKUP_SIZE); + m_fileHashPrefixLookup = new HashPrefixLookup(FILE_HASH_LOOKUP_SIZE); + + m_blockHashFullLookup = new HashLookup(HASH_GUESS_SIZE, BLOCK_HASH_LOOKUP_SIZE); + m_fileHashFullLookup = new HashLookupWithData(HASH_GUESS_SIZE, FILE_HASH_LOOKUP_SIZE); using (var cmd = m_connection.CreateCommand()) using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Block"".""Hash"" FROM ""Block"", ""RemoteVolume"" WHERE ""RemoteVolume"".""Name"" = ""Block"".""File"" AND ""RemoteVolume"".""State"" IN (?,?,?,?) ", RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString())) while (rd.Read()) - m_blockHashLookup.AddHash(Convert.FromBase64String(rd.GetValue(0).ToString())); + { + var str = rd.GetValue(0).ToString(); + var key = HashPrefixLookup.DecodeBase64Hash(str); + m_blockHashPrefixLookup.AddHash(key); + m_blockHashFullLookup.AddHash(key, str); + } using (var cmd = m_connection.CreateCommand()) - using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""FullHash"" FROM ""BlockSet""")) + using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""FullHash"", ""ID"" FROM ""BlockSet""")) while (rd.Read()) - m_fileHashLookup.AddHash(Convert.FromBase64String(rd.GetValue(0).ToString())); + { + var str = rd.GetValue(0).ToString(); + var id = Convert.ToInt64(rd.GetValue(1)); + var key = HashPrefixLookup.DecodeBase64Hash(str); + m_fileHashPrefixLookup.AddHash(key); + m_fileHashFullLookup.AddHash(key, str, id); + } using (var cmd = m_connection.CreateCommand()) m_missingBlockHashes = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT (*) FROM (SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"" FROM ""Block"", ""RemoteVolume"" WHERE ""RemoteVolume"".""Name"" = ""Block"".""File"" AND ""RemoteVolume"".""State"" NOT IN (?,?,?,?))", RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString())); @@ -161,9 +195,12 @@ namespace Duplicati.Library.Main.ForestHash.Database public bool AddBlock(string key, long size, string archivename, System.Data.IDbTransaction transaction = null) { object r = null; - var hashdata = Convert.FromBase64String(key); - if (m_blockHashLookup.HashExists(hashdata)) + var hashdata = HashPrefixLookup.DecodeBase64Hash(key); + if (m_blockHashPrefixLookup.HashExists(hashdata)) { + if (m_missingBlockHashes == 0 && m_blockHashFullLookup.HashExists(hashdata, key)) + return false; + m_findblockCommand.Transaction = transaction; r = m_findblockCommand.ExecuteScalar(null, key, size); if (r == null || r == DBNull.Value) @@ -172,8 +209,12 @@ namespace Duplicati.Library.Main.ForestHash.Database } else { + m_falseBlockNegatives++; if (m_missingBlockHashes == 0) + { + m_blockHashFullLookup.AddHash(hashdata, key); return false; + } } } @@ -184,11 +225,15 @@ namespace Duplicati.Library.Main.ForestHash.Database m_insertblockCommand.SetParameterValue(1, archivename); m_insertblockCommand.SetParameterValue(2, size); m_insertblockCommand.ExecuteNonQuery(); - m_blockHashLookup.AddHash(hashdata); + m_blockHashPrefixLookup.AddHash(hashdata); + m_blockHashFullLookup.AddHash(hashdata, key); return true; } else { + //We add/update it now + m_blockHashFullLookup.AddHash(hashdata, key); + //If the block is found and the volume is broken somehow. m_findremotevolumestateCommand.Transaction = transaction; r = m_findremotevolumestateCommand.ExecuteScalar(null, r); @@ -220,20 +265,24 @@ namespace Duplicati.Library.Main.ForestHash.Database public bool AddBlockset(string filehash, long size, int blocksize, IEnumerable hashes, IEnumerable blocklistHashes, out long blocksetid, System.Data.IDbTransaction transaction = null) { object r = null; - var hashdata = Convert.FromBase64String(filehash); - - if (m_fileHashLookup.HashExists(hashdata)) + var hashdata = HashPrefixLookup.DecodeBase64Hash(filehash); + if (m_fileHashPrefixLookup.HashExists(hashdata)) { + if (m_fileHashFullLookup.HashExists(hashdata, filehash, out blocksetid)) + return false; + m_findblocksetCommand.Transaction = transaction; r = m_findblocksetCommand.ExecuteScalar(null, filehash, size); if (r == null || r == DBNull.Value) m_falseFilePositives++; + else + m_falseFileNegatives++; } if (r != null && r != DBNull.Value) { blocksetid = Convert.ToInt64(r); - //TODO: Make a sweep to see if the blocks match the record? + m_fileHashFullLookup.AddHash(hashdata, filehash, blocksetid); return false; } @@ -241,7 +290,8 @@ namespace Duplicati.Library.Main.ForestHash.Database { m_insertblocksetCommand.Transaction = tr.Parent; blocksetid = Convert.ToInt64(m_insertblocksetCommand.ExecuteScalar(null, size, filehash)); - m_fileHashLookup.AddHash(hashdata); + m_fileHashPrefixLookup.AddHash(hashdata); + m_fileHashFullLookup.AddHash(hashdata, filehash, blocksetid); long ix = 0; if (blocklistHashes != null) @@ -555,6 +605,22 @@ namespace Duplicati.Library.Main.ForestHash.Database } } + public override void Dispose () + { + if (m_scantimelookupTablename != null) + try + { + using(var cmd = m_connection.CreateCommand()) + cmd.ExecuteNonQuery(string.Format(@"DROP TABLE ""{0}"" ", m_scantimelookupTablename)); + } + finally + { + m_scantimelookupTablename = null; + } + + base.Dispose(); + } + internal void UpdateChangeStatistics (BackupStatistics m_stat) { using (var cmd = m_connection.CreateCommand()) @@ -576,22 +642,26 @@ namespace Duplicati.Library.Main.ForestHash.Database m_stat.ModifiedFolders = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" = ? AND ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, m_operationid)); m_stat.ModifiedSymlinks = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" = ? AND ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, SYMLINK_BLOCKSET_ID, m_operationid)); m_stat.ModifiedFiles = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" != ? AND ""Fileset"".""BlocksetID"" != ? AND ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, m_operationid)); - - /*m_stat.ModifiedFolders = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT * FROM ( (SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""Fileset"".""ID"" WHERE ""OperationID"" = ?) AS A, (SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""Fileset"".""ID"" WHERE ""OperationID"" = ?) AS B WHERE ""A"".""Path"" = ""B"".""Path"" AND ""A"".""BlocsetID"" != ? AND ""A"".""MetadataID"" != ""B"".""MetadataID"" )", m_operationid, lastOperationId, FOLDER_BLOCKSET_ID)); - m_stat.ModifiedSymlinks = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT * FROM ( (SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""Fileset"".""ID"" WHERE ""OperationID"" = ?) AS A, (SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""Fileset"".""ID"" WHERE ""OperationID"" = ?) AS B WHERE ""A"".""Path"" = ""B"".""Path"" AND ""A"".""BlocsetID"" != ? AND ""A"".""MetadataID"" != ""B"".""MetadataID"" )", m_operationid, lastOperationId, SYMLINK_BLOCKSET_ID)); - m_stat.ModifiedFiles = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT * FROM ( (SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""Fileset"".""ID"" WHERE ""OperationID"" = ?) AS A, (SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""Fileset"".""ID"" WHERE ""OperationID"" = ?) AS B WHERE ""A"".""Path"" = ""B"".""Path"" AND (""A"".""BlocsetID"" != ""B"".""BlocksetID"" OR ""A"".""MetadataID"" != ""B"".""MetadataID"") )", m_operationid, lastOperationId)); - * */ } if (m_falseBlockPositives > 200 || m_falseFilePositives > 20) { m_stat.LogWarning(string.Format("Lookup tables gave false positives, this may indicate too small tables. Block: {0}, File: {0}", m_falseBlockPositives, m_falseFilePositives), null); - if (m_blockHashLookup.TableUsageRatio > 0.5) - m_stat.LogWarning(string.Format("Block hash lookup table is too small, usage is: {0}%", m_blockHashLookup.TableUsageRatio * 100), null); + if (m_blockHashPrefixLookup.TableUsageRatio > 0.5) + m_stat.LogWarning(string.Format("Block hash lookup table is too small, usage is: {0}%", m_blockHashPrefixLookup.TableUsageRatio * 100), null); - if (m_fileHashLookup.TableUsageRatio > 0.5) - m_stat.LogWarning(string.Format("File hash lookup table is too small, usage is: {0}%", m_fileHashLookup.TableUsageRatio * 100), null); + if (m_fileHashPrefixLookup.TableUsageRatio > 0.5) + m_stat.LogWarning(string.Format("File hash lookup table is too small, usage is: {0}%", m_fileHashPrefixLookup.TableUsageRatio * 100), null); + } + + // Good for profiling, but takes some time to calculate these stats + if (Logging.Log.LogLevel == Logging.LogMessageType.Profiling) + { + Logging.Log.WriteMessage(string.Format("Prefix BlockHash Lookup entries {0}, usage: {1}, falsePositives: {2}", m_blockHashPrefixLookup.BitsUsed, m_blockHashPrefixLookup.TableUsageRatio, m_falseBlockPositives), Logging.LogMessageType.Profiling); + Logging.Log.WriteMessage(string.Format("Prefix BlockHash Lookup entries {0}, usage: {1}, falsePositives: {2}", m_fileHashPrefixLookup.BitsUsed, m_fileHashPrefixLookup.TableUsageRatio, m_falseFilePositives), Logging.LogMessageType.Profiling); + Logging.Log.WriteMessage(string.Format("Full BlockHash Lookup entries {0}, usage: {1}, falseNegatives: {2}", m_blockHashFullLookup.Entries, m_blockHashFullLookup.TableUsageRatio, m_falseBlockNegatives), Logging.LogMessageType.Profiling); + Logging.Log.WriteMessage(string.Format("Full FileHash Lookup entries {0}, usage: {1}, falseNegatives: {2}", m_fileHashFullLookup.Entries, m_fileHashFullLookup.TableUsageRatio, m_falseFileNegatives), Logging.LogMessageType.Profiling); } }