diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index d68a87c9e..606a90094 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -83,9 +83,6 @@ namespace Duplicati.Library.Main.Database private readonly System.Data.IDbCommand m_insertfileOperationCommand; - private HashLookupHelper> m_blockHashLookup; - private HashLookupHelper m_fileHashLookup; - private HashLookupHelper m_metadataLookup; private PathLookupHelper m_pathLookup; private long m_missingBlockHashes; @@ -178,12 +175,6 @@ namespace Duplicati.Library.Main.Database /// The option settings public void BuildLookupTable(Options options) { - if (options.BlockHashLookupMemory > 0) - m_blockHashLookup = new HashLookupHelper>((ulong)options.BlockHashLookupMemory); - if (options.FileHashLookupMemory > 0) - m_fileHashLookup = new HashLookupHelper((ulong)options.FileHashLookupMemory); - if (options.MetadataHashMemory > 0) - m_metadataLookup = new HashLookupHelper((ulong)options.MetadataHashMemory); if (options.UseFilepathCache) m_pathLookup = new PathLookupHelper(true); @@ -206,61 +197,6 @@ namespace Duplicati.Library.Main.Database @" AND ""A1"".""MetadataID"" = ""E"".""ID"" " + @" AND ""F"".""ID"" = ""E"".""BlocksetID"" "; - if (m_blockHashLookup != null) - try - { - using(new Logging.Timer("Build blockhash lookup table")) - using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Block"".""Hash"", ""Block"".""ID"", ""Block"".""Size"" FROM ""Block"" ")) - while (rd.Read()) - { - var str = rd.GetValue(0).ToString(); - var id = rd.GetInt64(1); - var size = rd.GetInt64(2); - m_blockHashLookup.Add(str, size, new KeyValuePair(id, size)); - } - } - catch (Exception ex) - { - throw new InvalidDataException("Duplicate blockhashes detected, either repair the database or rebuild it", ex); - } - - if (m_fileHashLookup != null) - try - { - using(new Logging.Timer("Build filehash lookup table")) - using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""FullHash"", ""Length"", ""ID"" FROM ""BlockSet""")) - while (rd.Read()) - { - var str = rd.GetValue(0).ToString(); - var size = rd.GetInt64(1); - var id = rd.GetInt64(2); - m_fileHashLookup.Add(str, size, id); - } - } - catch (Exception ex) - { - throw new InvalidDataException("Duplicate filehashes detected, either repair the database or rebuild it", ex); - } - - - if (m_metadataLookup != null) - try - { - using(new Logging.Timer("Build metahash lookup table")) - using (var rd = cmd.ExecuteReader(@"SELECT ""Metadataset"".""ID"", ""Blockset"".""FullHash"", ""Blockset"".""Length"" FROM ""Metadataset"", ""Blockset"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" ")) - while (rd.Read()) - { - var metadataid = rd.GetInt64(0); - var hash = rd.GetValue(1).ToString(); - var size = rd.GetInt64(2);; - m_metadataLookup.Add(hash, size, metadataid); - } - } - catch (Exception ex) - { - throw new InvalidDataException("Duplicate metadatahash detected, run repair to fix it", ex); - } - if (m_pathLookup != null) using(new Logging.Timer("Build path lastmodified lookup table")) using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""FileID"", ""Lastmodified"", ""Length"", ""Path"", ""Metahash"", ""Metasize"" FROM ({0}) WHERE ""BlocksetID"" >= 0 ", scantableDefinition))) @@ -320,20 +256,10 @@ namespace Duplicati.Library.Main.Database /// True if the block should be added to the current output public bool AddBlock (string key, long size, long volumeid, System.Data.IDbTransaction transaction = null) { - var r = -1L; - if (m_blockHashLookup != null) - { - KeyValuePair blockid; - if (m_blockHashLookup.TryGet(key, size, out blockid)) - return false; - } - else - { - m_findblockCommand.Transaction = transaction; - m_findblockCommand.SetParameterValue(0, key); - m_findblockCommand.SetParameterValue(1, size); - r = m_findblockCommand.ExecuteScalarInt64(-1); - } + m_findblockCommand.Transaction = transaction; + m_findblockCommand.SetParameterValue(0, key); + m_findblockCommand.SetParameterValue(1, size); + var r = m_findblockCommand.ExecuteScalarInt64(-1); if (r == -1L) { @@ -342,16 +268,11 @@ namespace Duplicati.Library.Main.Database m_insertblockCommand.SetParameterValue(1, volumeid); m_insertblockCommand.SetParameterValue(2, size); r = m_insertblockCommand.ExecuteScalarInt64(); - if (m_blockHashLookup != null) - m_blockHashLookup.Add(key, size, new KeyValuePair(r, size)); return true; } else { //Update lookup cache if required - if (m_blockHashLookup != null) - m_blockHashLookup.Add(key, size, new KeyValuePair(r, size)); - return false; } } @@ -369,27 +290,17 @@ namespace Duplicati.Library.Main.Database /// True if the blockset was created, false otherwise public bool AddBlockset(string filehash, long size, int blocksize, IEnumerable hashes, IEnumerable blocklistHashes, out long blocksetid, System.Data.IDbTransaction transaction = null) { - if (m_fileHashLookup != null) - { - if (m_fileHashLookup.TryGet(filehash, size, out blocksetid)) - return false; - } - else - { - m_findblocksetCommand.Transaction = transaction; - blocksetid = m_findblocksetCommand.ExecuteScalarInt64(null, -1, filehash, size); - if (blocksetid != -1) - return false; //Found it - } - + m_findblocksetCommand.Transaction = transaction; + blocksetid = m_findblocksetCommand.ExecuteScalarInt64(null, -1, filehash, size); + if (blocksetid != -1) + return false; //Found it + using(var tr = new TemporaryTransactionWrapper(m_connection, transaction)) { m_insertblocksetCommand.Transaction = tr.Parent; m_insertblocksetCommand.SetParameterValue(0, size); m_insertblocksetCommand.SetParameterValue(1, filehash); blocksetid = m_insertblocksetCommand.ExecuteScalarInt64(); - if (m_fileHashLookup != null) - m_fileHashLookup.Add(filehash, size, blocksetid); long ix = 0; if (blocklistHashes != null) @@ -417,19 +328,6 @@ namespace Duplicati.Library.Main.Database { var exsize = remainsize < blocksize ? remainsize : blocksize; var found = false; - if (m_blockHashLookup != null) - { - KeyValuePair id; - if (m_blockHashLookup.TryGet(h, exsize, out id) && id.Value == exsize) - { - m_insertblocksetentryFastCommand.SetParameterValue(1, ix); - m_insertblocksetentryFastCommand.SetParameterValue(2, id.Key); - var cx = m_insertblocksetentryFastCommand.ExecuteNonQuery(); - if (cx != 1) - throw new Exception(string.Format("Unexpected result count: {0}, expected {1}", cx, 1)); - found = true; - } - } if (!found) { @@ -462,18 +360,10 @@ namespace Duplicati.Library.Main.Database { if (size > 0) { - if (m_metadataLookup != null) - { - if(m_metadataLookup.TryGet(filehash, size, out metadataid)) - return false; - } - else - { - m_findmetadatasetCommand.Transaction = transaction; - metadataid = m_findmetadatasetCommand.ExecuteScalarInt64(null, -1, filehash, size); - if (metadataid != -1) - return false; - } + m_findmetadatasetCommand.Transaction = transaction; + metadataid = m_findmetadatasetCommand.ExecuteScalarInt64(null, -1, filehash, size); + if (metadataid != -1) + return false; long blocksetid; @@ -485,8 +375,6 @@ namespace Duplicati.Library.Main.Database m_insertmetadatasetCommand.SetParameterValue(0, blocksetid); metadataid = m_insertmetadatasetCommand.ExecuteScalarInt64(); tr.Commit(); - if (m_metadataLookup != null) - m_metadataLookup.Add(filehash, size, metadataid); } return true; @@ -645,9 +533,6 @@ namespace Duplicati.Library.Main.Database public override void Dispose () { - m_fileHashLookup = null; - m_metadataLookup = null; - m_blockHashLookup = null; m_pathLookup = null; base.Dispose(); diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index 600f04cb1..f5d95e524 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -273,41 +273,19 @@ namespace Duplicati.Library.Main.Database private class BlockQuery : IBlockQuery { private System.Data.IDbCommand m_command; - private HashLookupHelper m_lookup; - + public BlockQuery(System.Data.IDbConnection con, Options options, System.Data.IDbTransaction transaction) { m_command = con.CreateCommand(); m_command.Transaction = transaction; - - if (options.BlockHashLookupMemory > 0) - { - m_lookup = new HashLookupHelper((ulong)options.BlockHashLookupMemory); - using(var reader = m_command.ExecuteReader(@"SELECT ""Hash"", ""Size"" FROM ""Block"" ")) - while (reader.Read()) - { - var hash = reader.GetString(0); - var size = reader.GetInt64(1); - m_lookup.Add(hash, size, size); - } - } - + m_command.Parameters.Clear(); m_command.CommandText = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ? "; m_command.AddParameters(2); } public bool UseBlock(string hash, long size, System.Data.IDbTransaction transaction) - { - if (m_lookup != null) - { - long nsize; - if(m_lookup.TryGet(hash, size, out nsize) && nsize == size) - return true; - else - return false; - } - + { m_command.Transaction = transaction; m_command.SetParameterValue(0, hash); m_command.SetParameterValue(1, size); @@ -317,7 +295,6 @@ namespace Duplicati.Library.Main.Database public void Dispose() { - m_lookup = null; if (m_command != null) try { m_command.Dispose(); } finally { m_command = null; } diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index 0353902c5..db8f7b1b6 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -60,10 +60,6 @@ namespace Duplicati.Library.Main.Database private System.Data.IDbCommand m_insertBlockCommand; private System.Data.IDbCommand m_insertDuplicateBlockCommand; - private HashLookupHelper m_blockListHashLookup; - private HashLookupHelper m_blockHashLookup; - private HashLookupHelper m_fileHashLookup; - private HashLookupHelper m_metadataLookup; private PathLookupHelper m_filesetLookup; private string m_tempblocklist; @@ -197,15 +193,6 @@ namespace Duplicati.Library.Main.Database m_insertDuplicateBlockCommand.CommandText = @"INSERT INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") VALUES ((SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?), ?)"; m_insertDuplicateBlockCommand.AddParameters(3); - if (options.BlockHashLookupMemory > 0) - { - m_blockHashLookup = new HashLookupHelper((ulong)options.BlockHashLookupMemory/2); - m_blockListHashLookup = new HashLookupHelper((ulong)options.BlockHashLookupMemory/2); - } - if (options.FileHashLookupMemory > 0) - m_fileHashLookup = new HashLookupHelper((ulong)options.FileHashLookupMemory); - if (options.MetadataHashMemory > 0) - m_metadataLookup = new HashLookupHelper((ulong)options.MetadataHashMemory); if (options.UseFilepathCache) m_filesetLookup = new PathLookupHelper(); } @@ -247,18 +234,6 @@ namespace Duplicati.Library.Main.Database // Insert all known blocks into block table with volumeid = -1 cmd.ExecuteNonQuery(insertBlocksCommand); - // Update the cache with new blocks - if (m_blockHashLookup != null) - { - using(var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Hash"", ""Size"" FROM ""Block"" WHERE ""VolumeID"" = -1 ")) - while(rd.Read()) - { - var hash = rd.GetString(0); - var size = rd.GetInt64(1); - m_blockHashLookup.TryAdd(hash, size, -1); - } - } - var selectBlocklistBlocksetEntries = string.Format( @"SELECT ""E"".""BlocksetID"" AS ""BlocksetID"", ""D"".""FullIndex"" AS ""Index"", ""F"".""ID"" AS ""BlockID"" FROM ( " + SELECT_BLOCKLIST_ENTRIES + @@ -268,8 +243,6 @@ namespace Duplicati.Library.Main.Database m_tempblocklist, blocksize / hashsize ); - - var selectBlocksetEntries = string.Format( @"SELECT ""Blockset"".""ID"" AS ""BlocksetID"", 0 AS ""Index"", ""Block"".""ID"" AS ""BlockID"" FROM ""Blockset"", ""Block"", ""{1}"" S WHERE ""Blockset"".""Fullhash"" = ""S"".""FileHash"" AND ""S"".""BlockHash"" = ""Block"".""Hash"" AND ""S"".""BlockSize"" = ""Block"".""Size"" AND ""Blockset"".""Length"" = ""S"".""BlockSize"" AND ""Blockset"".""Length"" <= {0} ", @@ -362,63 +335,36 @@ namespace Duplicati.Library.Main.Database if (metahash == null) return metadataid; - if (m_metadataLookup != null) - { - if (m_metadataLookup.TryGet(metahash, metahashsize, out metadataid)) - return metadataid; - else - metadataid = -1; - } - else - { - m_findMetadatasetCommand.Transaction = transaction; - m_findMetadatasetCommand.SetParameterValue(0, metahash); - m_findMetadatasetCommand.SetParameterValue(1, metahashsize); - metadataid = m_findMetadatasetCommand.ExecuteScalarInt64(-1); - if (metadataid != -1) - return metadataid; - } + m_findMetadatasetCommand.Transaction = transaction; + m_findMetadatasetCommand.SetParameterValue(0, metahash); + m_findMetadatasetCommand.SetParameterValue(1, metahashsize); + metadataid = m_findMetadatasetCommand.ExecuteScalarInt64(-1); + if (metadataid != -1) + return metadataid; var blocksetid = AddBlockset(metahash, metahashsize, metablocklisthashes, expectedmetablocklisthashes, transaction); m_insertMetadatasetCommand.Transaction = transaction; m_insertMetadatasetCommand.SetParameterValue(0, blocksetid); metadataid = m_insertMetadatasetCommand.ExecuteScalarInt64(-1); - - if (m_metadataLookup != null) - m_metadataLookup.Add(metahash, metahashsize, metadataid); - + return metadataid; } public long AddBlockset(string fullhash, long size, IEnumerable blocklisthashes, long expectedblocklisthashes, System.Data.IDbTransaction transaction) { - var blocksetid = -1L; - if (m_fileHashLookup != null) - { - if (m_fileHashLookup.TryGet(fullhash, size, out blocksetid)) - return blocksetid; - else - blocksetid = -1; - } - else - { - m_findBlocksetCommand.Transaction = transaction; - m_findBlocksetCommand.SetParameterValue(0, size); - m_findBlocksetCommand.SetParameterValue(1, fullhash); - blocksetid = m_findBlocksetCommand.ExecuteScalarInt64(-1); - if (blocksetid != -1) - return blocksetid; - } + m_findBlocksetCommand.Transaction = transaction; + m_findBlocksetCommand.SetParameterValue(0, size); + m_findBlocksetCommand.SetParameterValue(1, fullhash); + var blocksetid = m_findBlocksetCommand.ExecuteScalarInt64(-1); + if (blocksetid != -1) + return blocksetid; m_insertBlocksetCommand.Transaction = transaction; m_insertBlocksetCommand.SetParameterValue(0, size); m_insertBlocksetCommand.SetParameterValue(1, fullhash); blocksetid = m_insertBlocksetCommand.ExecuteScalarInt64(-1); - if (m_fileHashLookup != null) - m_fileHashLookup.Add(fullhash, size, blocksetid); - long c = 0; if (blocklisthashes != null) { @@ -449,20 +395,11 @@ namespace Duplicati.Library.Main.Database public bool UpdateBlock(string hash, long size, long volumeID, System.Data.IDbTransaction transaction) { - var currentVolumeId = -2L; - if (m_blockHashLookup != null) - { - if (!m_blockHashLookup.TryGet(hash, size, out currentVolumeId)) - currentVolumeId = -2; - } - else - { - m_findHashBlockCommand.Transaction = transaction; - m_findHashBlockCommand.SetParameterValue(0, hash); - m_findHashBlockCommand.SetParameterValue(1, size); - currentVolumeId = m_findHashBlockCommand.ExecuteScalarInt64(-2); - } - + m_findHashBlockCommand.Transaction = transaction; + m_findHashBlockCommand.SetParameterValue(0, hash); + m_findHashBlockCommand.SetParameterValue(1, size); + var currentVolumeId = m_findHashBlockCommand.ExecuteScalarInt64(-2); + if (currentVolumeId == volumeID) return false; @@ -475,9 +412,6 @@ namespace Duplicati.Library.Main.Database m_insertBlockCommand.SetParameterValue(2, volumeID); m_insertBlockCommand.ExecuteNonQuery(); - if (m_blockHashLookup != null) - m_blockHashLookup.Add(hash, size, volumeID); - return true; } else if (currentVolumeId == -1) @@ -491,9 +425,6 @@ namespace Duplicati.Library.Main.Database if (c != 1) throw new Exception(string.Format("Failed to update table, found {0} entries for key {1} with size {2}", c ,hash, size)); - if (m_blockHashLookup != null) - m_blockHashLookup.Add(hash, size, volumeID); - return true; } else @@ -519,23 +450,11 @@ namespace Duplicati.Library.Main.Database public bool UpdateBlockset(string hash, IEnumerable blocklisthashes, System.Data.IDbTransaction transaction) { - if (m_blockListHashLookup != null) - { - bool b; - if (m_blockListHashLookup.TryGet(hash, -1, out b)) - return false; - } - else - { - m_findblocklisthashCommand.Transaction = transaction; - m_findblocklisthashCommand.SetParameterValue(0, hash); - var r = m_findblocklisthashCommand.ExecuteScalar(); - if (r != null && r != DBNull.Value) - return false; - } - - if (m_blockListHashLookup != null) - m_blockListHashLookup.Add(hash, -1, false); + m_findblocklisthashCommand.Transaction = transaction; + m_findblocklisthashCommand.SetParameterValue(0, hash); + var r = m_findblocklisthashCommand.ExecuteScalar(); + if (r != null && r != DBNull.Value) + return false; m_insertBlockset.Transaction = transaction; m_insertBlockset.SetParameterValue(0, hash); diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index 88eb42596..889b51d95 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -474,10 +474,6 @@ namespace Duplicati.Library.Main new CommandLineArgument("skip-metadata", CommandLineArgument.ArgumentType.Boolean, Strings.Options.SkipmetadataShort, Strings.Options.SkipmetadataLong, "false"), new CommandLineArgument("restore-permissions", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RestorepermissionsShort, Strings.Options.RestorepermissionsLong, "false"), new CommandLineArgument("skip-restore-verification", CommandLineArgument.ArgumentType.Boolean, Strings.Options.SkiprestoreverificationShort, Strings.Options.SkiprestoreverificationLong, "false"), - new CommandLineArgument("blockhash-lookup-memory", CommandLineArgument.ArgumentType.Size, Strings.Options.BlockhashlookupsizeShort, Strings.Options.BlockhashlookupsizeLong, "0mb"), - new CommandLineArgument("filehash-lookup-memory", CommandLineArgument.ArgumentType.Size, Strings.Options.FilehashlookupsizeShort, Strings.Options.FilehashlookupsizeLong, "0mb"), - new CommandLineArgument("metadatahash-lookup-memory", CommandLineArgument.ArgumentType.Size, Strings.Options.MetadatahashlookupsizeShort, Strings.Options.MetadatahashlookupsizeLong, "0mb"), - new CommandLineArgument("old-lookup-memory-defaults", CommandLineArgument.ArgumentType.Size, Strings.Options.OldmemorylookupdefaultsShort, Strings.Options.OldmemorylookupdefaultsLong, "false"), new CommandLineArgument("disable-filepath-cache", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablefilepathcacheShort, Strings.Options.DisablefilepathcacheLong, "true"), new CommandLineArgument("changed-files", CommandLineArgument.ArgumentType.Path, Strings.Options.ChangedfilesShort, Strings.Options.ChangedfilesLong), new CommandLineArgument("deleted-files", CommandLineArgument.ArgumentType.Path, Strings.Options.DeletedfilesShort, Strings.Options.DeletedfilesLong("changed-files")), @@ -1413,61 +1409,6 @@ namespace Duplicati.Library.Main get { return !Library.Utility.Utility.ParseBoolOption(m_options, "skip-restore-verification"); } } - /// - /// Gets a flag indicating whether this old memory defaults. - /// - public bool OldMemoryDefaults - { - get { return Library.Utility.Utility.ParseBoolOption(m_options, "old-lookup-memory-defaults"); } - } - - /// - /// Gets the block hash lookup size - /// - public long BlockHashLookupMemory - { - get - { - string v; - m_options.TryGetValue("blockhash-lookup-memory", out v); - if (string.IsNullOrEmpty(v)) - v = OldMemoryDefaults ? DEFAULT_BLOCK_HASH_LOOKUP_SIZE : "0"; - - return Library.Utility.Sizeparser.ParseSize(v, "mb"); - } - } - - /// - /// Gets the file hash size - /// - public long FileHashLookupMemory - { - get - { - string v; - m_options.TryGetValue("filehash-lookup-memory", out v); - if (string.IsNullOrEmpty(v)) - v = OldMemoryDefaults ? DEFAULT_FILE_HASH_LOOKUP_SIZE : "0"; - - return Library.Utility.Sizeparser.ParseSize(v, "mb"); - } - } - - /// - /// Gets the block hash size - /// - public long MetadataHashMemory - { - get - { - string v; - m_options.TryGetValue("metadatahash-lookup-memory", out v); - if (string.IsNullOrEmpty(v)) - v = OldMemoryDefaults ? DEFAULT_METADATA_HASH_LOOKUP_SIZE : "0"; - - return Library.Utility.Sizeparser.ParseSize(v, "mb"); - } - } /// /// Gets the file hash size @@ -1476,15 +1417,9 @@ namespace Duplicati.Library.Main { get { - - if (OldMemoryDefaults) - return !Library.Utility.Utility.ParseBoolOption(m_options, "disable-filepath-cache"); - else - { - string s; - m_options.TryGetValue("disable-filepath-cache", out s); - return !Library.Utility.Utility.ParseBool(s, true); - } + string s; + m_options.TryGetValue("disable-filepath-cache", out s); + return !Library.Utility.Utility.ParseBool(s, true); } }