Removed unused memory cache code

This commit is contained in:
Kenneth Skovhede
2016-12-16 17:30:39 +01:00
parent 6161096c76
commit d8972b0f2b
4 changed files with 42 additions and 326 deletions
@@ -83,9 +83,6 @@ namespace Duplicati.Library.Main.Database
private readonly System.Data.IDbCommand m_insertfileOperationCommand;
private HashLookupHelper<KeyValuePair<long, long>> m_blockHashLookup;
private HashLookupHelper<long> m_fileHashLookup;
private HashLookupHelper<long> m_metadataLookup;
private PathLookupHelper<PathEntryKeeper> m_pathLookup;
private long m_missingBlockHashes;
@@ -178,12 +175,6 @@ namespace Duplicati.Library.Main.Database
/// <param name="options">The option settings</param>
public void BuildLookupTable(Options options)
{
if (options.BlockHashLookupMemory > 0)
m_blockHashLookup = new HashLookupHelper<KeyValuePair<long, long>>((ulong)options.BlockHashLookupMemory);
if (options.FileHashLookupMemory > 0)
m_fileHashLookup = new HashLookupHelper<long>((ulong)options.FileHashLookupMemory);
if (options.MetadataHashMemory > 0)
m_metadataLookup = new HashLookupHelper<long>((ulong)options.MetadataHashMemory);
if (options.UseFilepathCache)
m_pathLookup = new PathLookupHelper<PathEntryKeeper>(true);
@@ -206,61 +197,6 @@ namespace Duplicati.Library.Main.Database
@" AND ""A1"".""MetadataID"" = ""E"".""ID"" " +
@" AND ""F"".""ID"" = ""E"".""BlocksetID"" ";
if (m_blockHashLookup != null)
try
{
using(new Logging.Timer("Build blockhash lookup table"))
using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Block"".""Hash"", ""Block"".""ID"", ""Block"".""Size"" FROM ""Block"" "))
while (rd.Read())
{
var str = rd.GetValue(0).ToString();
var id = rd.GetInt64(1);
var size = rd.GetInt64(2);
m_blockHashLookup.Add(str, size, new KeyValuePair<long, long>(id, size));
}
}
catch (Exception ex)
{
throw new InvalidDataException("Duplicate blockhashes detected, either repair the database or rebuild it", ex);
}
if (m_fileHashLookup != null)
try
{
using(new Logging.Timer("Build filehash lookup table"))
using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""FullHash"", ""Length"", ""ID"" FROM ""BlockSet"""))
while (rd.Read())
{
var str = rd.GetValue(0).ToString();
var size = rd.GetInt64(1);
var id = rd.GetInt64(2);
m_fileHashLookup.Add(str, size, id);
}
}
catch (Exception ex)
{
throw new InvalidDataException("Duplicate filehashes detected, either repair the database or rebuild it", ex);
}
if (m_metadataLookup != null)
try
{
using(new Logging.Timer("Build metahash lookup table"))
using (var rd = cmd.ExecuteReader(@"SELECT ""Metadataset"".""ID"", ""Blockset"".""FullHash"", ""Blockset"".""Length"" FROM ""Metadataset"", ""Blockset"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "))
while (rd.Read())
{
var metadataid = rd.GetInt64(0);
var hash = rd.GetValue(1).ToString();
var size = rd.GetInt64(2);;
m_metadataLookup.Add(hash, size, metadataid);
}
}
catch (Exception ex)
{
throw new InvalidDataException("Duplicate metadatahash detected, run repair to fix it", ex);
}
if (m_pathLookup != null)
using(new Logging.Timer("Build path lastmodified lookup table"))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""FileID"", ""Lastmodified"", ""Length"", ""Path"", ""Metahash"", ""Metasize"" FROM ({0}) WHERE ""BlocksetID"" >= 0 ", scantableDefinition)))
@@ -320,20 +256,10 @@ namespace Duplicati.Library.Main.Database
/// <returns>True if the block should be added to the current output</returns>
public bool AddBlock (string key, long size, long volumeid, System.Data.IDbTransaction transaction = null)
{
var r = -1L;
if (m_blockHashLookup != null)
{
KeyValuePair<long, long> blockid;
if (m_blockHashLookup.TryGet(key, size, out blockid))
return false;
}
else
{
m_findblockCommand.Transaction = transaction;
m_findblockCommand.SetParameterValue(0, key);
m_findblockCommand.SetParameterValue(1, size);
r = m_findblockCommand.ExecuteScalarInt64(-1);
}
m_findblockCommand.Transaction = transaction;
m_findblockCommand.SetParameterValue(0, key);
m_findblockCommand.SetParameterValue(1, size);
var r = m_findblockCommand.ExecuteScalarInt64(-1);
if (r == -1L)
{
@@ -342,16 +268,11 @@ namespace Duplicati.Library.Main.Database
m_insertblockCommand.SetParameterValue(1, volumeid);
m_insertblockCommand.SetParameterValue(2, size);
r = m_insertblockCommand.ExecuteScalarInt64();
if (m_blockHashLookup != null)
m_blockHashLookup.Add(key, size, new KeyValuePair<long, long>(r, size));
return true;
}
else
{
//Update lookup cache if required
if (m_blockHashLookup != null)
m_blockHashLookup.Add(key, size, new KeyValuePair<long, long>(r, size));
return false;
}
}
@@ -369,27 +290,17 @@ namespace Duplicati.Library.Main.Database
/// <returns>True if the blockset was created, false otherwise</returns>
public bool AddBlockset(string filehash, long size, int blocksize, IEnumerable<string> hashes, IEnumerable<string> blocklistHashes, out long blocksetid, System.Data.IDbTransaction transaction = null)
{
if (m_fileHashLookup != null)
{
if (m_fileHashLookup.TryGet(filehash, size, out blocksetid))
return false;
}
else
{
m_findblocksetCommand.Transaction = transaction;
blocksetid = m_findblocksetCommand.ExecuteScalarInt64(null, -1, filehash, size);
if (blocksetid != -1)
return false; //Found it
}
m_findblocksetCommand.Transaction = transaction;
blocksetid = m_findblocksetCommand.ExecuteScalarInt64(null, -1, filehash, size);
if (blocksetid != -1)
return false; //Found it
using(var tr = new TemporaryTransactionWrapper(m_connection, transaction))
{
m_insertblocksetCommand.Transaction = tr.Parent;
m_insertblocksetCommand.SetParameterValue(0, size);
m_insertblocksetCommand.SetParameterValue(1, filehash);
blocksetid = m_insertblocksetCommand.ExecuteScalarInt64();
if (m_fileHashLookup != null)
m_fileHashLookup.Add(filehash, size, blocksetid);
long ix = 0;
if (blocklistHashes != null)
@@ -417,19 +328,6 @@ namespace Duplicati.Library.Main.Database
{
var exsize = remainsize < blocksize ? remainsize : blocksize;
var found = false;
if (m_blockHashLookup != null)
{
KeyValuePair<long, long> id;
if (m_blockHashLookup.TryGet(h, exsize, out id) && id.Value == exsize)
{
m_insertblocksetentryFastCommand.SetParameterValue(1, ix);
m_insertblocksetentryFastCommand.SetParameterValue(2, id.Key);
var cx = m_insertblocksetentryFastCommand.ExecuteNonQuery();
if (cx != 1)
throw new Exception(string.Format("Unexpected result count: {0}, expected {1}", cx, 1));
found = true;
}
}
if (!found)
{
@@ -462,18 +360,10 @@ namespace Duplicati.Library.Main.Database
{
if (size > 0)
{
if (m_metadataLookup != null)
{
if(m_metadataLookup.TryGet(filehash, size, out metadataid))
return false;
}
else
{
m_findmetadatasetCommand.Transaction = transaction;
metadataid = m_findmetadatasetCommand.ExecuteScalarInt64(null, -1, filehash, size);
if (metadataid != -1)
return false;
}
m_findmetadatasetCommand.Transaction = transaction;
metadataid = m_findmetadatasetCommand.ExecuteScalarInt64(null, -1, filehash, size);
if (metadataid != -1)
return false;
long blocksetid;
@@ -485,8 +375,6 @@ namespace Duplicati.Library.Main.Database
m_insertmetadatasetCommand.SetParameterValue(0, blocksetid);
metadataid = m_insertmetadatasetCommand.ExecuteScalarInt64();
tr.Commit();
if (m_metadataLookup != null)
m_metadataLookup.Add(filehash, size, metadataid);
}
return true;
@@ -645,9 +533,6 @@ namespace Duplicati.Library.Main.Database
public override void Dispose ()
{
m_fileHashLookup = null;
m_metadataLookup = null;
m_blockHashLookup = null;
m_pathLookup = null;
base.Dispose();
@@ -273,41 +273,19 @@ namespace Duplicati.Library.Main.Database
private class BlockQuery : IBlockQuery
{
private System.Data.IDbCommand m_command;
private HashLookupHelper<long> m_lookup;
public BlockQuery(System.Data.IDbConnection con, Options options, System.Data.IDbTransaction transaction)
{
m_command = con.CreateCommand();
m_command.Transaction = transaction;
if (options.BlockHashLookupMemory > 0)
{
m_lookup = new HashLookupHelper<long>((ulong)options.BlockHashLookupMemory);
using(var reader = m_command.ExecuteReader(@"SELECT ""Hash"", ""Size"" FROM ""Block"" "))
while (reader.Read())
{
var hash = reader.GetString(0);
var size = reader.GetInt64(1);
m_lookup.Add(hash, size, size);
}
}
m_command.Parameters.Clear();
m_command.CommandText = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ? ";
m_command.AddParameters(2);
}
public bool UseBlock(string hash, long size, System.Data.IDbTransaction transaction)
{
if (m_lookup != null)
{
long nsize;
if(m_lookup.TryGet(hash, size, out nsize) && nsize == size)
return true;
else
return false;
}
{
m_command.Transaction = transaction;
m_command.SetParameterValue(0, hash);
m_command.SetParameterValue(1, size);
@@ -317,7 +295,6 @@ namespace Duplicati.Library.Main.Database
public void Dispose()
{
m_lookup = null;
if (m_command != null)
try { m_command.Dispose(); }
finally { m_command = null; }
@@ -60,10 +60,6 @@ namespace Duplicati.Library.Main.Database
private System.Data.IDbCommand m_insertBlockCommand;
private System.Data.IDbCommand m_insertDuplicateBlockCommand;
private HashLookupHelper<bool> m_blockListHashLookup;
private HashLookupHelper<long> m_blockHashLookup;
private HashLookupHelper<long> m_fileHashLookup;
private HashLookupHelper<long> m_metadataLookup;
private PathLookupHelper<PathEntryKeeper> m_filesetLookup;
private string m_tempblocklist;
@@ -197,15 +193,6 @@ namespace Duplicati.Library.Main.Database
m_insertDuplicateBlockCommand.CommandText = @"INSERT INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") VALUES ((SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?), ?)";
m_insertDuplicateBlockCommand.AddParameters(3);
if (options.BlockHashLookupMemory > 0)
{
m_blockHashLookup = new HashLookupHelper<long>((ulong)options.BlockHashLookupMemory/2);
m_blockListHashLookup = new HashLookupHelper<bool>((ulong)options.BlockHashLookupMemory/2);
}
if (options.FileHashLookupMemory > 0)
m_fileHashLookup = new HashLookupHelper<long>((ulong)options.FileHashLookupMemory);
if (options.MetadataHashMemory > 0)
m_metadataLookup = new HashLookupHelper<long>((ulong)options.MetadataHashMemory);
if (options.UseFilepathCache)
m_filesetLookup = new PathLookupHelper<PathEntryKeeper>();
}
@@ -247,18 +234,6 @@ namespace Duplicati.Library.Main.Database
// Insert all known blocks into block table with volumeid = -1
cmd.ExecuteNonQuery(insertBlocksCommand);
// Update the cache with new blocks
if (m_blockHashLookup != null)
{
using(var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Hash"", ""Size"" FROM ""Block"" WHERE ""VolumeID"" = -1 "))
while(rd.Read())
{
var hash = rd.GetString(0);
var size = rd.GetInt64(1);
m_blockHashLookup.TryAdd(hash, size, -1);
}
}
var selectBlocklistBlocksetEntries = string.Format(
@"SELECT ""E"".""BlocksetID"" AS ""BlocksetID"", ""D"".""FullIndex"" AS ""Index"", ""F"".""ID"" AS ""BlockID"" FROM ( " +
SELECT_BLOCKLIST_ENTRIES +
@@ -268,8 +243,6 @@ namespace Duplicati.Library.Main.Database
m_tempblocklist,
blocksize / hashsize
);
var selectBlocksetEntries = string.Format(
@"SELECT ""Blockset"".""ID"" AS ""BlocksetID"", 0 AS ""Index"", ""Block"".""ID"" AS ""BlockID"" FROM ""Blockset"", ""Block"", ""{1}"" S WHERE ""Blockset"".""Fullhash"" = ""S"".""FileHash"" AND ""S"".""BlockHash"" = ""Block"".""Hash"" AND ""S"".""BlockSize"" = ""Block"".""Size"" AND ""Blockset"".""Length"" = ""S"".""BlockSize"" AND ""Blockset"".""Length"" <= {0} ",
@@ -362,63 +335,36 @@ namespace Duplicati.Library.Main.Database
if (metahash == null)
return metadataid;
if (m_metadataLookup != null)
{
if (m_metadataLookup.TryGet(metahash, metahashsize, out metadataid))
return metadataid;
else
metadataid = -1;
}
else
{
m_findMetadatasetCommand.Transaction = transaction;
m_findMetadatasetCommand.SetParameterValue(0, metahash);
m_findMetadatasetCommand.SetParameterValue(1, metahashsize);
metadataid = m_findMetadatasetCommand.ExecuteScalarInt64(-1);
if (metadataid != -1)
return metadataid;
}
m_findMetadatasetCommand.Transaction = transaction;
m_findMetadatasetCommand.SetParameterValue(0, metahash);
m_findMetadatasetCommand.SetParameterValue(1, metahashsize);
metadataid = m_findMetadatasetCommand.ExecuteScalarInt64(-1);
if (metadataid != -1)
return metadataid;
var blocksetid = AddBlockset(metahash, metahashsize, metablocklisthashes, expectedmetablocklisthashes, transaction);
m_insertMetadatasetCommand.Transaction = transaction;
m_insertMetadatasetCommand.SetParameterValue(0, blocksetid);
metadataid = m_insertMetadatasetCommand.ExecuteScalarInt64(-1);
if (m_metadataLookup != null)
m_metadataLookup.Add(metahash, metahashsize, metadataid);
return metadataid;
}
public long AddBlockset(string fullhash, long size, IEnumerable<string> blocklisthashes, long expectedblocklisthashes, System.Data.IDbTransaction transaction)
{
var blocksetid = -1L;
if (m_fileHashLookup != null)
{
if (m_fileHashLookup.TryGet(fullhash, size, out blocksetid))
return blocksetid;
else
blocksetid = -1;
}
else
{
m_findBlocksetCommand.Transaction = transaction;
m_findBlocksetCommand.SetParameterValue(0, size);
m_findBlocksetCommand.SetParameterValue(1, fullhash);
blocksetid = m_findBlocksetCommand.ExecuteScalarInt64(-1);
if (blocksetid != -1)
return blocksetid;
}
m_findBlocksetCommand.Transaction = transaction;
m_findBlocksetCommand.SetParameterValue(0, size);
m_findBlocksetCommand.SetParameterValue(1, fullhash);
var blocksetid = m_findBlocksetCommand.ExecuteScalarInt64(-1);
if (blocksetid != -1)
return blocksetid;
m_insertBlocksetCommand.Transaction = transaction;
m_insertBlocksetCommand.SetParameterValue(0, size);
m_insertBlocksetCommand.SetParameterValue(1, fullhash);
blocksetid = m_insertBlocksetCommand.ExecuteScalarInt64(-1);
if (m_fileHashLookup != null)
m_fileHashLookup.Add(fullhash, size, blocksetid);
long c = 0;
if (blocklisthashes != null)
{
@@ -449,20 +395,11 @@ namespace Duplicati.Library.Main.Database
public bool UpdateBlock(string hash, long size, long volumeID, System.Data.IDbTransaction transaction)
{
var currentVolumeId = -2L;
if (m_blockHashLookup != null)
{
if (!m_blockHashLookup.TryGet(hash, size, out currentVolumeId))
currentVolumeId = -2;
}
else
{
m_findHashBlockCommand.Transaction = transaction;
m_findHashBlockCommand.SetParameterValue(0, hash);
m_findHashBlockCommand.SetParameterValue(1, size);
currentVolumeId = m_findHashBlockCommand.ExecuteScalarInt64(-2);
}
m_findHashBlockCommand.Transaction = transaction;
m_findHashBlockCommand.SetParameterValue(0, hash);
m_findHashBlockCommand.SetParameterValue(1, size);
var currentVolumeId = m_findHashBlockCommand.ExecuteScalarInt64(-2);
if (currentVolumeId == volumeID)
return false;
@@ -475,9 +412,6 @@ namespace Duplicati.Library.Main.Database
m_insertBlockCommand.SetParameterValue(2, volumeID);
m_insertBlockCommand.ExecuteNonQuery();
if (m_blockHashLookup != null)
m_blockHashLookup.Add(hash, size, volumeID);
return true;
}
else if (currentVolumeId == -1)
@@ -491,9 +425,6 @@ namespace Duplicati.Library.Main.Database
if (c != 1)
throw new Exception(string.Format("Failed to update table, found {0} entries for key {1} with size {2}", c ,hash, size));
if (m_blockHashLookup != null)
m_blockHashLookup.Add(hash, size, volumeID);
return true;
}
else
@@ -519,23 +450,11 @@ namespace Duplicati.Library.Main.Database
public bool UpdateBlockset(string hash, IEnumerable<string> blocklisthashes, System.Data.IDbTransaction transaction)
{
if (m_blockListHashLookup != null)
{
bool b;
if (m_blockListHashLookup.TryGet(hash, -1, out b))
return false;
}
else
{
m_findblocklisthashCommand.Transaction = transaction;
m_findblocklisthashCommand.SetParameterValue(0, hash);
var r = m_findblocklisthashCommand.ExecuteScalar();
if (r != null && r != DBNull.Value)
return false;
}
if (m_blockListHashLookup != null)
m_blockListHashLookup.Add(hash, -1, false);
m_findblocklisthashCommand.Transaction = transaction;
m_findblocklisthashCommand.SetParameterValue(0, hash);
var r = m_findblocklisthashCommand.ExecuteScalar();
if (r != null && r != DBNull.Value)
return false;
m_insertBlockset.Transaction = transaction;
m_insertBlockset.SetParameterValue(0, hash);
+3 -68
View File
@@ -474,10 +474,6 @@ namespace Duplicati.Library.Main
new CommandLineArgument("skip-metadata", CommandLineArgument.ArgumentType.Boolean, Strings.Options.SkipmetadataShort, Strings.Options.SkipmetadataLong, "false"),
new CommandLineArgument("restore-permissions", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RestorepermissionsShort, Strings.Options.RestorepermissionsLong, "false"),
new CommandLineArgument("skip-restore-verification", CommandLineArgument.ArgumentType.Boolean, Strings.Options.SkiprestoreverificationShort, Strings.Options.SkiprestoreverificationLong, "false"),
new CommandLineArgument("blockhash-lookup-memory", CommandLineArgument.ArgumentType.Size, Strings.Options.BlockhashlookupsizeShort, Strings.Options.BlockhashlookupsizeLong, "0mb"),
new CommandLineArgument("filehash-lookup-memory", CommandLineArgument.ArgumentType.Size, Strings.Options.FilehashlookupsizeShort, Strings.Options.FilehashlookupsizeLong, "0mb"),
new CommandLineArgument("metadatahash-lookup-memory", CommandLineArgument.ArgumentType.Size, Strings.Options.MetadatahashlookupsizeShort, Strings.Options.MetadatahashlookupsizeLong, "0mb"),
new CommandLineArgument("old-lookup-memory-defaults", CommandLineArgument.ArgumentType.Size, Strings.Options.OldmemorylookupdefaultsShort, Strings.Options.OldmemorylookupdefaultsLong, "false"),
new CommandLineArgument("disable-filepath-cache", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablefilepathcacheShort, Strings.Options.DisablefilepathcacheLong, "true"),
new CommandLineArgument("changed-files", CommandLineArgument.ArgumentType.Path, Strings.Options.ChangedfilesShort, Strings.Options.ChangedfilesLong),
new CommandLineArgument("deleted-files", CommandLineArgument.ArgumentType.Path, Strings.Options.DeletedfilesShort, Strings.Options.DeletedfilesLong("changed-files")),
@@ -1413,61 +1409,6 @@ namespace Duplicati.Library.Main
get { return !Library.Utility.Utility.ParseBoolOption(m_options, "skip-restore-verification"); }
}
/// <summary>
/// Gets a flag indicating whether this <see cref="Duplicati.Library.Main.Options"/> old memory defaults.
/// </summary>
public bool OldMemoryDefaults
{
get { return Library.Utility.Utility.ParseBoolOption(m_options, "old-lookup-memory-defaults"); }
}
/// <summary>
/// Gets the block hash lookup size
/// </summary>
public long BlockHashLookupMemory
{
get
{
string v;
m_options.TryGetValue("blockhash-lookup-memory", out v);
if (string.IsNullOrEmpty(v))
v = OldMemoryDefaults ? DEFAULT_BLOCK_HASH_LOOKUP_SIZE : "0";
return Library.Utility.Sizeparser.ParseSize(v, "mb");
}
}
/// <summary>
/// Gets the file hash size
/// </summary>
public long FileHashLookupMemory
{
get
{
string v;
m_options.TryGetValue("filehash-lookup-memory", out v);
if (string.IsNullOrEmpty(v))
v = OldMemoryDefaults ? DEFAULT_FILE_HASH_LOOKUP_SIZE : "0";
return Library.Utility.Sizeparser.ParseSize(v, "mb");
}
}
/// <summary>
/// Gets the block hash size
/// </summary>
public long MetadataHashMemory
{
get
{
string v;
m_options.TryGetValue("metadatahash-lookup-memory", out v);
if (string.IsNullOrEmpty(v))
v = OldMemoryDefaults ? DEFAULT_METADATA_HASH_LOOKUP_SIZE : "0";
return Library.Utility.Sizeparser.ParseSize(v, "mb");
}
}
/// <summary>
/// Gets the file hash size
@@ -1476,15 +1417,9 @@ namespace Duplicati.Library.Main
{
get
{
if (OldMemoryDefaults)
return !Library.Utility.Utility.ParseBoolOption(m_options, "disable-filepath-cache");
else
{
string s;
m_options.TryGetValue("disable-filepath-cache", out s);
return !Library.Utility.Utility.ParseBool(s, true);
}
string s;
m_options.TryGetValue("disable-filepath-cache", out s);
return !Library.Utility.Utility.ParseBool(s, true);
}
}