haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Made the hash memory requirement configurable

Browse Source
This commit is contained in:
Kenneth Skovhede
2013-03-11 20:45:48 +01:00
parent b7531b3766
commit 1231dc24da
6 changed files with 179 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs
+9 -18
View File
@@ -8,14 +8,6 @@ namespace Duplicati.Library.Main.ForestHash.Database
{
public class LocalBackupDatabase : Localdatabase
{
/// <summary>
/// The amount of memory allocated to store block hashes in memory (will allocate twice this memory)
/// </summary>
private const ulong BLOCK_HASH_LOOKUP_SIZE = 64 * 1024 * 1024;
/// <summary>
/// The amount of memory allocated to store file in memory (will allocate twice this memory)
/// </summary>
private const ulong FILE_HASH_LOOKUP_SIZE = 16 * 1024 * 1024;
/// <summary>
/// An approximate size of a hash-string in memory (44 chars * 2 for unicode + 8 bytes for pointer = 104)
/// </summary>
@@ -60,17 +52,17 @@ namespace Duplicati.Library.Main.ForestHash.Database
private long m_missingBlockHashes;
private string m_scantimelookupTablename;
public LocalBackupDatabase(string path)
: this(CreateConnection(path))
public LocalBackupDatabase(string path, FhOptions options)
: this(CreateConnection(path), options)
{
}
public LocalBackupDatabase(LocalRestoredatabase connection)
: this(connection.Connection)
public LocalBackupDatabase(LocalRestoredatabase connection, FhOptions options)
: this(connection.Connection, options)
{
}
public LocalBackupDatabase(System.Data.IDbConnection connection)
public LocalBackupDatabase(System.Data.IDbConnection connection, FhOptions options)
: base(connection, "Backup")
{
m_findblockCommand = m_connection.CreateCommand();
@@ -135,7 +127,6 @@ namespace Duplicati.Library.Main.ForestHash.Database
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT ""OperationFileset"".""FilesetID"" AS ""FilesetID"", MAX(""OperationFileset"".""Scantime"") AS ""Scantime"", ""Fileset"".""Path"" AS ""Path"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" GROUP BY ""FilesetID"", ""Path"" ", m_scantimelookupTablename));
m_selectfileSimpleCommand.CommandText = string.Format(@"SELECT ""FilesetID"", ""Scantime"" FROM ""{0}"" WHERE ""Path"" = ?", m_scantimelookupTablename);
//m_selectfileSimpleCommand.CommandText = @"SELECT ""OperationFileset"".""FilesetID"", ""OperationFileset"".""Scantime"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""Fileset"".""Path"" = ? ORDER BY ""OperationFileset"".""Scantime"" DESC";
m_selectfileSimpleCommand.AddParameters(1);
m_selectfileHashCommand.CommandText = @"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"" INNER JOIN ""Fileset"" ON ""Blockset"".""ID"" = ""Fileset"".""BlocksetID"" WHERE ""Fileset"".""ID"" = ? ";
@@ -154,11 +145,11 @@ namespace Duplicati.Library.Main.ForestHash.Database
m_createremotevolumeCommand.AddParameter(m_operationid);
m_createremotevolumeCommand.AddParameters(3);
m_blockHashPrefixLookup = new HashPrefixLookup(BLOCK_HASH_LOOKUP_SIZE);
m_fileHashPrefixLookup = new HashPrefixLookup(FILE_HASH_LOOKUP_SIZE);
m_blockHashPrefixLookup = new HashPrefixLookup((ulong)options.FhBlockHashSize / 2);
m_fileHashPrefixLookup = new HashPrefixLookup((ulong)options.FhFileHashSize / 2);
m_blockHashFullLookup = new HashLookup<string>(HASH_GUESS_SIZE, BLOCK_HASH_LOOKUP_SIZE);
m_fileHashFullLookup = new HashLookupWithData<string, long>(HASH_GUESS_SIZE, FILE_HASH_LOOKUP_SIZE);
m_blockHashFullLookup = new HashLookup<string>(HASH_GUESS_SIZE, (ulong)options.FhBlockHashSize / 2);
m_fileHashFullLookup = new HashLookupWithData<string, long>(HASH_GUESS_SIZE, (ulong)options.FhFileHashSize / 2);
using (var cmd = m_connection.CreateCommand())
using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Block"".""Hash"" FROM ""Block"", ""RemoteVolume"" WHERE ""RemoteVolume"".""Name"" = ""Block"".""File"" AND ""RemoteVolume"".""State"" IN (?,?,?,?) ", RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()))
Duplicati/Library/Main/ForestHash/FhOptions.cs
+120 -84
View File
@@ -1,73 +1,78 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Duplicati.Library.Interface;
namespace Duplicati.Library.Main.ForestHash
{
public class FhOptions : Options
{
/// <summary>
/// The default block size for Foresthash
/// </summary>
private const string DEFAULT_FH_BLOCKSIZE = "10kb";
public FhOptions(Dictionary<string, string> options)
: base(options)
{
}
/// <summary>
/// List of commands relating to the ForestHash module
/// </summary>
public static IList<ICommandLineArgument> Commands
{
get
{
return new List<ICommandLineArgument>(new ICommandLineArgument[] {
new CommandLineArgument("fh-dbpath", CommandLineArgument.ArgumentType.Path, Strings.FhOptions.FhdbpathShort, Strings.FhOptions.FhdbpathLong),
new CommandLineArgument("fh-blocksize", CommandLineArgument.ArgumentType.Size, Strings.FhOptions.FhblocksizeShort, Strings.FhOptions.FhblocksizeLong, DEFAULT_FH_BLOCKSIZE),
new CommandLineArgument("fh-nometadata", CommandLineArgument.ArgumentType.Boolean, Strings.FhOptions.FhnometadataShort, Strings.FhOptions.FhnometadataLong, "false"),
#if DEBUG
new CommandLineArgument("fh-no-local-blocks", CommandLineArgument.ArgumentType.Boolean, "Prevents using local blocks for restore", "", "false"),
new CommandLineArgument("fh-no-local-db", CommandLineArgument.ArgumentType.Boolean, "Prevents using local database for restore", "", "false"),
#endif
});
}
}
/// <summary>
/// Gets the path to the database
/// </summary>
public string Fhdbpath
{
get
{
string tmp;
m_options.TryGetValue("fh-dbpath", out tmp);
return tmp;
}
}
/// <summary>
/// Gets the current block size
/// </summary>
public int Fhblocksize
{
get
{
string tmp;
if (!m_options.TryGetValue("fh-blocksize", out tmp))
tmp = DEFAULT_FH_BLOCKSIZE;
long t = Utility.Sizeparser.ParseSize(tmp, "kb");
if (t > int.MaxValue || t < 1024)
throw new ArgumentOutOfRangeException("FhBlocksize", string.Format("The blocksize cannot be less than {0}, nor larger than {1}", 1024, int.MaxValue));
return (int)t;
}
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Duplicati.Library.Interface;
namespace Duplicati.Library.Main.ForestHash
{
public class FhOptions : Options
{
private const string DEFAULT_BLOCK_HASH_LOOKUP_SIZE = "64mb";
private const string DEFAULT_FILE_HASH_LOOKUP_SIZE = "32mb";
/// <summary>
/// The default block size for Foresthash
/// </summary>
private const string DEFAULT_FH_BLOCKSIZE = "10kb";
public FhOptions(Dictionary<string, string> options)
: base(options)
{
}
/// <summary>
/// List of commands relating to the ForestHash module
/// </summary>
public static IList<ICommandLineArgument> Commands
{
get
{
return new List<ICommandLineArgument>(new ICommandLineArgument[] {
new CommandLineArgument("fh-dbpath", CommandLineArgument.ArgumentType.Path, Strings.FhOptions.FhdbpathShort, Strings.FhOptions.FhdbpathLong),
new CommandLineArgument("fh-blocksize", CommandLineArgument.ArgumentType.Size, Strings.FhOptions.FhblocksizeShort, Strings.FhOptions.FhblocksizeLong, DEFAULT_FH_BLOCKSIZE),
new CommandLineArgument("fh-nometadata", CommandLineArgument.ArgumentType.Boolean, Strings.FhOptions.FhnometadataShort, Strings.FhOptions.FhnometadataLong, "false"),
new CommandLineArgument("fh-blockhash-lookup-size", CommandLineArgument.ArgumentType.Size, Strings.FhOptions.FhblockhashlookupsizeShort, Strings.FhOptions.FhblockhashlookupsizeLong, DEFAULT_BLOCK_HASH_LOOKUP_SIZE),
new CommandLineArgument("fh-filehash-lookup-size", CommandLineArgument.ArgumentType.Size, Strings.FhOptions.FhfilehashlookupsizeShort, Strings.FhOptions.FhfilehashlookupsizeLong, DEFAULT_FILE_HASH_LOOKUP_SIZE),
#if DEBUG
new CommandLineArgument("fh-no-local-blocks", CommandLineArgument.ArgumentType.Boolean, "Prevents using local blocks for restore", "", "false"),
new CommandLineArgument("fh-no-local-db", CommandLineArgument.ArgumentType.Boolean, "Prevents using local database for restore", "", "false"),
#endif
});
}
}
/// <summary>
/// Gets the path to the database
/// </summary>
public string Fhdbpath
{
get
{
string tmp;
m_options.TryGetValue("fh-dbpath", out tmp);
return tmp;
}
}
/// <summary>
/// Gets the current block size
/// </summary>
public int Fhblocksize
{
get
{
string tmp;
if (!m_options.TryGetValue("fh-blocksize", out tmp))
tmp = DEFAULT_FH_BLOCKSIZE;
long t = Utility.Sizeparser.ParseSize(tmp, "kb");
if (t > int.MaxValue || t < 1024)
throw new ArgumentOutOfRangeException("FhBlocksize", string.Format("The blocksize cannot be less than {0}, nor larger than {1}", 1024, int.MaxValue));
return (int)t;
}
}
/// <summary>
@@ -76,17 +81,48 @@ namespace Duplicati.Library.Main.ForestHash
public bool FhNoMetadata
{
get { return Utility.Utility.ParseBoolOption(m_options, "fh-nometadata"); }
}
#if DEBUG
public bool NoLocalBlocks
{
get { return Utility.Utility.ParseBoolOption(m_options, "fh-no-local-blocks"); }
}
public bool NoLocalDb
{
get { return Utility.Utility.ParseBoolOption(m_options, "fh-no-local-db"); }
}
#endif
}
}
}
/// <summary>
/// Gets the block hash size
/// </summary>
public long FhBlockHashSize
{
get
{
string v;
m_options.TryGetValue("fh-blockhash-lookup-size", out v);
if (string.IsNullOrEmpty(v))
v = DEFAULT_BLOCK_HASH_LOOKUP_SIZE;
return Utility.Sizeparser.ParseSize(v, "mb");
}
}
/// <summary>
/// Gets the file hash size
/// </summary>
public long FhFileHashSize
{
get
{
string v;
m_options.TryGetValue("fh-filehash-lookup-size", out v);
if (string.IsNullOrEmpty(v))
v = DEFAULT_FILE_HASH_LOOKUP_SIZE;
return Utility.Sizeparser.ParseSize(v, "mb");
}
}
#if DEBUG
public bool NoLocalBlocks
{
get { return Utility.Utility.ParseBoolOption(m_options, "fh-no-local-blocks"); }
}
public bool NoLocalDb
{
get { return Utility.Utility.ParseBoolOption(m_options, "fh-no-local-db"); }
}
#endif
}
}
Duplicati/Library/Main/ForestHash/Operation/BackupHandler.cs
+1 -1
View File
@@ -36,7 +36,7 @@ namespace Duplicati.Library.Main.ForestHash.Operation
{
m_options = options;
m_stat = stat;
m_database = new LocalBackupDatabase(m_options.Fhdbpath);
m_database = new LocalBackupDatabase(m_options.Fhdbpath, m_options);
m_backend = new FhBackend(backendurl, options, m_database, m_stat);
m_sources = sources;
Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs
+1 -1
View File
@@ -63,7 +63,7 @@ namespace Duplicati.Library.Main.ForestHash.Operation
if (filelistfilter != null)
filelists = filelistfilter(filelists);
using (var backupdb = new LocalBackupDatabase(restoredb))
using (var backupdb = new LocalBackupDatabase(restoredb, m_options))
{
foreach (var fl in remotefiles)
backupdb.RegisterRemoteVolume(fl.File.Name, fl.FileType, RemoteVolumeState.Uploaded);
Duplicati/Library/Main/Strings/FhOptions.Designer.cs
Generated
+36
View File
@@ -60,6 +60,24 @@ namespace Duplicati.Library.Main.Strings {
}
}
/// <summary>
/// Looks up a localized string similar to A fragment of memory is used to reduce database lookups. You should not change this value unless you get warnings in the log..
/// </summary>
internal static string FhblockhashlookupsizeLong {
get {
return ResourceManager.GetString("FhblockhashlookupsizeLong", resourceCulture);
}
}
/// <summary>
/// Looks up a localized string similar to Memory used by the block hash.
/// </summary>
internal static string FhblockhashlookupsizeShort {
get {
return ResourceManager.GetString("FhblockhashlookupsizeShort", resourceCulture);
}
}
/// <summary>
/// Looks up a localized string similar to The blocksize determines how files are fragmented. Choosing a large value will cause a larger overhead on file changes, choosing a small value will cause a large overhead on storage of file lists. Note that the value cannot be changed after remote files are created..
/// </summary>
@@ -96,6 +114,24 @@ namespace Duplicati.Library.Main.Strings {
}
}
/// <summary>
/// Looks up a localized string similar to A fragment of memory is used to reduce database lookups. You should not change this value unless you get warnings in the log..
/// </summary>
internal static string FhfilehashlookupsizeLong {
get {
return ResourceManager.GetString("FhfilehashlookupsizeLong", resourceCulture);
}
}
/// <summary>
/// Looks up a localized string similar to Memory used by the file hash.
/// </summary>
internal static string FhfilehashlookupsizeShort {
get {
return ResourceManager.GetString("FhfilehashlookupsizeShort", resourceCulture);
}
}
/// <summary>
/// Looks up a localized string similar to Stores empty metadata, such as file timestamps and attributes. This decreases the required storage space as well as the processing time..
/// </summary>
Duplicati/Library/Main/Strings/FhOptions.resx
+12
View File
@@ -117,6 +117,12 @@
<resheader name="writer">
<value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</resheader>
<data name="FhblockhashlookupsizeLong" xml:space="preserve">
<value>A fragment of memory is used to reduce database lookups. You should not change this value unless you get warnings in the log.</value>
</data>
<data name="FhblockhashlookupsizeShort" xml:space="preserve">
<value>Memory used by the block hash</value>
</data>
<data name="FhblocksizeLong" xml:space="preserve">
<value>The blocksize determines how files are fragmented. Choosing a large value will cause a larger overhead on file changes, choosing a small value will cause a large overhead on storage of file lists. Note that the value cannot be changed after remote files are created.</value>
</data>
@@ -129,6 +135,12 @@
<data name="FhdbpathShort" xml:space="preserve">
<value>Path to the local state database</value>
</data>
<data name="FhfilehashlookupsizeLong" xml:space="preserve">
<value>A fragment of memory is used to reduce database lookups. You should not change this value unless you get warnings in the log.</value>
</data>
<data name="FhfilehashlookupsizeShort" xml:space="preserve">
<value>Memory used by the file hash</value>
</data>
<data name="FhnometadataLong" xml:space="preserve">
<value>Stores empty metadata, such as file timestamps and attributes. This decreases the required storage space as well as the processing time.</value>
</data>