From 1231dc24daaa283cf2c5501bf097ecb3f12619ab Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 11 Mar 2013 20:45:48 +0100 Subject: [PATCH] Made the hash memory requirement configurable --- .../Database/LocalBackupDatabase.cs | 27 +-- .../Library/Main/ForestHash/FhOptions.cs | 204 ++++++++++-------- .../ForestHash/Operation/BackupHandler.cs | 2 +- .../Operation/RecreateDatabaseHandler.cs | 2 +- .../Main/Strings/FhOptions.Designer.cs | 36 ++++ Duplicati/Library/Main/Strings/FhOptions.resx | 12 ++ 6 files changed, 179 insertions(+), 104 deletions(-) diff --git a/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs index d563b0ebb..33ee1cf51 100644 --- a/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs @@ -8,14 +8,6 @@ namespace Duplicati.Library.Main.ForestHash.Database { public class LocalBackupDatabase : Localdatabase { - /// - /// The amount of memory allocated to store block hashes in memory (will allocate twice this memory) - /// - private const ulong BLOCK_HASH_LOOKUP_SIZE = 64 * 1024 * 1024; - /// - /// The amount of memory allocated to store file in memory (will allocate twice this memory) - /// - private const ulong FILE_HASH_LOOKUP_SIZE = 16 * 1024 * 1024; /// /// An approximate size of a hash-string in memory (44 chars * 2 for unicode + 8 bytes for pointer = 104) /// @@ -60,17 +52,17 @@ namespace Duplicati.Library.Main.ForestHash.Database private long m_missingBlockHashes; private string m_scantimelookupTablename; - public LocalBackupDatabase(string path) - : this(CreateConnection(path)) + public LocalBackupDatabase(string path, FhOptions options) + : this(CreateConnection(path), options) { } - public LocalBackupDatabase(LocalRestoredatabase connection) - : this(connection.Connection) + public LocalBackupDatabase(LocalRestoredatabase connection, FhOptions options) + : this(connection.Connection, options) { } - public LocalBackupDatabase(System.Data.IDbConnection connection) + public LocalBackupDatabase(System.Data.IDbConnection connection, FhOptions options) : base(connection, "Backup") { m_findblockCommand = m_connection.CreateCommand(); @@ -135,7 +127,6 @@ namespace Duplicati.Library.Main.ForestHash.Database cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT ""OperationFileset"".""FilesetID"" AS ""FilesetID"", MAX(""OperationFileset"".""Scantime"") AS ""Scantime"", ""Fileset"".""Path"" AS ""Path"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" GROUP BY ""FilesetID"", ""Path"" ", m_scantimelookupTablename)); m_selectfileSimpleCommand.CommandText = string.Format(@"SELECT ""FilesetID"", ""Scantime"" FROM ""{0}"" WHERE ""Path"" = ?", m_scantimelookupTablename); - //m_selectfileSimpleCommand.CommandText = @"SELECT ""OperationFileset"".""FilesetID"", ""OperationFileset"".""Scantime"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""Fileset"".""Path"" = ? ORDER BY ""OperationFileset"".""Scantime"" DESC"; m_selectfileSimpleCommand.AddParameters(1); m_selectfileHashCommand.CommandText = @"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"" INNER JOIN ""Fileset"" ON ""Blockset"".""ID"" = ""Fileset"".""BlocksetID"" WHERE ""Fileset"".""ID"" = ? "; @@ -154,11 +145,11 @@ namespace Duplicati.Library.Main.ForestHash.Database m_createremotevolumeCommand.AddParameter(m_operationid); m_createremotevolumeCommand.AddParameters(3); - m_blockHashPrefixLookup = new HashPrefixLookup(BLOCK_HASH_LOOKUP_SIZE); - m_fileHashPrefixLookup = new HashPrefixLookup(FILE_HASH_LOOKUP_SIZE); + m_blockHashPrefixLookup = new HashPrefixLookup((ulong)options.FhBlockHashSize / 2); + m_fileHashPrefixLookup = new HashPrefixLookup((ulong)options.FhFileHashSize / 2); - m_blockHashFullLookup = new HashLookup(HASH_GUESS_SIZE, BLOCK_HASH_LOOKUP_SIZE); - m_fileHashFullLookup = new HashLookupWithData(HASH_GUESS_SIZE, FILE_HASH_LOOKUP_SIZE); + m_blockHashFullLookup = new HashLookup(HASH_GUESS_SIZE, (ulong)options.FhBlockHashSize / 2); + m_fileHashFullLookup = new HashLookupWithData(HASH_GUESS_SIZE, (ulong)options.FhFileHashSize / 2); using (var cmd = m_connection.CreateCommand()) using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Block"".""Hash"" FROM ""Block"", ""RemoteVolume"" WHERE ""RemoteVolume"".""Name"" = ""Block"".""File"" AND ""RemoteVolume"".""State"" IN (?,?,?,?) ", RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString())) diff --git a/Duplicati/Library/Main/ForestHash/FhOptions.cs b/Duplicati/Library/Main/ForestHash/FhOptions.cs index ad93c0adb..0e81f7dfb 100644 --- a/Duplicati/Library/Main/ForestHash/FhOptions.cs +++ b/Duplicati/Library/Main/ForestHash/FhOptions.cs @@ -1,73 +1,78 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using Duplicati.Library.Interface; - -namespace Duplicati.Library.Main.ForestHash -{ - public class FhOptions : Options - { - /// - /// The default block size for Foresthash - /// - private const string DEFAULT_FH_BLOCKSIZE = "10kb"; - - public FhOptions(Dictionary options) - : base(options) - { - } - - - /// - /// List of commands relating to the ForestHash module - /// - public static IList Commands - { - get - { - return new List(new ICommandLineArgument[] { - new CommandLineArgument("fh-dbpath", CommandLineArgument.ArgumentType.Path, Strings.FhOptions.FhdbpathShort, Strings.FhOptions.FhdbpathLong), - new CommandLineArgument("fh-blocksize", CommandLineArgument.ArgumentType.Size, Strings.FhOptions.FhblocksizeShort, Strings.FhOptions.FhblocksizeLong, DEFAULT_FH_BLOCKSIZE), - new CommandLineArgument("fh-nometadata", CommandLineArgument.ArgumentType.Boolean, Strings.FhOptions.FhnometadataShort, Strings.FhOptions.FhnometadataLong, "false"), -#if DEBUG - new CommandLineArgument("fh-no-local-blocks", CommandLineArgument.ArgumentType.Boolean, "Prevents using local blocks for restore", "", "false"), - new CommandLineArgument("fh-no-local-db", CommandLineArgument.ArgumentType.Boolean, "Prevents using local database for restore", "", "false"), -#endif - }); - } - } - - /// - /// Gets the path to the database - /// - public string Fhdbpath - { - get - { - string tmp; - m_options.TryGetValue("fh-dbpath", out tmp); - return tmp; - } - } - - /// - /// Gets the current block size - /// - public int Fhblocksize - { - get - { - string tmp; - if (!m_options.TryGetValue("fh-blocksize", out tmp)) - tmp = DEFAULT_FH_BLOCKSIZE; - - long t = Utility.Sizeparser.ParseSize(tmp, "kb"); - if (t > int.MaxValue || t < 1024) - throw new ArgumentOutOfRangeException("FhBlocksize", string.Format("The blocksize cannot be less than {0}, nor larger than {1}", 1024, int.MaxValue)); - - return (int)t; - } +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Duplicati.Library.Interface; + +namespace Duplicati.Library.Main.ForestHash +{ + public class FhOptions : Options + { + private const string DEFAULT_BLOCK_HASH_LOOKUP_SIZE = "64mb"; + private const string DEFAULT_FILE_HASH_LOOKUP_SIZE = "32mb"; + + /// + /// The default block size for Foresthash + /// + private const string DEFAULT_FH_BLOCKSIZE = "10kb"; + + public FhOptions(Dictionary options) + : base(options) + { + } + + + /// + /// List of commands relating to the ForestHash module + /// + public static IList Commands + { + get + { + return new List(new ICommandLineArgument[] { + new CommandLineArgument("fh-dbpath", CommandLineArgument.ArgumentType.Path, Strings.FhOptions.FhdbpathShort, Strings.FhOptions.FhdbpathLong), + new CommandLineArgument("fh-blocksize", CommandLineArgument.ArgumentType.Size, Strings.FhOptions.FhblocksizeShort, Strings.FhOptions.FhblocksizeLong, DEFAULT_FH_BLOCKSIZE), + new CommandLineArgument("fh-nometadata", CommandLineArgument.ArgumentType.Boolean, Strings.FhOptions.FhnometadataShort, Strings.FhOptions.FhnometadataLong, "false"), + new CommandLineArgument("fh-blockhash-lookup-size", CommandLineArgument.ArgumentType.Size, Strings.FhOptions.FhblockhashlookupsizeShort, Strings.FhOptions.FhblockhashlookupsizeLong, DEFAULT_BLOCK_HASH_LOOKUP_SIZE), + new CommandLineArgument("fh-filehash-lookup-size", CommandLineArgument.ArgumentType.Size, Strings.FhOptions.FhfilehashlookupsizeShort, Strings.FhOptions.FhfilehashlookupsizeLong, DEFAULT_FILE_HASH_LOOKUP_SIZE), +#if DEBUG + new CommandLineArgument("fh-no-local-blocks", CommandLineArgument.ArgumentType.Boolean, "Prevents using local blocks for restore", "", "false"), + new CommandLineArgument("fh-no-local-db", CommandLineArgument.ArgumentType.Boolean, "Prevents using local database for restore", "", "false"), +#endif + }); + } + } + + /// + /// Gets the path to the database + /// + public string Fhdbpath + { + get + { + string tmp; + m_options.TryGetValue("fh-dbpath", out tmp); + return tmp; + } + } + + /// + /// Gets the current block size + /// + public int Fhblocksize + { + get + { + string tmp; + if (!m_options.TryGetValue("fh-blocksize", out tmp)) + tmp = DEFAULT_FH_BLOCKSIZE; + + long t = Utility.Sizeparser.ParseSize(tmp, "kb"); + if (t > int.MaxValue || t < 1024) + throw new ArgumentOutOfRangeException("FhBlocksize", string.Format("The blocksize cannot be less than {0}, nor larger than {1}", 1024, int.MaxValue)); + + return (int)t; + } } /// @@ -76,17 +81,48 @@ namespace Duplicati.Library.Main.ForestHash public bool FhNoMetadata { get { return Utility.Utility.ParseBoolOption(m_options, "fh-nometadata"); } - } - -#if DEBUG - public bool NoLocalBlocks - { - get { return Utility.Utility.ParseBoolOption(m_options, "fh-no-local-blocks"); } - } - public bool NoLocalDb - { - get { return Utility.Utility.ParseBoolOption(m_options, "fh-no-local-db"); } - } -#endif - } -} + } + + /// + /// Gets the block hash size + /// + public long FhBlockHashSize + { + get + { + string v; + m_options.TryGetValue("fh-blockhash-lookup-size", out v); + if (string.IsNullOrEmpty(v)) + v = DEFAULT_BLOCK_HASH_LOOKUP_SIZE; + + return Utility.Sizeparser.ParseSize(v, "mb"); + } + } + + /// + /// Gets the file hash size + /// + public long FhFileHashSize + { + get + { + string v; + m_options.TryGetValue("fh-filehash-lookup-size", out v); + if (string.IsNullOrEmpty(v)) + v = DEFAULT_FILE_HASH_LOOKUP_SIZE; + + return Utility.Sizeparser.ParseSize(v, "mb"); + } + } +#if DEBUG + public bool NoLocalBlocks + { + get { return Utility.Utility.ParseBoolOption(m_options, "fh-no-local-blocks"); } + } + public bool NoLocalDb + { + get { return Utility.Utility.ParseBoolOption(m_options, "fh-no-local-db"); } + } +#endif + } +} diff --git a/Duplicati/Library/Main/ForestHash/Operation/BackupHandler.cs b/Duplicati/Library/Main/ForestHash/Operation/BackupHandler.cs index 42afeb339..79f667337 100644 --- a/Duplicati/Library/Main/ForestHash/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/ForestHash/Operation/BackupHandler.cs @@ -36,7 +36,7 @@ namespace Duplicati.Library.Main.ForestHash.Operation { m_options = options; m_stat = stat; - m_database = new LocalBackupDatabase(m_options.Fhdbpath); + m_database = new LocalBackupDatabase(m_options.Fhdbpath, m_options); m_backend = new FhBackend(backendurl, options, m_database, m_stat); m_sources = sources; diff --git a/Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs index 6e7d02511..a2c422683 100644 --- a/Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs @@ -63,7 +63,7 @@ namespace Duplicati.Library.Main.ForestHash.Operation if (filelistfilter != null) filelists = filelistfilter(filelists); - using (var backupdb = new LocalBackupDatabase(restoredb)) + using (var backupdb = new LocalBackupDatabase(restoredb, m_options)) { foreach (var fl in remotefiles) backupdb.RegisterRemoteVolume(fl.File.Name, fl.FileType, RemoteVolumeState.Uploaded); diff --git a/Duplicati/Library/Main/Strings/FhOptions.Designer.cs b/Duplicati/Library/Main/Strings/FhOptions.Designer.cs index 061745fdb..3a2c7f428 100644 --- a/Duplicati/Library/Main/Strings/FhOptions.Designer.cs +++ b/Duplicati/Library/Main/Strings/FhOptions.Designer.cs @@ -60,6 +60,24 @@ namespace Duplicati.Library.Main.Strings { } } + /// + /// Looks up a localized string similar to A fragment of memory is used to reduce database lookups. You should not change this value unless you get warnings in the log.. + /// + internal static string FhblockhashlookupsizeLong { + get { + return ResourceManager.GetString("FhblockhashlookupsizeLong", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Memory used by the block hash. + /// + internal static string FhblockhashlookupsizeShort { + get { + return ResourceManager.GetString("FhblockhashlookupsizeShort", resourceCulture); + } + } + /// /// Looks up a localized string similar to The blocksize determines how files are fragmented. Choosing a large value will cause a larger overhead on file changes, choosing a small value will cause a large overhead on storage of file lists. Note that the value cannot be changed after remote files are created.. /// @@ -96,6 +114,24 @@ namespace Duplicati.Library.Main.Strings { } } + /// + /// Looks up a localized string similar to A fragment of memory is used to reduce database lookups. You should not change this value unless you get warnings in the log.. + /// + internal static string FhfilehashlookupsizeLong { + get { + return ResourceManager.GetString("FhfilehashlookupsizeLong", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Memory used by the file hash. + /// + internal static string FhfilehashlookupsizeShort { + get { + return ResourceManager.GetString("FhfilehashlookupsizeShort", resourceCulture); + } + } + /// /// Looks up a localized string similar to Stores empty metadata, such as file timestamps and attributes. This decreases the required storage space as well as the processing time.. /// diff --git a/Duplicati/Library/Main/Strings/FhOptions.resx b/Duplicati/Library/Main/Strings/FhOptions.resx index e272c306a..f1e68bf5b 100644 --- a/Duplicati/Library/Main/Strings/FhOptions.resx +++ b/Duplicati/Library/Main/Strings/FhOptions.resx @@ -117,6 +117,12 @@ System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + A fragment of memory is used to reduce database lookups. You should not change this value unless you get warnings in the log. + + + Memory used by the block hash + The blocksize determines how files are fragmented. Choosing a large value will cause a larger overhead on file changes, choosing a small value will cause a large overhead on storage of file lists. Note that the value cannot be changed after remote files are created. @@ -129,6 +135,12 @@ Path to the local state database + + A fragment of memory is used to reduce database lookups. You should not change this value unless you get warnings in the log. + + + Memory used by the file hash + Stores empty metadata, such as file timestamps and attributes. This decreases the required storage space as well as the processing time.