diff --git a/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs
index d563b0ebb..33ee1cf51 100644
--- a/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs
+++ b/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs
@@ -8,14 +8,6 @@ namespace Duplicati.Library.Main.ForestHash.Database
{
public class LocalBackupDatabase : Localdatabase
{
- ///
- /// The amount of memory allocated to store block hashes in memory (will allocate twice this memory)
- ///
- private const ulong BLOCK_HASH_LOOKUP_SIZE = 64 * 1024 * 1024;
- ///
- /// The amount of memory allocated to store file in memory (will allocate twice this memory)
- ///
- private const ulong FILE_HASH_LOOKUP_SIZE = 16 * 1024 * 1024;
///
/// An approximate size of a hash-string in memory (44 chars * 2 for unicode + 8 bytes for pointer = 104)
///
@@ -60,17 +52,17 @@ namespace Duplicati.Library.Main.ForestHash.Database
private long m_missingBlockHashes;
private string m_scantimelookupTablename;
- public LocalBackupDatabase(string path)
- : this(CreateConnection(path))
+ public LocalBackupDatabase(string path, FhOptions options)
+ : this(CreateConnection(path), options)
{
}
- public LocalBackupDatabase(LocalRestoredatabase connection)
- : this(connection.Connection)
+ public LocalBackupDatabase(LocalRestoredatabase connection, FhOptions options)
+ : this(connection.Connection, options)
{
}
- public LocalBackupDatabase(System.Data.IDbConnection connection)
+ public LocalBackupDatabase(System.Data.IDbConnection connection, FhOptions options)
: base(connection, "Backup")
{
m_findblockCommand = m_connection.CreateCommand();
@@ -135,7 +127,6 @@ namespace Duplicati.Library.Main.ForestHash.Database
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT ""OperationFileset"".""FilesetID"" AS ""FilesetID"", MAX(""OperationFileset"".""Scantime"") AS ""Scantime"", ""Fileset"".""Path"" AS ""Path"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" GROUP BY ""FilesetID"", ""Path"" ", m_scantimelookupTablename));
m_selectfileSimpleCommand.CommandText = string.Format(@"SELECT ""FilesetID"", ""Scantime"" FROM ""{0}"" WHERE ""Path"" = ?", m_scantimelookupTablename);
- //m_selectfileSimpleCommand.CommandText = @"SELECT ""OperationFileset"".""FilesetID"", ""OperationFileset"".""Scantime"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""Fileset"".""Path"" = ? ORDER BY ""OperationFileset"".""Scantime"" DESC";
m_selectfileSimpleCommand.AddParameters(1);
m_selectfileHashCommand.CommandText = @"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"" INNER JOIN ""Fileset"" ON ""Blockset"".""ID"" = ""Fileset"".""BlocksetID"" WHERE ""Fileset"".""ID"" = ? ";
@@ -154,11 +145,11 @@ namespace Duplicati.Library.Main.ForestHash.Database
m_createremotevolumeCommand.AddParameter(m_operationid);
m_createremotevolumeCommand.AddParameters(3);
- m_blockHashPrefixLookup = new HashPrefixLookup(BLOCK_HASH_LOOKUP_SIZE);
- m_fileHashPrefixLookup = new HashPrefixLookup(FILE_HASH_LOOKUP_SIZE);
+ m_blockHashPrefixLookup = new HashPrefixLookup((ulong)options.FhBlockHashSize / 2);
+ m_fileHashPrefixLookup = new HashPrefixLookup((ulong)options.FhFileHashSize / 2);
- m_blockHashFullLookup = new HashLookup(HASH_GUESS_SIZE, BLOCK_HASH_LOOKUP_SIZE);
- m_fileHashFullLookup = new HashLookupWithData(HASH_GUESS_SIZE, FILE_HASH_LOOKUP_SIZE);
+ m_blockHashFullLookup = new HashLookup(HASH_GUESS_SIZE, (ulong)options.FhBlockHashSize / 2);
+ m_fileHashFullLookup = new HashLookupWithData(HASH_GUESS_SIZE, (ulong)options.FhFileHashSize / 2);
using (var cmd = m_connection.CreateCommand())
using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Block"".""Hash"" FROM ""Block"", ""RemoteVolume"" WHERE ""RemoteVolume"".""Name"" = ""Block"".""File"" AND ""RemoteVolume"".""State"" IN (?,?,?,?) ", RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()))
diff --git a/Duplicati/Library/Main/ForestHash/FhOptions.cs b/Duplicati/Library/Main/ForestHash/FhOptions.cs
index ad93c0adb..0e81f7dfb 100644
--- a/Duplicati/Library/Main/ForestHash/FhOptions.cs
+++ b/Duplicati/Library/Main/ForestHash/FhOptions.cs
@@ -1,73 +1,78 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using Duplicati.Library.Interface;
-
-namespace Duplicati.Library.Main.ForestHash
-{
- public class FhOptions : Options
- {
- ///
- /// The default block size for Foresthash
- ///
- private const string DEFAULT_FH_BLOCKSIZE = "10kb";
-
- public FhOptions(Dictionary options)
- : base(options)
- {
- }
-
-
- ///
- /// List of commands relating to the ForestHash module
- ///
- public static IList Commands
- {
- get
- {
- return new List(new ICommandLineArgument[] {
- new CommandLineArgument("fh-dbpath", CommandLineArgument.ArgumentType.Path, Strings.FhOptions.FhdbpathShort, Strings.FhOptions.FhdbpathLong),
- new CommandLineArgument("fh-blocksize", CommandLineArgument.ArgumentType.Size, Strings.FhOptions.FhblocksizeShort, Strings.FhOptions.FhblocksizeLong, DEFAULT_FH_BLOCKSIZE),
- new CommandLineArgument("fh-nometadata", CommandLineArgument.ArgumentType.Boolean, Strings.FhOptions.FhnometadataShort, Strings.FhOptions.FhnometadataLong, "false"),
-#if DEBUG
- new CommandLineArgument("fh-no-local-blocks", CommandLineArgument.ArgumentType.Boolean, "Prevents using local blocks for restore", "", "false"),
- new CommandLineArgument("fh-no-local-db", CommandLineArgument.ArgumentType.Boolean, "Prevents using local database for restore", "", "false"),
-#endif
- });
- }
- }
-
- ///
- /// Gets the path to the database
- ///
- public string Fhdbpath
- {
- get
- {
- string tmp;
- m_options.TryGetValue("fh-dbpath", out tmp);
- return tmp;
- }
- }
-
- ///
- /// Gets the current block size
- ///
- public int Fhblocksize
- {
- get
- {
- string tmp;
- if (!m_options.TryGetValue("fh-blocksize", out tmp))
- tmp = DEFAULT_FH_BLOCKSIZE;
-
- long t = Utility.Sizeparser.ParseSize(tmp, "kb");
- if (t > int.MaxValue || t < 1024)
- throw new ArgumentOutOfRangeException("FhBlocksize", string.Format("The blocksize cannot be less than {0}, nor larger than {1}", 1024, int.MaxValue));
-
- return (int)t;
- }
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Duplicati.Library.Interface;
+
+namespace Duplicati.Library.Main.ForestHash
+{
+ public class FhOptions : Options
+ {
+ private const string DEFAULT_BLOCK_HASH_LOOKUP_SIZE = "64mb";
+ private const string DEFAULT_FILE_HASH_LOOKUP_SIZE = "32mb";
+
+ ///
+ /// The default block size for Foresthash
+ ///
+ private const string DEFAULT_FH_BLOCKSIZE = "10kb";
+
+ public FhOptions(Dictionary options)
+ : base(options)
+ {
+ }
+
+
+ ///
+ /// List of commands relating to the ForestHash module
+ ///
+ public static IList Commands
+ {
+ get
+ {
+ return new List(new ICommandLineArgument[] {
+ new CommandLineArgument("fh-dbpath", CommandLineArgument.ArgumentType.Path, Strings.FhOptions.FhdbpathShort, Strings.FhOptions.FhdbpathLong),
+ new CommandLineArgument("fh-blocksize", CommandLineArgument.ArgumentType.Size, Strings.FhOptions.FhblocksizeShort, Strings.FhOptions.FhblocksizeLong, DEFAULT_FH_BLOCKSIZE),
+ new CommandLineArgument("fh-nometadata", CommandLineArgument.ArgumentType.Boolean, Strings.FhOptions.FhnometadataShort, Strings.FhOptions.FhnometadataLong, "false"),
+ new CommandLineArgument("fh-blockhash-lookup-size", CommandLineArgument.ArgumentType.Size, Strings.FhOptions.FhblockhashlookupsizeShort, Strings.FhOptions.FhblockhashlookupsizeLong, DEFAULT_BLOCK_HASH_LOOKUP_SIZE),
+ new CommandLineArgument("fh-filehash-lookup-size", CommandLineArgument.ArgumentType.Size, Strings.FhOptions.FhfilehashlookupsizeShort, Strings.FhOptions.FhfilehashlookupsizeLong, DEFAULT_FILE_HASH_LOOKUP_SIZE),
+#if DEBUG
+ new CommandLineArgument("fh-no-local-blocks", CommandLineArgument.ArgumentType.Boolean, "Prevents using local blocks for restore", "", "false"),
+ new CommandLineArgument("fh-no-local-db", CommandLineArgument.ArgumentType.Boolean, "Prevents using local database for restore", "", "false"),
+#endif
+ });
+ }
+ }
+
+ ///
+ /// Gets the path to the database
+ ///
+ public string Fhdbpath
+ {
+ get
+ {
+ string tmp;
+ m_options.TryGetValue("fh-dbpath", out tmp);
+ return tmp;
+ }
+ }
+
+ ///
+ /// Gets the current block size
+ ///
+ public int Fhblocksize
+ {
+ get
+ {
+ string tmp;
+ if (!m_options.TryGetValue("fh-blocksize", out tmp))
+ tmp = DEFAULT_FH_BLOCKSIZE;
+
+ long t = Utility.Sizeparser.ParseSize(tmp, "kb");
+ if (t > int.MaxValue || t < 1024)
+ throw new ArgumentOutOfRangeException("FhBlocksize", string.Format("The blocksize cannot be less than {0}, nor larger than {1}", 1024, int.MaxValue));
+
+ return (int)t;
+ }
}
///
@@ -76,17 +81,48 @@ namespace Duplicati.Library.Main.ForestHash
public bool FhNoMetadata
{
get { return Utility.Utility.ParseBoolOption(m_options, "fh-nometadata"); }
- }
-
-#if DEBUG
- public bool NoLocalBlocks
- {
- get { return Utility.Utility.ParseBoolOption(m_options, "fh-no-local-blocks"); }
- }
- public bool NoLocalDb
- {
- get { return Utility.Utility.ParseBoolOption(m_options, "fh-no-local-db"); }
- }
-#endif
- }
-}
+ }
+
+ ///
+ /// Gets the block hash size
+ ///
+ public long FhBlockHashSize
+ {
+ get
+ {
+ string v;
+ m_options.TryGetValue("fh-blockhash-lookup-size", out v);
+ if (string.IsNullOrEmpty(v))
+ v = DEFAULT_BLOCK_HASH_LOOKUP_SIZE;
+
+ return Utility.Sizeparser.ParseSize(v, "mb");
+ }
+ }
+
+ ///
+ /// Gets the file hash size
+ ///
+ public long FhFileHashSize
+ {
+ get
+ {
+ string v;
+ m_options.TryGetValue("fh-filehash-lookup-size", out v);
+ if (string.IsNullOrEmpty(v))
+ v = DEFAULT_FILE_HASH_LOOKUP_SIZE;
+
+ return Utility.Sizeparser.ParseSize(v, "mb");
+ }
+ }
+#if DEBUG
+ public bool NoLocalBlocks
+ {
+ get { return Utility.Utility.ParseBoolOption(m_options, "fh-no-local-blocks"); }
+ }
+ public bool NoLocalDb
+ {
+ get { return Utility.Utility.ParseBoolOption(m_options, "fh-no-local-db"); }
+ }
+#endif
+ }
+}
diff --git a/Duplicati/Library/Main/ForestHash/Operation/BackupHandler.cs b/Duplicati/Library/Main/ForestHash/Operation/BackupHandler.cs
index 42afeb339..79f667337 100644
--- a/Duplicati/Library/Main/ForestHash/Operation/BackupHandler.cs
+++ b/Duplicati/Library/Main/ForestHash/Operation/BackupHandler.cs
@@ -36,7 +36,7 @@ namespace Duplicati.Library.Main.ForestHash.Operation
{
m_options = options;
m_stat = stat;
- m_database = new LocalBackupDatabase(m_options.Fhdbpath);
+ m_database = new LocalBackupDatabase(m_options.Fhdbpath, m_options);
m_backend = new FhBackend(backendurl, options, m_database, m_stat);
m_sources = sources;
diff --git a/Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs
index 6e7d02511..a2c422683 100644
--- a/Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs
+++ b/Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs
@@ -63,7 +63,7 @@ namespace Duplicati.Library.Main.ForestHash.Operation
if (filelistfilter != null)
filelists = filelistfilter(filelists);
- using (var backupdb = new LocalBackupDatabase(restoredb))
+ using (var backupdb = new LocalBackupDatabase(restoredb, m_options))
{
foreach (var fl in remotefiles)
backupdb.RegisterRemoteVolume(fl.File.Name, fl.FileType, RemoteVolumeState.Uploaded);
diff --git a/Duplicati/Library/Main/Strings/FhOptions.Designer.cs b/Duplicati/Library/Main/Strings/FhOptions.Designer.cs
index 061745fdb..3a2c7f428 100644
--- a/Duplicati/Library/Main/Strings/FhOptions.Designer.cs
+++ b/Duplicati/Library/Main/Strings/FhOptions.Designer.cs
@@ -60,6 +60,24 @@ namespace Duplicati.Library.Main.Strings {
}
}
+ ///
+ /// Looks up a localized string similar to A fragment of memory is used to reduce database lookups. You should not change this value unless you get warnings in the log..
+ ///
+ internal static string FhblockhashlookupsizeLong {
+ get {
+ return ResourceManager.GetString("FhblockhashlookupsizeLong", resourceCulture);
+ }
+ }
+
+ ///
+ /// Looks up a localized string similar to Memory used by the block hash.
+ ///
+ internal static string FhblockhashlookupsizeShort {
+ get {
+ return ResourceManager.GetString("FhblockhashlookupsizeShort", resourceCulture);
+ }
+ }
+
///
/// Looks up a localized string similar to The blocksize determines how files are fragmented. Choosing a large value will cause a larger overhead on file changes, choosing a small value will cause a large overhead on storage of file lists. Note that the value cannot be changed after remote files are created..
///
@@ -96,6 +114,24 @@ namespace Duplicati.Library.Main.Strings {
}
}
+ ///
+ /// Looks up a localized string similar to A fragment of memory is used to reduce database lookups. You should not change this value unless you get warnings in the log..
+ ///
+ internal static string FhfilehashlookupsizeLong {
+ get {
+ return ResourceManager.GetString("FhfilehashlookupsizeLong", resourceCulture);
+ }
+ }
+
+ ///
+ /// Looks up a localized string similar to Memory used by the file hash.
+ ///
+ internal static string FhfilehashlookupsizeShort {
+ get {
+ return ResourceManager.GetString("FhfilehashlookupsizeShort", resourceCulture);
+ }
+ }
+
///
/// Looks up a localized string similar to Stores empty metadata, such as file timestamps and attributes. This decreases the required storage space as well as the processing time..
///
diff --git a/Duplicati/Library/Main/Strings/FhOptions.resx b/Duplicati/Library/Main/Strings/FhOptions.resx
index e272c306a..f1e68bf5b 100644
--- a/Duplicati/Library/Main/Strings/FhOptions.resx
+++ b/Duplicati/Library/Main/Strings/FhOptions.resx
@@ -117,6 +117,12 @@
System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089
+
+ A fragment of memory is used to reduce database lookups. You should not change this value unless you get warnings in the log.
+
+
+ Memory used by the block hash
+
The blocksize determines how files are fragmented. Choosing a large value will cause a larger overhead on file changes, choosing a small value will cause a large overhead on storage of file lists. Note that the value cannot be changed after remote files are created.
@@ -129,6 +135,12 @@
Path to the local state database
+
+ A fragment of memory is used to reduce database lookups. You should not change this value unless you get warnings in the log.
+
+
+ Memory used by the file hash
+
Stores empty metadata, such as file timestamps and attributes. This decreases the required storage space as well as the processing time.