haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimized block queries

Browse Source
This commit is contained in:
Kenneth Skovhede
2013-03-10 20:33:30 +01:00
parent 5863292f11
commit f055dfd2b4
1 changed files with 25 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs
+25 -1
View File
@@ -43,6 +43,8 @@ namespace Duplicati.Library.Main.ForestHash.Database
private long m_falseBlockPositives = 0;
private long m_falseFilePositives = 0;
private long m_missingBlockHashes;
public LocalBackupDatabase(string path)
: this(CreateConnection(path))
{
@@ -136,7 +138,7 @@ namespace Duplicati.Library.Main.ForestHash.Database
m_fileHashLookup = new HashPrefixLookup(FILE_HASH_LOOKUP_SIZE);
using (var cmd = m_connection.CreateCommand())
using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Hash"" FROM ""Block"""))
using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Block"".""Hash"" FROM ""Block"", ""RemoteVolume"" WHERE ""RemoteVolume"".""Name"" = ""Block"".""File"" AND ""State"" IN (?,?,?,?) ", RemoteVolumeState.Temporary, RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified))
while (rd.Read())
m_blockHashLookup.AddHash(Convert.FromBase64String(rd.GetValue(0).ToString()));
@@ -145,6 +147,9 @@ namespace Duplicati.Library.Main.ForestHash.Database
while (rd.Read())
m_fileHashLookup.AddHash(Convert.FromBase64String(rd.GetValue(0).ToString()));
using (var cmd = m_connection.CreateCommand())
m_missingBlockHashes = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT (*) FROM (SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"" FROM ""Block"", ""RemoteVolume"" WHERE ""RemoteVolume"".""Name"" = ""Block"".""File"" AND ""State"" NOT IN (?,?,?,?))", RemoteVolumeState.Temporary, RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified));
}
/// <summary>
@@ -162,7 +167,14 @@ namespace Duplicati.Library.Main.ForestHash.Database
m_findblockCommand.Transaction = transaction;
r = m_findblockCommand.ExecuteScalar(null, key, size);
if (r == null || r == DBNull.Value)
{
m_falseBlockPositives++;
}
else
{
if (m_missingBlockHashes == 0)
return false;
}
}
if (r == null || r == DBNull.Value)
@@ -177,6 +189,7 @@ namespace Duplicati.Library.Main.ForestHash.Database
}
else
{
//If the block is found and the volume is broken somehow.
m_findremotevolumestateCommand.Transaction = transaction;
r = m_findremotevolumestateCommand.ExecuteScalar(null, r);
if (r != null && (r.ToString() == RemoteVolumeState.Temporary.ToString() || r.ToString() == RemoteVolumeState.Uploading.ToString() || r.ToString() == RemoteVolumeState.Uploaded.ToString() || r.ToString() == RemoteVolumeState.Verified.ToString()))
@@ -570,6 +583,17 @@ namespace Duplicati.Library.Main.ForestHash.Database
m_stat.ModifiedFiles = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT * FROM ( (SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""Fileset"".""ID"" WHERE ""OperationID"" = ?) AS A, (SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""Fileset"".""ID"" WHERE ""OperationID"" = ?) AS B WHERE ""A"".""Path"" = ""B"".""Path"" AND (""A"".""BlocsetID"" != ""B"".""BlocksetID"" OR ""A"".""MetadataID"" != ""B"".""MetadataID"") )", m_operationid, lastOperationId));
* */
}
if (m_falseBlockPositives > 200 || m_falseFilePositives > 20)
{
m_stat.LogWarning(string.Format("Lookup tables gave false positives, this may indicate too small tables. Block: {0}, File: {0}", m_falseBlockPositives, m_falseFilePositives), null);
if (m_blockHashLookup.TableUsageRatio > 0.5)
m_stat.LogWarning(string.Format("Block hash lookup table is too small, usage is: {0}%", m_blockHashLookup.TableUsageRatio * 100), null);
if (m_fileHashLookup.TableUsageRatio > 0.5)
m_stat.LogWarning(string.Format("File hash lookup table is too small, usage is: {0}%", m_fileHashLookup.TableUsageRatio * 100), null);
}
}
}