haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactored the local database.

Browse Source 
Fixed an issue when recreating the database, the operation was not created, making compact and delete commands not working.
This commit is contained in:
Kenneth Skovhede
2013-04-03 21:06:27 +02:00
parent a564891b79
commit 4566ea1c3d
10 changed files with 1407 additions and 1364 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Main/ForestHash/Database/Database schema/Schema.sql
+28 -13
View File
@@ -25,18 +25,24 @@ CREATE TABLE "Remotevolume" (
);
/*
The fileset is a list of files contained
in a single operation, grouped by an
operationID
The OperationFileset contains an
entry for each file scanned for
a single operation. The scantime
is the time the file was last
scanned
*/
CREATE TABLE "OperationFileset" (
"OperationID" INTEGER NOT NULL,
"FilesetID" INTEGER NOT NULL,
"FileEntryID" INTEGER NOT NULL,
"Scantime" DATETIME NOT NULL
);
CREATE TABLE "Fileset" (
/*
The FileEntry contains an ID
for each path and each version
of the data and metadata
*/
CREATE TABLE "FileEntry" (
"ID" INTEGER PRIMARY KEY,
"Path" TEXT NOT NULL,
"BlocksetID" INTEGER NOT NULL,
@@ -45,7 +51,9 @@ CREATE TABLE "Fileset" (
/*
The blocklist hashes are hashes of
fragments of the blocklists
fragments of the blocklists.
They are grouped by the BlocksetID
and ordered by the index
*/
CREATE TABLE "BlocklistHash" (
"BlocksetID" INTEGER NOT NULL,
@@ -53,7 +61,6 @@ CREATE TABLE "BlocklistHash" (
"Hash" TEXT NOT NULL
);
/*
The blockset is a list of blocks
@@ -73,7 +80,9 @@ CREATE TABLE "Blockset" (
/*
The elements of a blocklist,
the hash is the block hash
the hash is the block hash,
they are grouped by the BlocksetID
and ordered by the index
*/
CREATE TABLE "BlocksetEntry" (
"BlocksetID" INTEGER NOT NULL,
@@ -107,7 +116,8 @@ CREATE TABLE "DeletedBlock" (
/*
A metadata set
A metadata set, essentially a placeholder
to easily extend metadatasets with new properties
*/
CREATE TABLE "Metadataset" (
"ID" INTEGER PRIMARY KEY,
@@ -115,7 +125,9 @@ CREATE TABLE "Metadataset" (
);
/*
Operations performed on the backend
Operations performed on the backend,
intended to be used when constructing
an error report or when debugging
*/
CREATE TABLE "RemoteOperation" (
"ID" INTEGER PRIMARY KEY,
@@ -127,7 +139,9 @@ CREATE TABLE "RemoteOperation" (
);
/*
Logged events
Logged events, intended to be used when
constructing an error report or when
debugging
*/
CREATE TABLE "LogData" (
"ID" INTEGER PRIMARY KEY,
@@ -147,7 +161,8 @@ CREATE TABLE "Version" (
);
/*
Settings, such as hash and blocksize
Settings, such as hash and blocksize,
used for verification
*/
CREATE TABLE "Configuration" (
"Key" TEXT PRIMARY KEY NOT NULL,
Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs
+34 -33
View File
@@ -96,17 +96,17 @@ namespace Duplicati.Library.Main.ForestHash.Database
m_findmetadatasetCommand.CommandText = @"SELECT ""A"".""ID"" FROM ""Metadataset"" A, ""BlocksetEntry"" B, ""Block"" C WHERE ""A"".""BlocksetID"" = ""B"".""BlocksetID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" AND ""C"".""Hash"" = ? AND ""C"".""Size"" = ? LIMIT 1";
m_findmetadatasetCommand.AddParameters(2);
m_findfilesetCommand.CommandText = @"SELECT ""ID"" FROM ""Fileset"" WHERE ""BlocksetID"" = ? AND ""MetadatasetID"" = ? AND ""Path"" = ?";
m_findfilesetCommand.CommandText = @"SELECT ""ID"" FROM ""FileEntry"" WHERE ""BlocksetID"" = ? AND ""MetadatasetID"" = ? AND ""Path"" = ?";
m_findfilesetCommand.AddParameters(3);
m_insertblockCommand.CommandText = @"INSERT INTO ""Block"" (""Hash"", ""VolumeID"", ""Size"") VALUES (?, ?, ?)";
m_insertblockCommand.AddParameters(3);
m_insertfileOperationCommand.CommandText = @"INSERT INTO ""OperationFileset"" (""OperationID"", ""FilesetID"", ""Scantime"") VALUES (?, ?, ?)";
m_insertfileOperationCommand.CommandText = @"INSERT INTO ""OperationFileset"" (""OperationID"", ""FileEntryID"", ""Scantime"") VALUES (?, ?, ?)";
m_insertfileOperationCommand.AddParameter(m_operationid);
m_insertfileOperationCommand.AddParameters(2);
m_insertfileCommand.CommandText = @"INSERT INTO ""Fileset"" (""Path"",""BlocksetID"", ""MetadataID"") VALUES (?, ? ,?); SELECT last_insert_rowid();";
m_insertfileCommand.CommandText = @"INSERT INTO ""FileEntry"" (""Path"",""BlocksetID"", ""MetadataID"") VALUES (?, ? ,?); SELECT last_insert_rowid();";
m_insertfileCommand.AddParameters(3);
m_insertblocksetCommand.CommandText = @"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (?, ?); SELECT last_insert_rowid();";
@@ -124,12 +124,12 @@ namespace Duplicati.Library.Main.ForestHash.Database
//Need a temporary table with path/scantime lookups
m_scantimelookupTablename = "ScanTime-" + Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
using (var cmd = m_connection.CreateCommand())
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT ""OperationFileset"".""FilesetID"" AS ""FilesetID"", MAX(""OperationFileset"".""Scantime"") AS ""Scantime"", ""Fileset"".""Path"" AS ""Path"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" GROUP BY ""FilesetID"", ""Path"" ", m_scantimelookupTablename));
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT ""OperationFileset"".""FileEntryID"" AS ""FileEntryID"", MAX(""OperationFileset"".""Scantime"") AS ""Scantime"", ""FileEntry"".""Path"" AS ""Path"" FROM ""OperationFileset"" INNER JOIN ""FileEntry"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" GROUP BY ""OperationFileset"".""FileEntryID"", ""FileEntry"".""Path"" ", m_scantimelookupTablename));
m_selectfileSimpleCommand.CommandText = string.Format(@"SELECT ""FilesetID"", ""Scantime"" FROM ""{0}"" WHERE ""Path"" = ?", m_scantimelookupTablename);
m_selectfileSimpleCommand.CommandText = string.Format(@"SELECT ""FileEntryID"", ""Scantime"" FROM ""{0}"" WHERE ""Path"" = ?", m_scantimelookupTablename);
m_selectfileSimpleCommand.AddParameters(1);
m_selectfileHashCommand.CommandText = @"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"" INNER JOIN ""Fileset"" ON ""Blockset"".""ID"" = ""Fileset"".""BlocksetID"" WHERE ""Fileset"".""ID"" = ? ";
m_selectfileHashCommand.CommandText = @"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"" INNER JOIN ""FileEntry"" ON ""Blockset"".""ID"" = ""FileEntry"".""BlocksetID"" WHERE ""FileEntry"".""ID"" = ? ";
m_selectfileHashCommand.AddParameters(1);
m_selectblocklistHashesCommand.CommandText = @"SELECT ""Hash"" FROM ""BlocklistHash"" WHERE ""BlocksetID"" = ? ORDER BY ""Index"" ASC ";
@@ -187,7 +187,7 @@ namespace Duplicati.Library.Main.ForestHash.Database
}
if (m_fileScantimeLookup != null)
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""FilesetID"", ""Scantime"", ""Path"" FROM ""{0}"" ", m_scantimelookupTablename)))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""FileEntryID"", ""Scantime"", ""Path"" FROM ""{0}"" ", m_scantimelookupTablename)))
while (rd.Read())
{
var id = Convert.ToInt64(rd.GetValue(0));
@@ -197,7 +197,7 @@ namespace Duplicati.Library.Main.ForestHash.Database
}
if (m_filesetLookup != null)
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""Path"", ""BlocksetID"", ""MetadataID"", ""ID"" FROM ""Fileset"" ")))
using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""Path"", ""BlocksetID"", ""MetadataID"", ""ID"" FROM ""FileEntry"" ")))
while (rd.Read())
{
var path = rd.GetValue(0).ToString();
@@ -704,8 +704,7 @@ namespace Duplicati.Library.Main.ForestHash.Database
{
using (var cmd = m_connection.CreateCommand())
{
//cmd.CommandText = @"SELECT ""A"".""ID"", ""D"".""Scantime"", ""B"".""Length"", ""B"".""FullHash"", ""E"".""FullHash"", ""E"".""Length"", ""B"".""ID"" FROM ""Fileset"" A, ""Blockset"" B, ""Metadataset"" C, ""OperationFileset"" D, ""Blockset"" E, ""Operation"" F WHERE ""A"".""ID"" = ""D"".""FilesetID"" AND ""F"".""ID"" = ? AND ""A"".""BlocksetID"" = ""B"".""ID"" AND ""A"".""MetadataID"" = ""C"".""ID"" AND ""E"".""ID"" = ""C"".""BlocksetID"" AND ""F"".""ID"" = ""D"".""OperationID"" ORDER BY ""F"".""Timestamp"" DESC ";
cmd.CommandText = @"SELECT ""B"".""BlocksetID"", ""B"".""ID"", ""B"".""Path"", ""D"".""Length"", ""D"".""FullHash"", ""A"".""Scantime"" FROM ""OperationFileset"" A, ""Fileset"" B, ""Metadataset"" C, ""Blockset"" D WHERE ""A"".""FilesetID"" = ""B"".""ID"" AND ""B"".""MetadataID"" = ""C"".""ID"" AND ""C"".""BlocksetID"" = ""D"".""ID"" AND (""B"".""BlocksetID"" = ? OR ""B"".""BlocksetID"" = ?) AND ""A"".""OperationID"" = ? ";
cmd.CommandText = @"SELECT ""B"".""BlocksetID"", ""B"".""ID"", ""B"".""Path"", ""D"".""Length"", ""D"".""FullHash"", ""A"".""Scantime"" FROM ""OperationFileset"" A, ""FileEntry"" B, ""Metadataset"" C, ""Blockset"" D WHERE ""A"".""FileEntryID"" = ""B"".""ID"" AND ""B"".""MetadataID"" = ""C"".""ID"" AND ""C"".""BlocksetID"" = ""D"".""ID"" AND (""B"".""BlocksetID"" = ? OR ""B"".""BlocksetID"" = ?) AND ""A"".""OperationID"" = ? ";
cmd.AddParameter(FOLDER_BLOCKSET_ID);
cmd.AddParameter(SYMLINK_BLOCKSET_ID);
cmd.AddParameter(operationId == null ? m_operationid : operationId.Value);
@@ -724,7 +723,7 @@ namespace Duplicati.Library.Main.ForestHash.Database
filesetvolume.AddSymlink(path, metahash, metalength);
}
cmd.CommandText = @"SELECT ""F"".""Path"", ""F"".""Scantime"", ""F"".""Filelength"", ""F"".""Filehash"", ""F"".""Metahash"", ""F"".""Metalength"", ""G"".""Hash"" FROM (SELECT ""A"".""Path"" AS ""Path"", ""D"".""Scantime"" AS ""Scantime"", ""B"".""Length"" AS ""Filelength"", ""B"".""FullHash"" AS ""Filehash"", ""E"".""FullHash"" AS ""Metahash"", ""E"".""Length"" AS ""Metalength"", ""A"".""BlocksetID"" AS ""BlocksetID"" FROM ""Fileset"" A, ""Blockset"" B, ""Metadataset"" C, ""OperationFileset"" D, ""Blockset"" E WHERE ""A"".""ID"" = ""D"".""FilesetID"" AND ""D"".""OperationID"" = ? AND ""A"".""BlocksetID"" = ""B"".""ID"" AND ""A"".""MetadataID"" = ""C"".""ID"" AND ""E"".""ID"" = ""C"".""BlocksetID"") F LEFT OUTER JOIN ""BlocklistHash"" G ON ""G"".""BlocksetID"" = ""F"".""BlocksetID"" ORDER BY ""F"".""Path"", ""G"".""Index"" ";
cmd.CommandText = @"SELECT ""F"".""Path"", ""F"".""Scantime"", ""F"".""Filelength"", ""F"".""Filehash"", ""F"".""Metahash"", ""F"".""Metalength"", ""G"".""Hash"" FROM (SELECT ""A"".""Path"" AS ""Path"", ""D"".""Scantime"" AS ""Scantime"", ""B"".""Length"" AS ""Filelength"", ""B"".""FullHash"" AS ""Filehash"", ""E"".""FullHash"" AS ""Metahash"", ""E"".""Length"" AS ""Metalength"", ""A"".""BlocksetID"" AS ""BlocksetID"" FROM ""FileEntry"" A, ""Blockset"" B, ""Metadataset"" C, ""OperationFileset"" D, ""Blockset"" E WHERE ""A"".""ID"" = ""D"".""FileEntryID"" AND ""D"".""OperationID"" = ? AND ""A"".""BlocksetID"" = ""B"".""ID"" AND ""A"".""MetadataID"" = ""C"".""ID"" AND ""E"".""ID"" = ""C"".""BlocksetID"") F LEFT OUTER JOIN ""BlocklistHash"" G ON ""G"".""BlocksetID"" = ""F"".""BlocksetID"" ORDER BY ""F"".""Path"", ""G"".""Index"" ";
cmd.Parameters.Clear();
cmd.AddParameter(m_operationid);
@@ -795,28 +794,34 @@ namespace Duplicati.Library.Main.ForestHash.Database
base.Dispose();
}
private long GetPreviousBackupOperationID(System.Data.IDbCommand cmd)
{
long lastOperationId = -1;
var lastIdObj = cmd.ExecuteScalar(@"SELECT ""ID"" FROM ""Operation"" WHERE ""Timestamp"" < ? AND ""Description"" = ? ORDER BY ""Timestamp"" DESC ", OperationTimestamp, "Backup");
if (lastIdObj != null && lastIdObj != DBNull.Value)
lastOperationId = Convert.ToInt64(lastIdObj);
return lastOperationId;
}
internal void UpdateChangeStatistics (BackupStatistics m_stat)
{
using (var cmd = m_connection.CreateCommand())
{
long lastOperationId = -1;
var lastOperationId = GetPreviousBackupOperationID(cmd);
m_stat.AddedFolders = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""FileEntry"".""BlocksetID"" = ? AND NOT ""FileEntry"".""Path"" IN (SELECT ""Path"" FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ?)", m_operationid, FOLDER_BLOCKSET_ID, lastOperationId));
m_stat.AddedSymlinks = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""FileEntry"".""BlocksetID"" = ? AND NOT ""FileEntry"".""Path"" IN (SELECT ""Path"" FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ?)", m_operationid, SYMLINK_BLOCKSET_ID, lastOperationId));
m_stat.AddedFiles = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""FileEntry"".""BlocksetID"" != ? AND ""FileEntry"".""BlocksetID"" != ? AND NOT ""FileEntry"".""Path"" IN (SELECT ""Path"" FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ?)", m_operationid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, lastOperationId));
var lastIdObj = cmd.ExecuteScalar(@"SELECT ""ID"" FROM ""Operation"" WHERE ""Timestamp"" < ? AND ""Description"" = ? ORDER BY ""Timestamp"" DESC ", OperationTimestamp, "Backup");
if (lastIdObj != null && lastIdObj != DBNull.Value)
lastOperationId = Convert.ToInt64(lastIdObj);
m_stat.DeletedFolders = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""FileEntry"".""BlocksetID"" = ? AND NOT ""FileEntry"".""Path"" IN (SELECT ""Path"" FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, m_operationid));
m_stat.DeletedSymlinks = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""FileEntry"".""BlocksetID"" = ? AND NOT ""FileEntry"".""Path"" IN (SELECT ""Path"" FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, SYMLINK_BLOCKSET_ID, m_operationid));
m_stat.DeletedFiles = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""FileEntry"".""BlocksetID"" != ? AND ""FileEntry"".""BlocksetID"" != ? AND NOT ""FileEntry"".""Path"" IN (SELECT ""Path"" FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, m_operationid));
m_stat.AddedFolders = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" = ? AND NOT ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", m_operationid, FOLDER_BLOCKSET_ID, lastOperationId));
m_stat.AddedSymlinks = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" = ? AND NOT ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", m_operationid, SYMLINK_BLOCKSET_ID, lastOperationId));
m_stat.AddedFiles = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" != ? AND ""Fileset"".""BlocksetID"" != ? AND NOT ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", m_operationid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, lastOperationId));
m_stat.DeletedFolders = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" = ? AND NOT ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, m_operationid));
m_stat.DeletedSymlinks = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" = ? AND NOT ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, SYMLINK_BLOCKSET_ID, m_operationid));
m_stat.DeletedFiles = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" != ? AND ""Fileset"".""BlocksetID"" != ? AND NOT ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, m_operationid));
m_stat.ModifiedFolders = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" = ? AND ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, m_operationid));
m_stat.ModifiedSymlinks = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" = ? AND ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, SYMLINK_BLOCKSET_ID, m_operationid));
m_stat.ModifiedFiles = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" != ? AND ""Fileset"".""BlocksetID"" != ? AND ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, m_operationid));
m_stat.ModifiedFolders = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""FileEntry"".""BlocksetID"" = ? AND ""FileEntry"".""Path"" IN (SELECT ""Path"" FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, m_operationid));
m_stat.ModifiedSymlinks = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""FileEntry"".""BlocksetID"" = ? AND ""FileEntry"".""Path"" IN (SELECT ""Path"" FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, SYMLINK_BLOCKSET_ID, m_operationid));
m_stat.ModifiedFiles = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""FileEntry"".""BlocksetID"" != ? AND ""FileEntry"".""BlocksetID"" != ? AND ""FileEntry"".""Path"" IN (SELECT ""Path"" FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, m_operationid));
}
if (m_blockHashLookup != null && m_fileHashLookup != null && (m_blockHashLookup.FalsePositives > 200 || m_fileHashLookup.FalsePositives > 20))
@@ -878,18 +883,14 @@ namespace Duplicati.Library.Main.ForestHash.Database
using(var cmd = m_connection.CreateCommand())
using (var tr = new TemporaryTransactionWrapper(m_connection, transaction))
{
long lastOperationId = -1;
var lastIdObj = cmd.ExecuteScalar(@"SELECT ""ID"" FROM ""Operation"" WHERE ""Timestamp"" < ? AND ""Description"" = ? ORDER BY ""Timestamp"" DESC ", OperationTimestamp, "Backup");
if (lastIdObj != null && lastIdObj != DBNull.Value)
lastOperationId = Convert.ToInt64(lastIdObj);
long lastOperationId = GetPreviousBackupOperationID(cmd);
cmd.Transaction = tr.Parent;
cmd.ExecuteNonQuery(@"INSERT INTO ""OperationFileset"" (""OperationID"", ""FilesetID"", ""Scantime"") SELECT ? AS ""OperationID"", ""FilesetID"", ""Scantime"" FROM ""OperationFile"" WHERE ""OperationID"" = ? AND ""FilesetID"" NOT IN (SELECT ""FilesetID"" FROM ""OperationFileset"" WHERE ""OperationID"" = ?) ", m_operationid, lastOperationId, m_operationid);
cmd.ExecuteNonQuery(@"INSERT INTO ""OperationFileset"" (""OperationID"", ""FileEntryID"", ""Scantime"") SELECT ? AS ""OperationID"", ""FileEntryID"", ""Scantime"" FROM ""OperationFile"" WHERE ""OperationID"" = ? AND ""FileEntryID"" NOT IN (SELECT ""FileEntryID"" FROM ""OperationFileset"" WHERE ""OperationID"" = ?) ", m_operationid, lastOperationId, m_operationid);
if (deleted != null)
{
cmd.CommandText = @"DELETE FROM ""OperationFileset"" WHERE ""OperationID"" = ? AND ""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" WHERE ""Path"" = ?) ";
cmd.CommandText = @"DELETE FROM ""OperationFileset"" WHERE ""OperationID"" = ? AND ""FileEntryID"" IN (SELECT ""ID"" FROM ""FileEntry"" WHERE ""Path"" = ?) ";
cmd.AddParameter(m_operationid);
cmd.AddParameter();
Duplicati/Library/Main/ForestHash/Database/LocalBlocklistUpdateDatabase.cs
+114 -96
View File
@@ -1,56 +1,56 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
namespace Duplicati.Library.Main.ForestHash.Database
{
public partial class LocalBlocklistUpdateDatabase : LocalRestoreDatabase
{
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
namespace Duplicati.Library.Main.ForestHash.Database
{
public partial class LocalBlocklistUpdateDatabase : LocalRestoreDatabase
{
protected string m_tempblockvolumetable;
protected string m_temphashtable;
public LocalBlocklistUpdateDatabase(string path, long blocksize)
: base("Rebuild", path, blocksize)
{
}
public void FindMissingBlocklistHashes()
{
protected string m_temphashtable;
public LocalBlocklistUpdateDatabase(string path, long blocksize)
: base("Rebuild", path, blocksize)
{
}
public void FindMissingBlocklistHashes()
{
m_tempblockvolumetable = "MissingBlocks-" + Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
m_temphashtable = "Hashlist-" + Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
using (var cmd = m_connection.CreateCommand())
{
using (var cmd = m_connection.CreateCommand())
{
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT DISTINCT ""Block"".""VolumeID"" AS ""VolumeID"", ""BlocklistHash"".""Hash"" AS ""Hash"", 0 AS ""Restored"" FROM ""Block"", ""BlocklistHash"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" ", m_tempblockvolumetable));
cmd.ExecuteNonQuery(string.Format(@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") SELECT DISTINCT ""Blockset"".""ID"", 0, ""Block"".""ID"" FROM ""Blockset"", ""Block"" WHERE ""Blockset"".""Fullhash"" = ""Block"".""Hash"" AND ""Blockset"".""Length"" < {0} AND ""Blockset"".""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"") ", m_blocksize));
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Hash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)", m_temphashtable));
}
}
public IEnumerable<string> GetBlockLists(long volumeid)
{
return new BlocklistsEnumerable(m_connection, m_tempblockvolumetable, volumeid);
}
public void UpdateBlocklist(string hash, IEnumerable<string> hashes, long hashsize, System.Data.IDbTransaction transaction)
{
using (var cmd = m_connection.CreateCommand())
{
cmd.Transaction = transaction;
cmd.CommandText = string.Format(@"UPDATE ""{0}"" SET ""Restored"" = 1 WHERE ""Restored"" = 0 AND ""Hash"" = ? ", m_tempblockvolumetable);
cmd.AddParameter(hash);
var c = cmd.ExecuteNonQuery();
if (c != 1)
}
}
public IEnumerable<string> GetBlockLists(long volumeid)
{
return new BlocklistsEnumerable(m_connection, m_tempblockvolumetable, volumeid);
}
public void UpdateBlocklist(string hash, IEnumerable<string> hashes, long hashsize, System.Data.IDbTransaction transaction)
{
using (var cmd = m_connection.CreateCommand())
{
cmd.Transaction = transaction;
cmd.CommandText = string.Format(@"UPDATE ""{0}"" SET ""Restored"" = 1 WHERE ""Restored"" = 0 AND ""Hash"" = ? ", m_tempblockvolumetable);
cmd.AddParameter(hash);
var c = cmd.ExecuteNonQuery();
if (c != 1)
throw new Exception(string.Format("Blocklist not found {0}", hash));
cmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""{0}""", m_temphashtable));
cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Hash"", ""Index"") VALUES (?, ?)", m_temphashtable);
long ix = 0;
foreach (var h in hashes)
{
{
c = cmd.ExecuteNonQuery(null, h, ix);
if (c != 1)
throw new Exception(string.Format("Insert row error {0}", h));
@@ -58,58 +58,76 @@ namespace Duplicati.Library.Main.ForestHash.Database
}
cmd.CommandText = string.Format(@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") SELECT ""A"".""BlocksetID"", (""A"".""Index"" * ?) + ""B"".""Index"", ""C"".""ID"" FROM ""BlocklistHash"" A, ""{0}"" B, ""Block"" C WHERE ""A"".""Hash"" = ? AND ""C"".""Hash"" = ""B"".""Hash"" ", m_temphashtable);
c = cmd.ExecuteNonQuery(null, (m_blocksize / hashsize), hash);
c = cmd.ExecuteNonQuery(null, (m_blocksize / hashsize), hash);
if (c == 0 || c % ix != 0)
throw new Exception(string.Format("Wrong number of inserts, got {0} records from {1} hashes!", c, ix));
}
}
public IList<IRemoteVolume> GetMissingBlockListVolumes()
{
using (var cmd = m_connection.CreateCommand())
{
List<IRemoteVolume> result = new List<IRemoteVolume>();
cmd.CommandText = string.Format(@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT DISTINCT ""VolumeID"" FROM ""{0}"")", m_tempblockvolumetable);
using (var rd = cmd.ExecuteReader())
{
object[] r = new object[3];
while (rd.Read())
{
rd.GetValues(r);
result.Add(new RemoteVolume(
r[0] == null ? null : r[0].ToString(),
r[1] == null ? null : r[1].ToString(),
r[2] == null ? -1 : Convert.ToInt64(r[2])
));
}
}
return result;
}
}
public void VerifyDatabaseIntegrity()
{
using (var cmd = m_connection.CreateCommand())
{
cmd.CommandText = @"SELECT COUNT(*) FROM (SELECT SUM(""Block"".""Size"") AS ""CalcLen"", ""Blockset"".""Length"" AS ""Length"", ""BlocksetEntry"".""BlocksetID"" FROM ""Block"", ""BlocksetEntry"", ""Blockset"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Blockset"".""ID"" = ""BlocksetEntry"".""BlocksetID"" GROUP BY ""BlocksetEntry"".""BlocksetID"") WHERE ""CalcLen"" != ""Length""";
var c = Convert.ToInt64(cmd.ExecuteScalar());
if (c != 0)
throw new InvalidDataException("Inconsistency detected, not all blocklists were restored correctly");
}
}
public override void Dispose()
{
base.Dispose();
using (var cmd = m_connection.CreateCommand())
{
if (m_tempblockvolumetable != null)
try
{
cmd.CommandText = string.Format(@"DROP TABLE ""{0}""", m_tempblockvolumetable);
cmd.ExecuteNonQuery();
}
throw new Exception(string.Format("Wrong number of inserts, got {0} records from {1} hashes!", c, ix));
}
}
public IList<IRemoteVolume> GetMissingBlockListVolumes()
{
using (var cmd = m_connection.CreateCommand())
{
List<IRemoteVolume> result = new List<IRemoteVolume>();
cmd.CommandText = string.Format(@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT DISTINCT ""VolumeID"" FROM ""{0}"")", m_tempblockvolumetable);
using (var rd = cmd.ExecuteReader())
{
object[] r = new object[3];
while (rd.Read())
{
rd.GetValues(r);
result.Add(new RemoteVolume(
r[0] == null ? null : r[0].ToString(),
r[1] == null ? null : r[1].ToString(),
r[2] == null ? -1 : Convert.ToInt64(r[2])
));
}
}
return result;
}
}
/// <summary>
/// Creates a timestamped backup operation to correctly associate the fileset with the time it was created.
/// </summary>
/// <param name="volumeid">The ID of the fileset volume to update</param>
/// <param name="timestamp">The timestamp of the operation to create</param>
/// <param name="transaction">An optional external transaction</param>
public void CreateBackupOperation(long volumeid, DateTime timestamp, System.Data.IDbTransaction transaction = null)
{
using (var cmd = m_connection.CreateCommand())
using (var tr = new TemporaryTransactionWrapper(m_connection, transaction))
{
cmd.Transaction = tr.Parent;
var operationid = cmd.ExecuteNonQuery(@"INSERT INTO ""Operation"" (""Description"", ""Timestamp"") VALUES (?, ?); SELECT last_insert_rowid()", "Backup", timestamp);
cmd.ExecuteNonQuery(@"UPDATE ""RemoteVolume"" SET ""OperationID"" = ? WHERE ""ID"" = ?", operationid, volumeid);
tr.Commit();
}
}
public void VerifyDatabaseIntegrity()
{
using (var cmd = m_connection.CreateCommand())
{
cmd.CommandText = @"SELECT COUNT(*) FROM (SELECT SUM(""Block"".""Size"") AS ""CalcLen"", ""Blockset"".""Length"" AS ""Length"", ""BlocksetEntry"".""BlocksetID"" FROM ""Block"", ""BlocksetEntry"", ""Blockset"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Blockset"".""ID"" = ""BlocksetEntry"".""BlocksetID"" GROUP BY ""BlocksetEntry"".""BlocksetID"") WHERE ""CalcLen"" != ""Length""";
var c = Convert.ToInt64(cmd.ExecuteScalar());
if (c != 0)
throw new InvalidDataException("Inconsistency detected, not all blocklists were restored correctly");
}
}
public override void Dispose()
{
base.Dispose();
using (var cmd = m_connection.CreateCommand())
{
if (m_tempblockvolumetable != null)
try
{
cmd.CommandText = string.Format(@"DROP TABLE ""{0}""", m_tempblockvolumetable);
cmd.ExecuteNonQuery();
}
finally { m_tempblockvolumetable = null; }
if (m_temphashtable != null)
@@ -119,7 +137,7 @@ namespace Duplicati.Library.Main.ForestHash.Database
cmd.ExecuteNonQuery();
}
finally { m_temphashtable = null; }
}
}
}
}
}
}
}
}
Duplicati/Library/Main/ForestHash/Database/LocalCleanupDatabase.cs
+4 -4
View File
@@ -147,7 +147,7 @@ namespace Duplicati.Library.Main.ForestHash.Database
public IEnumerable<IBlockWithSources> GetSourceFilesWithBlocks(long volumeid, long blocksize)
{
using(var cmd = m_connection.CreateCommand())
using(var rd = cmd.ExecuteReader(string.Format(@"SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"", ""Fileset"".""Path"", ""BlocksetEntry"".""Index"" * {0} FROM ""Block"", ""BlocksetEntry"", ""Fileset"" WHERE ""Fileset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""Block"".""VolumeID"" = ? ", blocksize), volumeid))
using(var rd = cmd.ExecuteReader(string.Format(@"SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"", ""FileEntry"".""Path"", ""BlocksetEntry"".""Index"" * {0} FROM ""Block"", ""BlocksetEntry"", ""FileEntry"" WHERE ""FileEntry"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""Block"".""VolumeID"" = ? ", blocksize), volumeid))
if (rd.Read())
{
var bs = new BlockWithSources(rd);
@@ -166,10 +166,10 @@ namespace Duplicati.Library.Main.ForestHash.Database
public IEnumerable<IRemoteVolume> GetFilesetsUsingBlock(string hash, int size)
{
var blocks = @"SELECT DISTINCT ""Fileset"".""ID"" AS ID FROM ""Block"", ""Blockset"", ""BlocksetEntry"", ""Fileset"" WHERE ""Block"".""Hash"" = ? AND ""Block"".""Size"" = ? AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Fileset"".""BlocksetID"" = ""Blockset"".""ID"" ";
var blocklists = @"SELECT DISTINCT ""Fileset"".""ID"" AS ID FROM ""Block"", ""Blockset"", ""BlocklistHash"", ""Fileset"" WHERE ""Block"".""Hash"" = ? AND ""Block"".""Size"" = ? AND ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""BlocklistHash"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Fileset"".""BlocksetID"" = ""Blockset"".""ID"" ";
var blocks = @"SELECT DISTINCT ""FileEntry"".""ID"" AS ID FROM ""Block"", ""Blockset"", ""BlocksetEntry"", ""FileEntry"" WHERE ""Block"".""Hash"" = ? AND ""Block"".""Size"" = ? AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileEntry"".""BlocksetID"" = ""Blockset"".""ID"" ";
var blocklists = @"SELECT DISTINCT ""FileEntry"".""ID"" AS ID FROM ""Block"", ""Blockset"", ""BlocklistHash"", ""FileEntry"" WHERE ""Block"".""Hash"" = ? AND ""Block"".""Size"" = ? AND ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""BlocklistHash"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileEntry"".""BlocksetID"" = ""Blockset"".""ID"" ";
var cmdtxt = @"SELECT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""OperationFileset"" WHERE ""RemoteVolume"".""OperationID"" = ""OperationFileset"".""OperationID"" AND ""RemoteVolume"".""Type"" = ? AND ""OperationFileset"".""FilesetID"" IN (SELECT DISTINCT ""ID"" FROM ( " + blocks + " UNION " + blocklists + " ))";
var cmdtxt = @"SELECT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""OperationFileset"" WHERE ""RemoteVolume"".""OperationID"" = ""OperationFileset"".""OperationID"" AND ""RemoteVolume"".""Type"" = ? AND ""OperationFileset"".""FileEntryID"" IN (SELECT DISTINCT ""ID"" FROM ( " + blocks + " UNION " + blocklists + " ))";
using(var cmd = m_connection.CreateCommand())
using(var rd = cmd.ExecuteReader(cmdtxt, RemoteVolumeType.Files.ToString(), hash, size, hash, size))
Duplicati/Library/Main/ForestHash/Database/LocalDatabase.cs
+5 -5
View File
@@ -210,7 +210,7 @@ namespace Duplicati.Library.Main.ForestHash.Database
var deletecmd = m_connection.CreateCommand();
deletecmd.Transaction = tr.Parent;
deletecmd.ExecuteNonQuery(@"UPDATE ""Fileset"" SET ""BlocksetID"" = -1 WHERE ""BlocksetID"" IN (SELECT DISTINCT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT DISTINCT ID FROM ""RemoteVolume"" WHERE ""Name"" = ?)))", name);
deletecmd.ExecuteNonQuery(@"UPDATE ""FileEntry"" SET ""BlocksetID"" = -1 WHERE ""BlocksetID"" IN (SELECT DISTINCT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT DISTINCT ID FROM ""RemoteVolume"" WHERE ""Name"" = ?)))", name);
deletecmd.ExecuteNonQuery(@"UPDATE ""Metadataset"" SET ""BlocksetID"" = -1 WHERE ""BlocksetID"" IN (SELECT DISTINCT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT DISTINCT ID FROM ""RemoteVolume"" WHERE ""Name"" = ?)))", name);
deletecmd.ExecuteNonQuery(@"DELETE FROM ""Blockset"" WHERE ""ID"" IN (SELECT DISTINCT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT DISTINCT ID FROM ""RemoteVolume"" WHERE ""Name"" = ?)))", name);
deletecmd.ExecuteNonQuery(@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN (SELECT DISTINCT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""ID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT DISTINCT ID FROM ""RemoteVolume"" WHERE ""Name"" = ?)))", name);
@@ -244,7 +244,7 @@ namespace Duplicati.Library.Main.ForestHash.Database
}
}
public long GetFilesetID(DateTime restoretime)
public long GetFilesetOperationID(DateTime restoretime)
{
if (restoretime.Kind == DateTimeKind.Unspecified)
throw new Exception("Invalid DateTime given, must be either local or UTC");
@@ -368,10 +368,10 @@ namespace Duplicati.Library.Main.ForestHash.Database
}
}
public IEnumerable<ILocalFileEntry> GetFiles(long filesetid)
public IEnumerable<ILocalFileEntry> GetFiles(long operationID)
{
using(var cmd = m_connection.CreateCommand())
using(var rd = cmd.ExecuteReader(@"SELECT ""A"".""Path"", ""B"".""Length"", ""B"".""FullHash"", ""D"".""FullHash"" FROM ""Fileset"" A, ""Blockset"" B, ""Metadataset"" C, ""Blockset"" D WHERE ""A"".""BlocksetID"" = ""B"".""ID"" AND ""A"".""MetadataID"" = ""C"".""ID"" AND ""C"".""BlocksetID"" = ""D"".""ID"" AND ""A"".""OperationID"" = ? ", filesetid))
using(var rd = cmd.ExecuteReader(@"SELECT ""A"".""Path"", ""B"".""Length"", ""B"".""FullHash"", ""D"".""FullHash"" FROM ""FileEntry"" A, ""Blockset"" B, ""Metadataset"" C, ""Blockset"" D WHERE ""A"".""BlocksetID"" = ""B"".""ID"" AND ""A"".""MetadataID"" = ""C"".""ID"" AND ""C"".""BlocksetID"" = ""D"".""ID"" AND ""A"".""OperationID"" = ? ", operationID))
while(rd.Read())
yield return new LocalFileEntry(rd);
}
@@ -421,7 +421,7 @@ namespace Duplicati.Library.Main.ForestHash.Database
cmd.ExecuteNonQuery(string.Format(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", filetable), s);
string first = null;
using(var rd = cmd.ExecuteReader(string.Format(@"SELECT ""{0}"".""Path"", ""RemoteVolume"".""Name"", ""Operation"".""Timestamp"" FROM ""{0}"", ""RemoteVolume"", ""Fileset"", ""OperationFileset"" WHERE ""OperationFileset"".""FilesetID"" = ""Fileset"".""ID"" AND ""OperationFileset"".""OperationID"" = ""RemoteVolume"".""OperationID"" AND ""Fileset"".""Path"" = ""{0}"".""Path"" AND ""RemoteVolume"".""Type"" = ? ORDER BY ""Operation"".""Timestamp"" DESC ", filetable), RemoteVolumeType.Files))
using(var rd = cmd.ExecuteReader(string.Format(@"SELECT ""{0}"".""Path"", ""RemoteVolume"".""Name"", ""Operation"".""Timestamp"" FROM ""{0}"", ""RemoteVolume"", ""FileEntry"", ""OperationFileset"" WHERE ""OperationFileset"".""FileEntryID"" = ""FileEntry"".""ID"" AND ""OperationFileset"".""OperationID"" = ""RemoteVolume"".""OperationID"" AND ""FileEntry"".""Path"" = ""{0}"".""Path"" AND ""RemoteVolume"".""Type"" = ? ORDER BY ""Operation"".""Timestamp"" DESC ", filetable), RemoteVolumeType.Files))
while(rd.Read())
{
var cur = rd.GetValue(0).ToString();
Duplicati/Library/Main/ForestHash/Database/LocalDeleteDatabase.cs
+15 -13
View File
@@ -72,6 +72,16 @@ namespace Duplicati.Library.Main.ForestHash.Database
return result;
}
private long GetLastBackupOperationID(System.Data.IDbCommand cmd)
{
long id = -1;
var r = cmd.ExecuteScalar(@"SELECT ""ID"" FROM ""Operation"" WHERE ""Description"" = ? ORDER BY ""Timestamp"" DESC LIMIT 1", "Backup");
if (r != null && r != DBNull.Value)
id = Convert.ToInt64(r);
return id;
}
/// <summary>
/// Deletes all but n backups from the database and remote storage.
@@ -86,11 +96,7 @@ namespace Duplicati.Library.Main.ForestHash.Database
cmd.Transaction = transaction;
long keepOperationId = -1;
if (!allowRemovingLast)
{
var r = cmd.ExecuteScalar(@"SELECT ""ID"" FROM ""Operation"" WHERE ""Description"" = ? ORDER BY ""Timestamp"" DESC LIMIT 1", "Backup");
if (r != null && r != DBNull.Value)
keepOperationId = Convert.ToInt64(keepOperationId);
}
keepOperationId = GetLastBackupOperationID(cmd);
//We create a table with the operationIDs that are about to be deleted
var tmptablename = "DeletedOperations-" + Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
@@ -117,11 +123,7 @@ namespace Duplicati.Library.Main.ForestHash.Database
cmd.Transaction = transaction;
long keepOperationId = -1;
if (!allowRemovingLast)
{
var r = cmd.ExecuteScalar(@"SELECT ""ID"" FROM ""Operation"" WHERE ""Description"" = ? ORDER BY ""Timestamp"" DESC LIMIT 1", "Backup");
if (r != null && r != DBNull.Value)
keepOperationId = Convert.ToInt64(keepOperationId);
}
keepOperationId = GetLastBackupOperationID(cmd);
var tmptablename = "DeletedOperations-" + Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""ID"" INTEGER NOT NULL)", tmptablename));
@@ -168,9 +170,9 @@ namespace Duplicati.Library.Main.ForestHash.Database
cmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""OperationFileset"" WHERE ""OperationID"" IN (SELECT ""ID"" FROM ""{0}"") ", tmptablename));
//Then we delete anything that is no longer being referenced
cmd.ExecuteNonQuery(@"DELETE FROM ""Fileset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FilesetID"" FROM ""OperationFileset"") ");
cmd.ExecuteNonQuery(@"DELETE FROM ""Metadataset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""MetadataID"" FROM ""Fileset"") ");
cmd.ExecuteNonQuery(@"DELETE FROM ""Blockset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""BlocksetID"" FROM ""Fileset"" UNION SELECT DISTINCT ""BlocksetID"" FROM ""Metadataset"") ");
cmd.ExecuteNonQuery(@"DELETE FROM ""FileEntry"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FileEntryID"" FROM ""OperationFileset"") ");
cmd.ExecuteNonQuery(@"DELETE FROM ""Metadataset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""MetadataID"" FROM ""FileEntry"") ");
cmd.ExecuteNonQuery(@"DELETE FROM ""Blockset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""BlocksetID"" FROM ""FileEntry"" UNION SELECT DISTINCT ""BlocksetID"" FROM ""Metadataset"") ");
cmd.ExecuteNonQuery(@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" NOT IN (SELECT DISTINCT ""ID"" FROM ""Blockset"") ");
//We save the block info for the remote files, before we delete it
Duplicati/Library/Main/ForestHash/Database/LocalRestoreDatabase.cs
+4 -4
View File
@@ -33,7 +33,7 @@ namespace Duplicati.Library.Main.ForestHash.Database
using (var cmd = m_connection.CreateCommand())
{
long operationID = GetFilesetID(restoretime);
long operationID = GetFilesetOperationID(restoretime);
cmd.CommandText = string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY, ""Path"" TEXT NOT NULL, ""BlocksetID"" INTEGER NOT NULL, ""MetadataID"" INTEGER NOT NULL, ""Targetpath"" TEXT NULL ) ", m_tempfiletable);
cmd.ExecuteNonQuery();
@@ -44,7 +44,7 @@ namespace Duplicati.Library.Main.ForestHash.Database
if (filenameFilter == null && (p == null || p.Length == 0))
{
// Simple case, restore everything
cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"") SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""Fileset"", ""OperationFileset"" WHERE ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" AND ""OperationFileset"".""OperationID"" = ? ", m_tempfiletable);
cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"") SELECT ""FileEntry"".""Path"", ""FileEntry"".""BlocksetID"", ""FileEntry"".""MetadataID"" FROM ""FileEntry"", ""OperationFileset"" WHERE ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" AND ""OperationFileset"".""OperationID"" = ? ", m_tempfiletable);
cmd.AddParameter(operationID);
cmd.ExecuteNonQuery();
}
@@ -65,7 +65,7 @@ namespace Duplicati.Library.Main.ForestHash.Database
}
//TODO: Handle case-insensitive filename lookup
cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"") SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""Fileset"", ""OperationFileset"" WHERE ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" AND ""OperationID"" = ? AND ""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{1}"") ", m_tempfiletable, m_filenamestable);
cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"") SELECT ""FileEntry"".""Path"", ""FileEntry"".""BlocksetID"", ""FileEntry"".""MetadataID"" FROM ""FileEntry"", ""OperationFileset"" WHERE ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" AND ""OperationID"" = ? AND ""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{1}"") ", m_tempfiletable, m_filenamestable);
cmd.SetParameterValue(0, operationID);
var c = cmd.ExecuteNonQuery();
@@ -93,7 +93,7 @@ namespace Duplicati.Library.Main.ForestHash.Database
else
{
// Restore but filter elements based on regexp
cmd.CommandText = string.Format(@"SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""Fileset"", ""OperationFileset"" WHERE ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" AND ""OperationID"" = ?");
cmd.CommandText = string.Format(@"SELECT ""FileEntry"".""Path"", ""FileEntry"".""BlocksetID"", ""FileEntry"".""MetadataID"" FROM ""FileEntry"", ""OperationFileset"" WHERE ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" AND ""OperationID"" = ?");
cmd.AddParameter(operationID);
object[] values = new object[3];
Duplicati/Library/Main/ForestHash/Database/RestoreIterators.cs
+1188 -1188
View File
File diff suppressed because it is too large Load Diff
Duplicati/Library/Main/ForestHash/Operation/ListFilesHandler.cs
+2 -2
View File
@@ -27,9 +27,9 @@ namespace Duplicati.Library.Main.ForestHash.Operation
if (System.IO.File.Exists(m_options.Fhdbpath))
using (var db = new Database.LocalDatabase(m_options.Fhdbpath, "ListFiles"))
{
var filesetid = db.GetFilesetID(m_options.RestoreTime);
var operationId = db.GetFilesetOperationID(m_options.RestoreTime);
return
(from n in db.GetFiles(filesetid)
(from n in db.GetFiles(operationId)
select n.Path).ToList();
}
Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs
+13 -6
View File
@@ -73,16 +73,23 @@ namespace Duplicati.Library.Main.ForestHash.Operation
using (var backupdb = new LocalBackupDatabase(restoredb, m_options))
{
foreach (var fl in remotefiles)
backupdb.RegisterRemoteVolume(fl.File.Name, fl.FileType, RemoteVolumeState.Uploaded);
var shadowfiles =
from n in remotefiles
where n.FileType == RemoteVolumeType.Shadow
select new RemoteVolume(n.File) as IRemoteVolume;
{
var id = backupdb.RegisterRemoteVolume(fl.File.Name, fl.FileType, RemoteVolumeState.Uploaded);
//For filelists, we associate them with a backup operation.
// This enables the delete-older-than and similar operations to work with the restored database
if (fl.FileType == RemoteVolumeType.Files)
restoredb.CreateBackupOperation(id, fl.Time);
}
//We grab all shadow files, and update the block table
using (var tr = restoredb.BeginTransaction())
{
var shadowfiles =
from n in remotefiles
where n.FileType == RemoteVolumeType.Shadow
select new RemoteVolume(n.File) as IRemoteVolume;
foreach (var sf in new AsyncDownloader(shadowfiles.ToList(), backend))
{
if (sf.Key.Hash != null && sf.Key.Size > 0)