diff --git a/Duplicati/Library/Main/Database/Database schema/Schema.sql b/Duplicati/Library/Main/Database/Database schema/Schema.sql index bf8851431..a292704fe 100644 --- a/Duplicati/Library/Main/Database/Database schema/Schema.sql +++ b/Duplicati/Library/Main/Database/Database schema/Schema.sql @@ -138,6 +138,15 @@ CREATE TABLE "DeletedBlock" ( "VolumeID" INTEGER NOT NULL ); +/* +If extra copies of blocks are detected, +they are recorded here +*/ +CREATE TABLE "DuplicateBlock" ( + "BlockID" INTEGER NOT NULL, + "VolumeID" INTEGER NOT NULL +); + /* A metadata set, essentially a placeholder diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 19ebd664a..bc9a392c7 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -12,7 +12,7 @@ namespace Duplicati.Library.Main.Database /// /// An approximate size of a hash-string in memory (44 chars * 2 for unicode + 8 bytes for pointer = 104) /// - private const uint HASH_GUESS_SIZE = 128; + internal const uint HASH_GUESS_SIZE = 128; /// /// An approximate size of a path string in bytes. @@ -23,7 +23,7 @@ namespace Duplicati.Library.Main.Database /// in slightly more memory being used that what the user /// specifies. /// - private const uint PATH_STRING_GUESS_SIZE = 256; + internal const uint PATH_STRING_GUESS_SIZE = 256; /// /// The usage threshold for a lookup table that triggers a warning @@ -58,7 +58,6 @@ namespace Duplicati.Library.Main.Database private readonly System.Data.IDbCommand m_updateblockCommand; private readonly System.Data.IDbCommand m_insertfileOperationCommand; - private readonly System.Data.IDbCommand m_insertIndexBlockLink; private HashDatabaseProtector m_blockHashLookup; private HashDatabaseProtector m_fileHashLookup; @@ -96,7 +95,6 @@ namespace Duplicati.Library.Main.Database m_selectfileSimpleCommand = m_connection.CreateCommand(); m_selectfileHashCommand = m_connection.CreateCommand(); m_findmetadatasetProbeCommand = m_connection.CreateCommand(); - m_insertIndexBlockLink = m_connection.CreateCommand(); m_findblockCommand.CommandText = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?"; m_findblockCommand.AddParameters(2); @@ -153,10 +151,7 @@ namespace Duplicati.Library.Main.Database m_updateblockCommand.CommandText = @"UPDATE ""Block"" SET ""VolumeID"" = ? WHERE ""Hash"" = ? AND ""Size"" = ? "; m_updateblockCommand.AddParameters(3); - - m_insertIndexBlockLink.CommandText = @"INSERT INTO ""IndexBlockLink"" (""IndexVolumeID"", ""BlockVolumeID"") VALUES (?, ?)"; - m_insertIndexBlockLink.AddParameters(2); - + if (options.BlockHashLookupMemory > 0) m_blockHashLookup = new HashDatabaseProtector(HASH_GUESS_SIZE, (ulong)options.BlockHashLookupMemory); if (options.FileHashLookupMemory > 0) @@ -812,33 +807,18 @@ namespace Duplicati.Library.Main.Database tr.Commit(); } } - - /// - /// Creates a timestamped backup operation to correctly associate the fileset with the time it was created. - /// - /// The ID of the fileset volume to update - /// The timestamp of the operation to create - /// An optional external transaction - public long CreateFileset(long volumeid, DateTime timestamp, System.Data.IDbTransaction transaction = null) - { - using (var cmd = m_connection.CreateCommand()) - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - { - cmd.Transaction = tr.Parent; - m_filesetId = Convert.ToInt64(cmd.ExecuteScalar(@"INSERT INTO ""Fileset"" (""OperationID"", ""Timestamp"", ""VolumeID"") VALUES (?, ?, ?); SELECT last_insert_rowid();", m_operationid, NormalizeDateTime(timestamp), volumeid)); - tr.Commit(); - return m_filesetId; - } - } - - public void AddIndexBlockLink(long indexVolumeID, long blockVolumeID, System.Data.IDbTransaction transaction) - { - m_insertIndexBlockLink.Transaction = transaction; - m_insertIndexBlockLink.SetParameterValue(0, indexVolumeID); - m_insertIndexBlockLink.SetParameterValue(1, blockVolumeID); - m_insertIndexBlockLink.ExecuteNonQuery(); - } - + + /// + /// Creates a timestamped backup operation to correctly associate the fileset with the time it was created. + /// + /// The ID of the fileset volume to update + /// The timestamp of the operation to create + /// An optional external transaction + public override long CreateFileset(long volumeid, DateTime timestamp, System.Data.IDbTransaction transaction = null) + { + return m_filesetId = base.CreateFileset(volumeid, timestamp, transaction); + } + public IEnumerable> GetIncompleteFilesets(System.Data.IDbTransaction transaction) { using(var cmd = m_connection.CreateCommand()) diff --git a/Duplicati/Library/Main/Database/LocalBlocklistUpdateDatabase.cs b/Duplicati/Library/Main/Database/LocalBlocklistUpdateDatabase.cs deleted file mode 100644 index 2e9601ac6..000000000 --- a/Duplicati/Library/Main/Database/LocalBlocklistUpdateDatabase.cs +++ /dev/null @@ -1,116 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; - -namespace Duplicati.Library.Main.Database -{ - internal partial class LocalBlocklistUpdateDatabase : LocalRestoreDatabase - { - protected string m_tempblockvolumetable; - protected string m_temphashtable; - - public LocalBlocklistUpdateDatabase(string path, long blocksize) - : base(new LocalDatabase(path, "Rebuild"), blocksize) - { - } - - public LocalBlocklistUpdateDatabase(LocalDatabase parentdb, long blocksize) - : base(parentdb, blocksize) - { - } - - public void FindMissingBlocklistHashes() - { - m_tempblockvolumetable = "MissingBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - m_temphashtable = "Hashlist-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using (var cmd = m_connection.CreateCommand()) - { - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT DISTINCT ""Block"".""VolumeID"" AS ""VolumeID"", ""BlocklistHash"".""Hash"" AS ""Hash"", 0 AS ""Restored"" FROM ""Block"", ""BlocklistHash"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" ", m_tempblockvolumetable)); - cmd.ExecuteNonQuery(string.Format(@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") SELECT DISTINCT ""Blockset"".""ID"", 0, ""Block"".""ID"" FROM ""Blockset"", ""Block"" WHERE ""Blockset"".""Fullhash"" = ""Block"".""Hash"" AND ""Blockset"".""Length"" < {0} AND ""Blockset"".""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"") ", m_blocksize)); - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Hash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)", m_temphashtable)); - - } - } - - public IEnumerable GetBlockLists(long volumeid) - { - return new BlocklistsEnumerable(m_connection, m_tempblockvolumetable, volumeid); - } - - public void UpdateBlocklist(string hash, IEnumerable hashes, long hashsize, System.Data.IDbTransaction transaction) - { - using (var cmd = m_connection.CreateCommand()) - { - cmd.Transaction = transaction; - - cmd.CommandText = string.Format(@"UPDATE ""{0}"" SET ""Restored"" = 1 WHERE ""Restored"" = 0 AND ""Hash"" = ? ", m_tempblockvolumetable); - cmd.AddParameter(hash); - var c = cmd.ExecuteNonQuery(); - if (c != 1) - throw new Exception(string.Format("Blocklist not found {0}", hash)); - - cmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""{0}""", m_temphashtable)); - cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Hash"", ""Index"") VALUES (?, ?)", m_temphashtable); - long ix = 0; - foreach (var h in hashes) - { - c = cmd.ExecuteNonQuery(null, h, ix); - if (c != 1) - throw new Exception(string.Format("Insert row error {0}", h)); - ix++; - } - - cmd.CommandText = string.Format(@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") SELECT ""A"".""BlocksetID"", (""A"".""Index"" * ?) + ""B"".""Index"", ""C"".""ID"" FROM ""BlocklistHash"" A, ""{0}"" B, ""Block"" C WHERE ""A"".""Hash"" = ? AND ""C"".""Hash"" = ""B"".""Hash"" ", m_temphashtable); - c = cmd.ExecuteNonQuery(null, (m_blocksize / hashsize), hash); - if (c == 0 || c % ix != 0) - throw new Exception(string.Format("Wrong number of inserts, got {0} records from {1} hashes!", c, ix)); - } - } - - public IEnumerable GetMissingBlockListVolumes() - { - using (var cmd = m_connection.CreateCommand()) - { - cmd.CommandText = string.Format(@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT DISTINCT ""VolumeID"" FROM ""{0}"")", m_tempblockvolumetable); - using (var rd = cmd.ExecuteReader()) - { - object[] r = new object[3]; - while (rd.Read()) - { - rd.GetValues(r); - yield return new RemoteVolume( - r[0] == null ? null : r[0].ToString(), - r[1] == null ? null : r[1].ToString(), - r[2] == null ? -1 : Convert.ToInt64(r[2]) - ); - } - } - } - } - - public override void Dispose() - { - base.Dispose(); - using (var cmd = m_connection.CreateCommand()) - { - if (m_tempblockvolumetable != null) - try - { - cmd.CommandText = string.Format(@"DROP TABLE ""{0}""", m_tempblockvolumetable); - cmd.ExecuteNonQuery(); - } - finally { m_tempblockvolumetable = null; } - - if (m_temphashtable != null) - try - { - cmd.CommandText = string.Format(@"DROP TABLE ""{0}""", m_temphashtable); - cmd.ExecuteNonQuery(); - } - finally { m_temphashtable = null; } - } - } - } -} diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index c78081a32..44fadc840 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -20,6 +20,7 @@ namespace Duplicati.Library.Main.Database private readonly System.Data.IDbCommand m_insertlogCommand; private readonly System.Data.IDbCommand m_insertremotelogCommand; + private readonly System.Data.IDbCommand m_insertIndexBlockLink; private BasicResults m_result; @@ -95,6 +96,7 @@ namespace Duplicati.Library.Main.Database m_removeremotevolumeCommand = connection.CreateCommand(); m_selectremotevolumeIdCommand = connection.CreateCommand(); m_createremotevolumeCommand = connection.CreateCommand(); + m_insertIndexBlockLink = connection.CreateCommand(); m_insertlogCommand.CommandText = @"INSERT INTO ""LogData"" (""OperationID"", ""Timestamp"", ""Type"", ""Message"", ""Exception"") VALUES (?, ?, ?, ?, ?)"; m_insertlogCommand.AddParameters(5); @@ -116,7 +118,10 @@ namespace Duplicati.Library.Main.Database m_selectremotevolumeIdCommand.CommandText = @"SELECT ""ID"" FROM ""Remotevolume"" WHERE ""Name"" = ?"; m_createremotevolumeCommand.CommandText = @"INSERT INTO ""Remotevolume"" (""OperationID"", ""Name"", ""Type"", ""State"") VALUES (?, ?, ?, ?); SELECT last_insert_rowid();"; - m_createremotevolumeCommand.AddParameters(4); + m_createremotevolumeCommand.AddParameters(4); + + m_insertIndexBlockLink.CommandText = @"INSERT INTO ""IndexBlockLink"" (""IndexVolumeID"", ""BlockVolumeID"") VALUES (?, ?)"; + m_insertIndexBlockLink.AddParameters(2); } internal void SetResult(BasicResults result) @@ -798,6 +803,32 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Creates a timestamped backup operation to correctly associate the fileset with the time it was created. + /// + /// The ID of the fileset volume to update + /// The timestamp of the operation to create + /// An optional external transaction + public virtual long CreateFileset(long volumeid, DateTime timestamp, System.Data.IDbTransaction transaction = null) + { + using (var cmd = m_connection.CreateCommand()) + using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) + { + cmd.Transaction = tr.Parent; + var id = Convert.ToInt64(cmd.ExecuteScalar(@"INSERT INTO ""Fileset"" (""OperationID"", ""Timestamp"", ""VolumeID"") VALUES (?, ?, ?); SELECT last_insert_rowid();", m_operationid, NormalizeDateTime(timestamp), volumeid)); + tr.Commit(); + return id; + } + } + + public void AddIndexBlockLink(long indexVolumeID, long blockVolumeID, System.Data.IDbTransaction transaction) + { + m_insertIndexBlockLink.Transaction = transaction; + m_insertIndexBlockLink.SetParameterValue(0, indexVolumeID); + m_insertIndexBlockLink.SetParameterValue(1, blockVolumeID); + m_insertIndexBlockLink.ExecuteNonQuery(); + } + public virtual void Dispose() { if (IsDisposed) diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs new file mode 100644 index 000000000..d07e59476 --- /dev/null +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -0,0 +1,688 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace Duplicati.Library.Main.Database +{ + internal partial class LocalRecreateDatabase : LocalRestoreDatabase + { + private System.Data.IDbCommand m_insertFileCommand; + private System.Data.IDbCommand m_insertFilesetEntryCommand; + private System.Data.IDbCommand m_insertMetadatasetCommand; + private System.Data.IDbCommand m_insertBlocksetCommand; + private System.Data.IDbCommand m_insertBlocklistHashCommand; + private System.Data.IDbCommand m_updateBlockVolumeCommand; + private System.Data.IDbCommand m_insertBlockset; + private System.Data.IDbCommand m_findBlocksetCommand; + private System.Data.IDbCommand m_findMetadatasetCommand; + private System.Data.IDbCommand m_findFilesetCommand; + private System.Data.IDbCommand m_findblocklisthashCommand; + private System.Data.IDbCommand m_findHashBlockCommand; + private System.Data.IDbCommand m_insertBlockCommand; + private System.Data.IDbCommand m_insertDuplicateBlockCommand; + + private HashDatabaseProtector m_blockListHashLookup; + private HashDatabaseProtector m_blockHashLookup; + private HashDatabaseProtector m_fileHashLookup; + private HashDatabaseProtector m_metadataLookup; + private HashDatabaseProtector, long> m_filesetLookup; + + private string m_tempblocklist; + + /// + /// A lookup table that prevents multiple downloads of the same volume + /// + private Dictionary m_proccessedVolumes; + + // SQL that finds index and block size for all blocklist hashes, based on the temporary hash list + private const string SELECT_BLOCKLIST_ENTRIES = + @"SELECT ""B"".""Length"", ""A"".""BlocklistHash"", ""A"".""BlockHash"", ""C"".""Index"", ""A"".""Index"" AS ""BlocIndex"", ""B"".""Length""/{0} AS ""LastBlock"", ""B"".""Length"" - (""B"".""Length""/{0})*{0} AS ""LastBlockSize"", " + + @" CASE WHEN ""A"".""Index"" + (""C"".""Index"" * ({0}/{1})) = ""B"".""Length""/{0} THEN ""B"".""Length"" - (""B"".""Length""/{0})*{0} ELSE {0} END AS ""BlockSize"", " + + @" ""A"".""Index"" + (""C"".""Index"" * ({0}/{1})) AS ""FullIndex"" " + + @" FROM ""Blockset"" B, ""BlocklistHash"" C, ""{2}"" A " + + @" WHERE ""B"".""ID"" = ""C"".""BlocksetID"" AND ""A"".""BlockListHash"" = ""C"".""Hash"" " + + @" ORDER BY ""A"".""BlockListHash"", ""A"".""Index"" "; + + public LocalRecreateDatabase(LocalDatabase parentdb, Options options) + : base(parentdb, options.Blocksize) + { + m_tempblocklist = "TempBlocklist-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + + using (var cmd = m_connection.CreateCommand()) + cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)", m_tempblocklist)); + + m_insertFileCommand = m_connection.CreateCommand(); + m_insertFilesetEntryCommand = m_connection.CreateCommand(); + m_insertMetadatasetCommand = m_connection.CreateCommand(); + m_insertBlocksetCommand = m_connection.CreateCommand(); + m_insertBlocklistHashCommand = m_connection.CreateCommand(); + m_updateBlockVolumeCommand = m_connection.CreateCommand(); + m_insertBlockset = m_connection.CreateCommand(); + m_findBlocksetCommand = m_connection.CreateCommand(); + m_findMetadatasetCommand = m_connection.CreateCommand(); + m_findFilesetCommand = m_connection.CreateCommand(); + m_findblocklisthashCommand = m_connection.CreateCommand(); + m_findHashBlockCommand = m_connection.CreateCommand(); + m_insertBlockCommand = m_connection.CreateCommand(); + m_insertDuplicateBlockCommand = m_connection.CreateCommand(); + + m_insertFileCommand.CommandText = @"INSERT INTO ""File"" (""Path"", ""BlocksetID"", ""MetadataID"") VALUES (?,?,?); SELECT last_insert_rowid();"; + m_insertFileCommand.AddParameters(3); + + m_insertFilesetEntryCommand.CommandText = @"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Scantime"") VALUES (?,?,?)"; + m_insertFilesetEntryCommand.AddParameters(3); + + m_insertMetadatasetCommand.CommandText = @"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (?); SELECT last_insert_rowid();"; + m_insertMetadatasetCommand.AddParameters(1); + + m_insertBlocksetCommand.CommandText = @"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (?,?); SELECT last_insert_rowid();"; + m_insertBlocksetCommand.AddParameters(2); + + m_insertBlocklistHashCommand.CommandText = @"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (?,?,?)"; + m_insertBlocklistHashCommand.AddParameters(3); + + m_updateBlockVolumeCommand.CommandText = @"UPDATE ""Block"" SET ""VolumeID"" = ? WHERE ""Hash"" = ? AND ""Size"" = ?"; + m_updateBlockVolumeCommand.AddParameters(3); + + m_insertBlockset.CommandText = string.Format(@"INSERT INTO ""{0}"" (""BlocklistHash"", ""BlockHash"", ""Index"") VALUES (?,?,?) ", m_tempblocklist); + m_insertBlockset.AddParameters(3); + + m_findBlocksetCommand.CommandText = @"SELECT ""ID"" FROM ""Blockset"" WHERE ""Size"" = ? AND ""FullHash"" = ? "; + m_findBlocksetCommand.AddParameters(2); + + m_findMetadatasetCommand.CommandText = @"SELECT ""Metadataset"".""ID"" FROM ""Metadataset"",""BlocksetEntry"",""Block"" WHERE ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""Block"".""Hash"" = ? AND ""Block"".""Size"" = ? "; + m_findMetadatasetCommand.AddParameters(2); + + m_findFilesetCommand.CommandText = @"SELECT ""ID"" FROM ""File"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? "; + m_findFilesetCommand.AddParameters(3); + + m_findblocklisthashCommand.CommandText = string.Format(@"SELECT DISTINCT ""BlockListHash"" FROM ""{0}"" WHERE ""BlockListHash"" = ? ", m_tempblocklist); + m_findblocklisthashCommand.AddParameters(1); + + m_findHashBlockCommand.CommandText = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ? "; + m_findHashBlockCommand.AddParameters(2); + + m_insertBlockCommand.CommandText = @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") VALUES (?,?,?)"; + m_insertBlockCommand.AddParameters(3); + + m_insertDuplicateBlockCommand.CommandText = @"INSERT INTO ""DuplicateBlock"" (""Hash"", ""Size"", ""VolumeID"") VALUE (?,?,?)"; + m_insertDuplicateBlockCommand.AddParameters(3); + + if (options.BlockHashLookupMemory > 0) + { + m_blockHashLookup = new HashDatabaseProtector(LocalBackupDatabase.HASH_GUESS_SIZE, (ulong)options.BlockHashLookupMemory/2); + m_blockListHashLookup = new HashDatabaseProtector(LocalBackupDatabase.HASH_GUESS_SIZE, (ulong)options.BlockHashLookupMemory/2); + } + if (options.FileHashLookupMemory > 0) + m_fileHashLookup = new HashDatabaseProtector(LocalBackupDatabase.HASH_GUESS_SIZE, (ulong)options.FileHashLookupMemory); + if (options.MetadataHashMemory > 0) + m_metadataLookup = new HashDatabaseProtector(LocalBackupDatabase.HASH_GUESS_SIZE, (ulong)options.MetadataHashMemory); + if (options.FilePathMemory > 0) + m_filesetLookup = new HashDatabaseProtector, long>(LocalBackupDatabase.PATH_STRING_GUESS_SIZE, (ulong)options.FilePathMemory); + } + + public void FindMissingBlocklistHashes(long hashsize, System.Data.IDbTransaction transaction) + { + using(var cmd = m_connection.CreateCommand()) + { + cmd.Transaction = transaction; + + //Update all small blocklists and matching blocks + var selectSmallBlocks = string.Format(@"SELECT ""Blockset"".""Fullhash"", ""Blockset"".""Length"" FROM ""Blockset"" WHERE ""Blockset"".""Length"" <= {0}", m_blocksize); + + var selectBlockHashes = string.Format( + @"SELECT ""BlockHash"" AS ""FullHash"", ""BlockSize"" AS ""Length"" FROM ( " + + SELECT_BLOCKLIST_ENTRIES + + @" )", + m_blocksize, + hashsize, + m_tempblocklist + ); + + var selectAllBlocks = @"SELECT DISTINCT ""FullHash"", ""Length"" FROM (" + selectBlockHashes + " UNION " + selectSmallBlocks + " )"; + + var selectNewBlocks = string.Format( + @"SELECT ""FullHash"" AS ""Hash"", ""Length"" AS ""Size"", -1 AS ""VolumeID"" " + + @" FROM (SELECT ""A"".""FullHash"", ""A"".""Length"", CASE WHEN ""B"".""Hash"" IS NULL THEN '' ELSE ""B"".""Hash"" END AS ""Hash"", CASE WHEN ""B"".""Size"" is NULL THEN -1 ELSE ""B"".""Size"" END AS ""Size"" FROM ({0}) A" + + @" LEFT OUTER JOIN ""Block"" B ON (""B"".""Hash"" || ':' || ""B"".""Size"") = (""A"".""FullHash"" || ':' || ""A"".""Length"") )" + + @" WHERE ""FullHash"" != ""Hash"" AND ""Length"" != ""Size"" ", + selectAllBlocks + ); + + var insertBlocksCommand = + @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") " + + selectNewBlocks; + + // Insert all known blocks into block table with volumeid = -1 + cmd.ExecuteNonQuery(insertBlocksCommand); + + // Update the cache with new wblocks + if (m_blockHashLookup != null) + { + using(var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Hash"" FROM ""Block"" WHERE ""VolumeID"" = -1 ")) + while(rd.Read()) + { + var hash = rd.GetValue(0).ToString(); + m_blockHashLookup.Add(HashPrefixLookup.DecodeBase64Hash(hash), hash, -1); + } + } + + + var selectBlocklistBlocksetEntries = string.Format( + @"SELECT ""E"".""BlocksetID"" AS ""BlocksetID"", ""D"".""FullIndex"" AS ""Index"", ""F"".""ID"" AS ""BlockID"" FROM ( " + + SELECT_BLOCKLIST_ENTRIES + + @") D, ""BlocklistHash"" E, ""Block"" F WHERE ""D"".""BlocklistHash"" = ""E"".""Hash"" AND ""D"".""Blockhash"" = ""F"".""Hash"" AND ""D"".""BlockSize"" = ""F"".""Size"" ", + m_blocksize, + hashsize, + m_tempblocklist + ); + + var selectBlocksetEntries = string.Format( + @"SELECT ""Blockset"".""ID"" AS ""BlocksetID"", 0 AS ""Index"", ""Block"".""ID"" AS ""BlockID"" FROM ""Blockset"", ""Block"" WHERE ""Blockset"".""Fullhash"" = ""Block"".""Hash"" AND ""Blockset"".""Length"" <= {0} ", + m_blocksize + ); + + var selectAllBlocksetEntries = + selectBlocklistBlocksetEntries + + @" UNION " + + selectBlocksetEntries; + + var selectFiltered = + @"SELECT DISTINCT ""BlocksetID"", ""Index"", ""BlockID"" FROM (" + + selectAllBlocksetEntries + + @") A WHERE (""A"".""BlocksetID"" || ':' || ""A"".""Index"") NOT IN (SELECT (""BlocksetID"" || ':' || ""Index"") FROM ""BlocksetEntry"" )"; + + var insertBlocksetEntriesCommand = + @"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") " + selectFiltered; + + cmd.ExecuteNonQuery(insertBlocksetEntriesCommand); + } + } + + public void AddDirectoryEntry(long filesetid, string path, DateTime time, string metahash, long metahashsize, System.Data.IDbTransaction transaction) + { + AddEntry(FilelistEntryType.Folder, filesetid, path, time, FOLDER_BLOCKSET_ID, metahash, metahashsize, transaction); + } + + public void AddSymlinkEntry(long filesetid, string path, DateTime time, string metahash, long metahashsize, System.Data.IDbTransaction transaction) + { + AddEntry(FilelistEntryType.Symlink, filesetid, path, time, SYMLINK_BLOCKSET_ID, metahash, metahashsize, transaction); + } + + public void AddFileEntry(long filesetid, string path, DateTime time, long blocksetid, string metahash, long metahashsize, System.Data.IDbTransaction transaction) + { + AddEntry(FilelistEntryType.File , filesetid, path, time, blocksetid, metahash, metahashsize, transaction); + } + + private void AddEntry(FilelistEntryType type, long filesetid, string path, DateTime time, long blocksetid, string metahash, long metahashsize, System.Data.IDbTransaction transaction) + { + var fileid = -1L; + var metadataid = AddMetadataset(metahash, metahashsize, transaction); + + var hashdata = (ulong)blocksetid ^ (ulong)metadataid ^ (ulong)path.GetHashCode(); + var tp = new Tuple(path, blocksetid, metadataid); + + if (m_filesetLookup != null) + { + switch (m_filesetLookup.HasValue(hashdata, tp, out fileid)) + { + case HashLookupResult.Found: + break; + case HashLookupResult.NotFound: + fileid = -1; + break; + case HashLookupResult.Uncertain: + m_findFilesetCommand.Transaction = transaction; + m_findFilesetCommand.SetParameterValue(0, path); + m_findFilesetCommand.SetParameterValue(1, blocksetid); + m_findFilesetCommand.SetParameterValue(2, metadataid); + var r = m_findFilesetCommand.ExecuteScalar(); + if (r == null || r == DBNull.Value) + { + m_filesetLookup.PositiveMisses++; + fileid = -1; + } + else + { + fileid = Convert.ToInt64(r); + m_filesetLookup.NegativeMisses++; + m_filesetLookup.Add(hashdata, tp, fileid); + } + break; + } + } + else + { + m_findFilesetCommand.Transaction = transaction; + m_findFilesetCommand.SetParameterValue(0, path); + m_findFilesetCommand.SetParameterValue(1, blocksetid); + m_findFilesetCommand.SetParameterValue(2, metadataid); + var r = m_findFilesetCommand.ExecuteScalar(); + if (r != null && r != DBNull.Value) + fileid = Convert.ToInt64(r); + } + + if (fileid < 0) + { + m_insertFileCommand.Transaction = transaction; + m_insertFileCommand.SetParameterValue(0, path); + m_insertFileCommand.SetParameterValue(1, blocksetid); + m_insertFileCommand.SetParameterValue(2, metadataid); + fileid = Convert.ToInt64(m_insertFileCommand.ExecuteScalar()); + if (m_filesetLookup != null) + m_filesetLookup.Add(hashdata, tp, fileid); + } + + m_insertFilesetEntryCommand.Transaction = transaction; + m_insertFilesetEntryCommand.SetParameterValue(0, filesetid); + m_insertFilesetEntryCommand.SetParameterValue(1, fileid); + m_insertFilesetEntryCommand.SetParameterValue(2, time); + m_insertFilesetEntryCommand.ExecuteNonQuery(); + } + + private long AddMetadataset(string metahash, long metahashsize, System.Data.IDbTransaction transaction) + { + var metadataid = -1L; + if (metahash == null) + return metadataid; + + var hashdata = HashPrefixLookup.DecodeBase64Hash(metahash); + if (m_metadataLookup != null) + { + switch (m_metadataLookup.HasValue(hashdata, metahash, out metadataid)) + { + case HashLookupResult.Found: + return metadataid; + case HashLookupResult.NotFound: + metadataid = -1; + break; + case HashLookupResult.Uncertain: + m_findMetadatasetCommand.Transaction = transaction; + m_findMetadatasetCommand.SetParameterValue(0, metahash); + m_findMetadatasetCommand.SetParameterValue(1, metahashsize); + var r = m_findMetadatasetCommand.ExecuteScalar(); + if (r == null || r == DBNull.Value) + { + metadataid = -1; + m_metadataLookup.PositiveMisses++; + } + else + { + metadataid = Convert.ToInt64(r); + m_metadataLookup.NegativeMisses++; + m_metadataLookup.Add(hashdata, metahash, metadataid); + return metadataid; + } + + break; + } + } + else + { + m_findMetadatasetCommand.Transaction = transaction; + m_findMetadatasetCommand.SetParameterValue(0, metahash); + m_findMetadatasetCommand.SetParameterValue(1, metahashsize); + var r = m_findMetadatasetCommand.ExecuteScalar(); + if (r != null && r != DBNull.Value) + return Convert.ToInt64(r); + } + + var blocksetid = AddBlockset(metahash, metahashsize, null, transaction); + + m_insertMetadatasetCommand.Transaction = transaction; + m_insertMetadatasetCommand.SetParameterValue(0, blocksetid); + metadataid = Convert.ToInt64(m_insertMetadatasetCommand.ExecuteScalar()); + + if (m_metadataLookup != null) + m_metadataLookup.Add(hashdata, metahash, metadataid); + + return metadataid; + } + + public long AddBlockset(string fullhash, long size, IEnumerable blocklisthashes, System.Data.IDbTransaction transaction) + { + var blocksetid = -1L; + var hashdata = HashPrefixLookup.DecodeBase64Hash(fullhash); + if (m_fileHashLookup != null) + { + switch (m_fileHashLookup.HasValue(hashdata, fullhash, out blocksetid)) + { + case HashLookupResult.Found: + return blocksetid; + case HashLookupResult.NotFound: + blocksetid = -1; + break; + case HashLookupResult.Uncertain: + m_findBlocksetCommand.Transaction = transaction; + m_findBlocksetCommand.SetParameterValue(0, size); + m_findBlocksetCommand.SetParameterValue(1, fullhash); + var r = m_findBlocksetCommand.ExecuteScalar(); + if (r == null || r == DBNull.Value) + { + m_fileHashLookup.PositiveMisses++; + blocksetid = -1; + } + else + { + blocksetid = Convert.ToInt64(r); + m_fileHashLookup.NegativeMisses++; + m_fileHashLookup.Add(hashdata, fullhash, blocksetid); + return blocksetid; + } + break; + } + } + else + { + m_findBlocksetCommand.Transaction = transaction; + m_findBlocksetCommand.SetParameterValue(0, size); + m_findBlocksetCommand.SetParameterValue(1, fullhash); + var r = m_findBlocksetCommand.ExecuteScalar(); + if (r != null && r != DBNull.Value) + return Convert.ToInt64(r); + } + + m_insertBlocksetCommand.Transaction = transaction; + m_insertBlocksetCommand.SetParameterValue(0, size); + m_insertBlocksetCommand.SetParameterValue(1, fullhash); + blocksetid = Convert.ToInt64(m_insertBlocksetCommand.ExecuteScalar()); + + if (m_fileHashLookup != null) + m_fileHashLookup.Add(hashdata, fullhash, blocksetid); + + if (blocklisthashes != null) + { + var index = 0L; + m_insertBlocklistHashCommand.Transaction = transaction; + m_insertBlocklistHashCommand.SetParameterValue(0, blocksetid); + foreach(var hash in blocklisthashes) + { + if (!string.IsNullOrEmpty(hash)) + { + m_insertBlocklistHashCommand.SetParameterValue(1, index++); + m_insertBlocklistHashCommand.SetParameterValue(2, hash); + m_insertBlocklistHashCommand.ExecuteNonQuery(); + } + } + } + + return blocksetid; + } + + public void UpdateBlock(string hash, long size, long volumeID, System.Data.IDbTransaction transaction) + { + var hashdata = HashPrefixLookup.DecodeBase64Hash(hash); + var currentVolumeId = -2L; + if (m_blockHashLookup != null) + { + switch (m_blockHashLookup.HasValue(hashdata, hash, out currentVolumeId)) + { + case HashLookupResult.Found: + break; + case HashLookupResult.NotFound: + currentVolumeId = -2; + break; + case HashLookupResult.Uncertain: + m_findHashBlockCommand.Transaction = transaction; + m_findHashBlockCommand.SetParameterValue(0, hash); + m_findHashBlockCommand.SetParameterValue(1, size); + var r = m_findHashBlockCommand.ExecuteScalar(); + if (r == null || r == DBNull.Value) + { + m_blockHashLookup.PositiveMisses++; + currentVolumeId = -2; + } + else + { + m_blockHashLookup.NegativeMisses++; + currentVolumeId = Convert.ToInt64(r); + } + break; + } + } + else + { + m_findHashBlockCommand.Transaction = transaction; + m_findHashBlockCommand.SetParameterValue(0, hash); + m_findHashBlockCommand.SetParameterValue(1, size); + var r = m_findHashBlockCommand.ExecuteScalar(); + if (r != null && r != DBNull.Value) + currentVolumeId = Convert.ToInt64(r); + } + + if (currentVolumeId == volumeID) + return; + + if (currentVolumeId == -2) + { + //Insert + m_insertBlockCommand.Transaction = transaction; + m_insertBlockCommand.SetParameterValue(0, hash); + m_insertBlockCommand.SetParameterValue(1, size); + m_insertBlockCommand.SetParameterValue(2, volumeID); + m_insertBlockCommand.ExecuteNonQuery(); + + if (m_blockHashLookup != null) + m_blockHashLookup.Add(hashdata, hash, volumeID); + } + else if (currentVolumeId == -1) + { + //Update + m_updateBlockVolumeCommand.Transaction = transaction; + m_updateBlockVolumeCommand.SetParameterValue(0, volumeID); + m_updateBlockVolumeCommand.SetParameterValue(1, hash); + m_updateBlockVolumeCommand.SetParameterValue(2, size); + var c = m_updateBlockVolumeCommand.ExecuteNonQuery(); + if (c != 1) + throw new Exception(string.Format("Failed to update table, found {0} entries for key {1} with size {2}", c ,hash, size)); + + if (m_blockHashLookup != null) + m_blockHashLookup.Add(hashdata, hash, volumeID); + } + else + { + m_insertDuplicateBlockCommand.Transaction = transaction; + m_insertDuplicateBlockCommand.SetParameterValue(0, hash); + m_insertDuplicateBlockCommand.SetParameterValue(1, size); + m_insertDuplicateBlockCommand.SetParameterValue(2, volumeID); + m_insertDuplicateBlockCommand.ExecuteNonQuery(); + } + + } + + public void UpdateBlockset(string hash, IEnumerable blocklisthashes, long hashsize, System.Data.IDbTransaction transaction) + { + var hashdata = HashPrefixLookup.DecodeBase64Hash(hash); + if (m_blockListHashLookup != null) + { + switch (m_blockListHashLookup.HasValue(hashdata, hash)) + { + case HashLookupResult.Found: + return; + case HashLookupResult.NotFound: + break; + case HashLookupResult.Uncertain: + m_findblocklisthashCommand.Transaction = transaction; + m_findblocklisthashCommand.SetParameterValue(0, hash); + var r = m_findblocklisthashCommand.ExecuteScalar(); + if (r != null && r != DBNull.Value) + { + m_blockListHashLookup.Add(hashdata, hash); + return; + } + break; + } + } + else + { + m_findblocklisthashCommand.Transaction = transaction; + m_findblocklisthashCommand.SetParameterValue(0, hash); + var r = m_findblocklisthashCommand.ExecuteScalar(); + if (r != null && r != DBNull.Value) + return; + } + + if (m_blockListHashLookup != null) + m_blockListHashLookup.Add(hashdata, hash); + + m_insertBlockset.Transaction = transaction; + m_insertBlockset.SetParameterValue(0, hash); + + var index = 0L; + + foreach(var s in blocklisthashes) + { + m_insertBlockset.SetParameterValue(1, s); + m_insertBlockset.SetParameterValue(2, index++); + m_insertBlockset.ExecuteNonQuery(); + } + } + + + public IEnumerable GetBlockLists(long volumeid) + { + using(var cmd = m_connection.CreateCommand()) + { + cmd.CommandText = string.Format(@"SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"", ""Block"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" AND ""Block"".""VolumeID"" = ?"); + cmd.AddParameter(volumeid); + + using(var rd = cmd.ExecuteReader()) + while (rd.Read()) + yield return rd.GetValue(0).ToString(); + } + } + + public IEnumerable GetMissingBlockListVolumes(int passNo) + { + using(var cmd = m_connection.CreateCommand()) + { + var selectCommand = @"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"", ""RemoteVolume"".""ID"" FROM ""RemoteVolume"""; + + var missingBlocklistEntries = + @"SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"" LEFT OUTER JOIN ""BlocksetEntry"" ON ""BlocksetEntry"".""Index"" = ""BlocklistHash"".""Index"" AND ""BlocksetEntry"".""BlocksetID"" = ""BlocklistHash"".""BlocksetID"" WHERE ""BlocksetEntry"".""BlocksetID"" IS NULL"; + + var missingBlockInfo = + @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""VolumeID"" < 0 "; + + var missingBlocklistVolumes = string.Format( + @"SELECT ""VolumeID"" FROM ""Block"", (" + + missingBlocklistEntries + + @") A WHERE ""A"".""Hash"" = ""Block"".""Hash"" " + ); + + var countMissingInformation = string.Format( + @"SELECT COUNT(*) FROM (SELECT DISTINCT ""VolumeID"" FROM ({0} UNION {1}))", + missingBlockInfo, + missingBlocklistEntries); + + if (passNo == 0) + { + // On the first pass, we select all the volumes we know we need, + // which may be an empty list + cmd.CommandText = string.Format(selectCommand + @" WHERE ""ID"" IN ({0})", missingBlocklistVolumes); + + // Reset the list + m_proccessedVolumes = new Dictionary(); + } + else + { + //On anything but the first pass, we check if we are done + var r = cmd.ExecuteScalar(countMissingInformation); + if (r == null || r == DBNull.Value || Convert.ToInt64(r) == 0) + yield break; + + if (passNo == 1) + { + // On the second pass, we select all volumes that are not mentioned in the db + + var mentionedVolumes = + @"SELECT DISTINCT ""VolumeID"" FROM ""Block"" "; + + cmd.CommandText = string.Format(selectCommand + @" WHERE ""ID"" NOT IN ({0}) AND ""Type"" = ? ", mentionedVolumes); + cmd.AddParameter(RemoteVolumeType.Blocks.ToString()); + } + else + { + // On the final pass, we select all volumes + // the filter will ensure that we do not download anything twice + cmd.CommandText = selectCommand + @" WHERE ""Type"" = ?"; + cmd.AddParameter(RemoteVolumeType.Blocks.ToString()); + } + } + + using(var rd = cmd.ExecuteReader()) + { + object[] r = new object[4]; + while (rd.Read()) + { + rd.GetValues(r); + + var volumeID = Convert.ToInt64(r[3]); + + // Guard against multiple downloads of the same file + if (!m_proccessedVolumes.ContainsKey(volumeID)) + { + m_proccessedVolumes.Add(volumeID, volumeID); + + yield return new RemoteVolume( + (r[0] == null || r[0] == DBNull.Value) ? null : r[0].ToString(), + (r[1] == null || r[1] == DBNull.Value) ? null : r[1].ToString(), + (r[2] == null || r[2] == DBNull.Value) ? -1 : Convert.ToInt64(r[2]) + ); + } + } + } + + + } + } + + public override void Dispose() + { + using (var cmd = m_connection.CreateCommand()) + { + if (m_tempblocklist != null) + try + { + cmd.CommandText = string.Format(@"DROP TABLE ""{0}""", m_tempblocklist); + cmd.ExecuteNonQuery(); + } + finally { m_tempblocklist = null; } + + } + + foreach(var cmd in new IDisposable [] { + m_insertFileCommand, + m_insertFilesetEntryCommand, + m_insertMetadatasetCommand, + m_insertBlocksetCommand, + m_insertBlocklistHashCommand, + m_updateBlockVolumeCommand, + m_insertBlockset, + m_findBlocksetCommand, + m_findMetadatasetCommand, + m_findFilesetCommand, + m_blockHashLookup, + m_fileHashLookup, + m_metadataLookup, + m_filesetLookup, + m_findblocklisthashCommand, + m_findHashBlockCommand, + m_insertBlockCommand, + m_insertDuplicateBlockCommand + }) + try + { + if (cmd != null) + cmd.Dispose(); + } + catch + { + } + + base.Dispose(); + } + } +} diff --git a/Duplicati/Library/Main/Database/RestoreIterators.cs b/Duplicati/Library/Main/Database/RestoreIterators.cs index 3171487a3..0047d48b1 100644 --- a/Duplicati/Library/Main/Database/RestoreIterators.cs +++ b/Duplicati/Library/Main/Database/RestoreIterators.cs @@ -967,83 +967,4 @@ namespace Duplicati.Library.Main.Database } } } - - internal partial class LocalBlocklistUpdateDatabase - { - private class BlocklistsEnumerable : IEnumerable - { - private class BlocklistEnumerator : IEnumerator - { - private System.Data.IDbConnection m_connection; - private string m_temptable; - private long m_volumeid; - private System.Data.IDataReader m_reader; - private System.Data.IDbCommand m_command; - private string m_current; - - public BlocklistEnumerator(System.Data.IDbConnection connection, string temptable, long volumeid) - { - m_connection = connection; - m_temptable = temptable; - m_volumeid = volumeid; - this.Reset(); - } - - public string Current { get { return m_current; } } - - public void Dispose() - { - if (m_reader != null) - try { m_reader.Dispose(); } - finally { m_reader = null; } - - if (m_command != null) - try { m_command.Dispose(); } - finally { m_command = null; } - } - - object System.Collections.IEnumerator.Current { get { return this.Current; } } - - public bool MoveNext() - { - if (!m_reader.Read()) - { - m_current = null; - return false; - } - - var c = m_reader.GetValue(0); - if (c == null || c == DBNull.Value) - m_current = null; - else - m_current = c.ToString(); - - return true; - } - - public void Reset() - { - this.Dispose(); - m_current = null; - m_command = m_connection.CreateCommand(); - m_command.CommandText = string.Format(@"SELECT ""Hash"" FROM ""{0}"" WHERE ""VolumeID"" = ""{1}"" AND ""Restored"" = 0 ", m_temptable, m_volumeid); - m_reader = m_command.ExecuteReader(); - } - } - - private System.Data.IDbConnection m_connection; - private string m_temptable; - private long m_volumeid; - - public BlocklistsEnumerable(System.Data.IDbConnection connection, string temptable, long volumeid) - { - m_connection = connection; - m_temptable = temptable; - m_volumeid = volumeid; - } - - public IEnumerator GetEnumerator() { return new BlocklistEnumerator(m_connection, m_temptable, m_volumeid); } - System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return this.GetEnumerator(); } - } - } } diff --git a/Duplicati/Library/Main/Duplicati.Library.Main.csproj b/Duplicati/Library/Main/Duplicati.Library.Main.csproj index 62c040b0a..2dda7af12 100644 --- a/Duplicati/Library/Main/Duplicati.Library.Main.csproj +++ b/Duplicati/Library/Main/Duplicati.Library.Main.csproj @@ -66,7 +66,6 @@ - @@ -115,6 +114,7 @@ + diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index f09f53c7a..f9907ec81 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -633,6 +633,11 @@ namespace Duplicati.Library.Main.Operation if (m_database.AddBlock(key, len, m_blockvolume.VolumeID, m_transaction)) { m_blockvolume.AddBlock(key, data, len, hint); + + //TODO: In theory a normal data block and blocklist block could be equal. + // this would cause the index file to not contain all data, + // if the data file is added before the blocklist data + // ... highly theoretical ... if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full && isBlocklistData) m_indexvolume.WriteBlocklist(key, data, len); diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index 510ca8302..c8da56fe2 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -14,89 +14,89 @@ namespace Duplicati.Library.Main.Operation private RecreateDatabaseResults m_result; public delegate IEnumerable> NumberedFilterFilelistDelegate(IEnumerable filelist); - public delegate void BlockVolumePostProcessor(string volumename,BlockVolumeReader reader); + public delegate void BlockVolumePostProcessor(string volumename,BlockVolumeReader reader); - public RecreateDatabaseHandler(string backendurl, Options options, RecreateDatabaseResults result) - { - m_options = options; - m_backendurl = backendurl; + public RecreateDatabaseHandler(string backendurl, Options options, RecreateDatabaseResults result) + { + m_options = options; + m_backendurl = backendurl; m_result = result; - } - - /// - /// Run the recreate procedure - /// - /// Path to the database that will be created - /// A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist - /// Filters the files in a filelist to prevent downloading unwanted data - /// A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists - public void Run(string path, Library.Utility.IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null) - { - if (System.IO.File.Exists(path)) - throw new Exception(string.Format("Cannot recreate database because file already exists: {0}", path)); + } - using(var db = new LocalBlocklistUpdateDatabase(path, m_options.Blocksize)) - { + /// + /// Run the recreate procedure + /// + /// Path to the database that will be created + /// A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist + /// Filters the files in a filelist to prevent downloading unwanted data + /// A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists + public void Run(string path, Library.Utility.IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null) + { + if (System.IO.File.Exists(path)) + throw new Exception(string.Format("Cannot recreate database because file already exists: {0}", path)); + + using(var db = new LocalDatabase(path, "Recreate")) + { m_result.SetDatabase(db); - DoRun(db, filter, filelistfilter, blockprocessor); - } - } - - /// - /// Run the recreate procedure - /// - /// Path to the database that will be created - /// A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist - /// Filters the files in a filelist to prevent downloading unwanted data - /// A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists + DoRun(db, filter, filelistfilter, blockprocessor); + } + } + + /// + /// Run the recreate procedure + /// + /// Path to the database that will be created + /// A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist + /// Filters the files in a filelist to prevent downloading unwanted data + /// A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists internal void DoRun(LocalDatabase dbparent, Library.Utility.IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null) { - var hashalg = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm); - if (hashalg == null) - throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.BlockHashAlgorithm)); + var hashalg = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm); + if (hashalg == null) + throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.BlockHashAlgorithm)); var hashsize = hashalg.HashSize / 8; //We build a local database in steps. - using (var restoredb = new LocalBlocklistUpdateDatabase(dbparent, m_options.Blocksize)) - using (var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, restoredb)) + using(var restoredb = new LocalRecreateDatabase(dbparent, m_options)) + using(var backend = new BackendManager(m_backendurl, m_options, m_result.BackendWriter, restoredb)) { - var volumeIds = new Dictionary(); + var volumeIds = new Dictionary(); - var rawlist = backend.List(); - + var rawlist = backend.List(); + //First step is to examine the remote storage to see what // kind of data we can find var remotefiles = - (from x in rawlist - let n = VolumeBase.ParseFilename(x) - where - n != null - && - n.Prefix == m_options.Prefix - select n).ToArray(); //ToArray() ensures that we do not remote-request it multiple times + (from x in rawlist + let n = VolumeBase.ParseFilename(x) + where + n != null + && + n.Prefix == m_options.Prefix + select n).ToArray(); //ToArray() ensures that we do not remote-request it multiple times - if (remotefiles.Length == 0) - { - if (rawlist.Count == 0) - throw new Exception("No files were found at the remote location, perhaps the target url is incorrect?"); - else - { - var tmp = - (from x in rawlist - let n = VolumeBase.ParseFilename(x) - where - n != null - select n.Prefix).ToArray(); - - var types = tmp.Distinct().ToArray(); - if (tmp.Length == 0) - throw new Exception(string.Format("Found {0} files at the remote storage, but none that could be parsed", rawlist.Count)); - else if (types.Length == 1) - throw new Exception(string.Format("Found {0} parse-able files with the prefix {1}, did you forget to set the backup-prefix?", tmp.Length, types[0])); - else - throw new Exception(string.Format("Found {0} parse-able files (of {1} files) with different prefixes: {2}, did you forget to set the backup-prefix?", tmp.Length, rawlist.Count, string.Join(", ", types))); - } - } + if (remotefiles.Length == 0) + { + if (rawlist.Count == 0) + throw new Exception("No files were found at the remote location, perhaps the target url is incorrect?"); + else + { + var tmp = + (from x in rawlist + let n = VolumeBase.ParseFilename(x) + where + n != null + select n.Prefix).ToArray(); + + var types = tmp.Distinct().ToArray(); + if (tmp.Length == 0) + throw new Exception(string.Format("Found {0} files at the remote storage, but none that could be parsed", rawlist.Count)); + else if (types.Length == 1) + throw new Exception(string.Format("Found {0} parse-able files with the prefix {1}, did you forget to set the backup-prefix?", tmp.Length, types[0])); + else + throw new Exception(string.Format("Found {0} parse-able files (of {1} files) with different prefixes: {2}, did you forget to set the backup-prefix?", tmp.Length, rawlist.Count, string.Join(", ", types))); + } + } //Then we select the filelist we should work with, // and create the filelist table to fit @@ -109,125 +109,155 @@ namespace Duplicati.Library.Main.Operation if (filelistfilter != null) filelists = filelistfilter(filelists).Select(x => x.Value).ToArray(); - using (var backupdb = new LocalBackupDatabase(restoredb, m_options)) + foreach(var fl in remotefiles) + volumeIds[fl.File.Name] = restoredb.RegisterRemoteVolume(fl.File.Name, fl.FileType, RemoteVolumeState.Uploaded); + + //Record all blocksets and files needed + using(var tr = restoredb.BeginTransaction()) { - foreach (var fl in remotefiles) - volumeIds[fl.File.Name] = backupdb.RegisterRemoteVolume(fl.File.Name, fl.FileType, RemoteVolumeState.Uploaded); - - //We grab all index files, and update the block table - using (var tr = restoredb.BeginTransaction()) - { - var indexfiles = - from n in remotefiles - where n.FileType == RemoteVolumeType.Index - select new RemoteVolume(n.File) as IRemoteVolume; - - foreach (var sf in new AsyncDownloader(indexfiles.ToList(), backend)) - using (var tmpfile = sf.TempFile) + var filelistWork = (from n in filelists orderby n.Time select new RemoteVolume(n.File) as IRemoteVolume).ToList(); + foreach(var entry in new AsyncDownloader(filelistWork, backend)) + try { - if (sf.Hash != null && sf.Size > 0) - backupdb.UpdateRemoteVolume(sf.Name, RemoteVolumeState.Verified, sf.Size, sf.Hash, tr); - - using (var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(sf.Name), tmpfile, m_options, hashsize)) + using(var tmpfile = entry.TempFile) { - Utility.VerifyParameters(restoredb, m_options); + if (entry.Hash != null && entry.Size > 0) + restoredb.UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, entry.Size, entry.Hash, tr); - //If there are blocklists in the index file, update the blocklists - foreach (var b in svr.BlockLists) - restoredb.UpdateBlocklist(b.Hash, b.Blocklist, hashsize, tr); - - foreach (var a in svr.Volumes) + var parsed = VolumeBase.ParseFilename(entry.Name); + // Create timestamped operations based on the file timestamp + var filesetid = restoredb.CreateFileset(volumeIds[entry.Name], parsed.Time, tr); + using(var filelistreader = new FilesetVolumeReader(parsed.CompressionModule, tmpfile, m_options)) + foreach(var fe in filelistreader.Files.Where(x => Library.Utility.FilterExpression.Matches(filter, x.Path))) + { + try + { + if (fe.Type == FilelistEntryType.Folder) + { + restoredb.AddDirectoryEntry(filesetid, fe.Path, fe.Time, fe.Metahash, fe.Metahash == null ? -1 : fe.Metasize, tr); + } + else if (fe.Type == FilelistEntryType.File) + { + var blocksetid = restoredb.AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, tr); + restoredb.AddFileEntry(filesetid, fe.Path, fe.Time, blocksetid, fe.Metahash, fe.Metahash == null ? -1 : fe.Metasize, tr); + } + else if (fe.Type == FilelistEntryType.Symlink) + { + restoredb.AddSymlinkEntry(filesetid, fe.Path, fe.Time, fe.Metahash, fe.Metahash == null ? -1 : fe.Metasize, tr); + } + else + { + m_result.AddWarning(string.Format("Skipping file-entry with unknown type {0}: {1} ", fe.Type, fe.Path), null); + } + } + catch (Exception ex) + { + m_result.AddWarning(string.Format("Failed to process file-entry: {0}", fe.Path), ex); + } + } + } + } + catch (Exception ex) + { + m_result.AddWarning(string.Format("Failed to process file: {0}", entry.Name), ex); + } + + using(new Logging.Timer("CommitUpdateFilesetFromRemote")) + tr.Commit(); + } + + //Grab all index files, and update the block table + using(var tr = restoredb.BeginTransaction()) + { + var indexfiles = + from n in remotefiles + where n.FileType == RemoteVolumeType.Index + select new RemoteVolume(n.File) as IRemoteVolume; + + foreach(var sf in new AsyncDownloader(indexfiles.ToList(), backend)) + try + { + using(var tmpfile = sf.TempFile) + { + if (sf.Hash != null && sf.Size > 0) + restoredb.UpdateRemoteVolume(sf.Name, RemoteVolumeState.Verified, sf.Size, sf.Hash, tr); + + using(var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(sf.Name), tmpfile, m_options, hashsize)) { - var volumeID = restoredb.GetRemoteVolumeID(a.Filename); - //Add all block/volume mappings - foreach (var b in a.Blocks) - backupdb.AddBlock(b.Key, b.Value, volumeID, tr); + Utility.VerifyParameters(restoredb, m_options); - backupdb.UpdateRemoteVolume(a.Filename, RemoteVolumeState.Verified, a.Length, a.Hash, tr); - backupdb.AddIndexBlockLink(restoredb.GetRemoteVolumeID(sf.Name), volumeID, tr); + foreach(var a in svr.Volumes) + { + var volumeID = restoredb.GetRemoteVolumeID(a.Filename); + //Add all block/volume mappings + foreach(var b in a.Blocks) + restoredb.UpdateBlock(b.Key, b.Value, volumeID, tr); + + restoredb.UpdateRemoteVolume(a.Filename, RemoteVolumeState.Verified, a.Length, a.Hash, tr); + restoredb.AddIndexBlockLink(restoredb.GetRemoteVolumeID(sf.Name), volumeID, tr); + } + + //If there are blocklists in the index file, update the blocklists + foreach(var b in svr.BlockLists) + restoredb.UpdateBlockset(b.Hash, b.Blocklist, hashsize, tr); } } } - - using(new Logging.Timer("CommitRecreatedDb")) - tr.Commit(); - } - - //We need this to prepare for the block-lists - var dummylist = new string[0]; - - //Now record all blocksets and files needed - using (var tr = backupdb.BeginTransaction()) - { - var filelistWork = (from n in filelists select new RemoteVolume(n.File) as IRemoteVolume).ToList(); - foreach (var entry in new AsyncDownloader(filelistWork, backend)) - using (var tmpfile = entry.TempFile) + catch (Exception ex) { - if (entry.Hash != null && entry.Size > 0) - backupdb.UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, entry.Size, entry.Hash, tr); - - var parsed = VolumeBase.ParseFilename(entry.Name); - // Create timestamped operations based on the file timestamp - backupdb.CreateFileset(volumeIds[entry.Name], parsed.Time, tr); - using (var filelistreader = new FilesetVolumeReader(parsed.CompressionModule, tmpfile, m_options)) - foreach (var fe in filelistreader.Files.Where(x => Library.Utility.FilterExpression.Matches(filter, x.Path))) - { - if (fe.Type == FilelistEntryType.Folder) - { - long metaid = -1; - if (fe.Metahash != null) - backupdb.AddMetadataset(fe.Metahash, fe.Metasize, out metaid, tr); - backupdb.AddDirectoryEntry(fe.Path, metaid, fe.Time, tr); - - } - else if (fe.Type == FilelistEntryType.File) - { - long metaid = -1; - long blocksetid; - if (fe.Metahash != null) - backupdb.AddMetadataset(fe.Metahash, fe.Metasize, out metaid, tr); - - backupdb.AddBlockset(fe.Hash, fe.Size, m_options.Blocksize, dummylist, fe.BlocklistHashes, out blocksetid, tr); - backupdb.AddFile(fe.Path, fe.Time, blocksetid, metaid, tr); - } - } + //Not fatal + m_result.AddWarning(string.Format("Failed to process index file: {0}", sf.Name), ex); } - - using(new Logging.Timer("CommitUpdateFilesetFromRemote")) - tr.Commit(); - } + + using(new Logging.Timer("CommitRecreatedDb")) + tr.Commit(); + + // TODO: In some cases, we can avoid downloading all index files, + // if we are lucky and pick the right ones } - //We now need some blocklists, so we start by grabbing a - // volume with one of the blocklists - //For each volume we then update the blocklist table - // and then restore the blocks we know - restoredb.FindMissingBlocklistHashes(); - - foreach (var sf in new AsyncDownloader(restoredb.GetMissingBlockListVolumes().ToList(), backend)) - using (var tmpfile = sf.TempFile) - using (var rd = new BlockVolumeReader(RestoreHandler.GetCompressionModule(sf.Name), tmpfile, m_options)) - using (var tr = restoredb.BeginTransaction()) - { - var volumeid = restoredb.GetRemoteVolumeID(sf.Name); - foreach (var blocklisthash in restoredb.GetBlockLists(volumeid)) - restoredb.UpdateBlocklist(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize), hashsize, tr); - - using(new Logging.Timer("CommitRestoredBlocklist")) - tr.Commit(); - - //At this point we can patch files with data from the block volume - if (blockprocessor != null) - blockprocessor(sf.Name, rd); - } - + // We have now grabbed as much information as possible, + // if we are still missing data, we must now fetch block files + restoredb.FindMissingBlocklistHashes(hashsize, null); + + //We do this in three passes + for(var i = 0; i < 3; i++) + { + // Grab the list matching the pass type + var lst = restoredb.GetMissingBlockListVolumes(i).ToList(); + foreach (var sf in new AsyncDownloader(lst, backend)) + using (var tmpfile = sf.TempFile) + using (var rd = new BlockVolumeReader(RestoreHandler.GetCompressionModule(sf.Name), tmpfile, m_options)) + using (var tr = restoredb.BeginTransaction()) + { + var volumeid = restoredb.GetRemoteVolumeID(sf.Name); + + // Update the block table so we know about the block/volume map + foreach(var h in rd.Blocks) + restoredb.UpdateBlock(h.Key, h.Value, volumeid, tr); + + // Grab all known blocklists from the volume + foreach (var blocklisthash in restoredb.GetBlockLists(volumeid)) + restoredb.UpdateBlockset(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize), hashsize, tr); + + // Update tables so we know if we are done + restoredb.FindMissingBlocklistHashes(hashsize, tr); + + using(new Logging.Timer("CommitRestoredBlocklist")) + tr.Commit(); + + //At this point we can patch files with data from the block volume + if (blockprocessor != null) + blockprocessor(sf.Name, rd); + } + } + backend.WaitForComplete(restoredb, null); //All done, we must verify that we have all blocklist fully intact // if this fails, the db will not be deleted, so it can be used, // except to continue a backup restoredb.VerifyConsistency(null); - } }