diff --git a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs index 5bfb87904..e463f44f1 100644 --- a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs @@ -66,10 +66,10 @@ namespace Duplicati.Library.Main.Database */ } - cmd.ExecuteNonQuery(@"UPDATE ""LogData"" SET ""Message"" = ""ERASED!"" WHERE ""Message"" LIKE ""%/%"" OR ""Message"" LIKE ""%:\%"" "); - cmd.ExecuteNonQuery(@"UPDATE ""LogData"" SET ""Exception"" = ""ERASED!"" WHERE ""Exception"" LIKE ""%/%"" OR ""Exception"" LIKE ""%:\%"" "); + cmd.ExecuteNonQuery(@"UPDATE ""LogData"" SET ""Message"" = 'ERASED!' WHERE ""Message"" LIKE '%/%' OR ""Message"" LIKE '%:\%' "); + cmd.ExecuteNonQuery(@"UPDATE ""LogData"" SET ""Exception"" = 'ERASED!' WHERE ""Exception"" LIKE '%/%' OR ""Exception"" LIKE '%:\%' "); - cmd.ExecuteNonQuery(@"UPDATE ""Configuration"" SET ""Value"" = ""ERASED!"" WHERE ""Key"" = ""passphrase"" "); + cmd.ExecuteNonQuery(@"UPDATE ""Configuration"" SET ""Value"" = 'ERASED!' WHERE ""Key"" = 'passphrase' "); cmd.ExecuteNonQuery(string.Format(@"CREATE TABLE ""FixedFile"" AS SELECT ""B"".""ID"" AS ""ID"", ""A"".""Obfuscated"" AS ""Path"", ""B"".""BlocksetID"" AS ""BlocksetID"", ""B"".""MetadataID"" AS ""MetadataID"" FROM ""{0}"" ""A"", ""File"" ""B"" WHERE ""A"".""RealPath"" = ""B"".""Path"" ", tablename)); cmd.ExecuteNonQuery(@"DROP VIEW ""File"" "); diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index 6f17056f9..d1ed59cbb 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -27,7 +27,7 @@ namespace Duplicati.Library.Main.Database { internal class LocalListBrokenFilesDatabase : LocalDatabase { - private const string BLOCK_VOLUME_IDS = @"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = ""{2}"""; + private const string BLOCK_VOLUME_IDS = @"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{2}'"; // Invalid blocksets include those that: // - Have BlocksetEntries with unknown/invalid blocks (meaning the data to rebuild the blockset isn't available) diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index 4c2d1bd33..b6f4c669d 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -39,15 +39,15 @@ namespace Duplicati.Library.Main.Database { public int Compare(KeyValuePair x, KeyValuePair y) { - return x.Key == y.Key ? - (x.Value == y.Value ? - 0 - : (x.Value < y.Value ? -1 : 1)) + return x.Key == y.Key ? + (x.Value == y.Value ? + 0 + : (x.Value < y.Value ? -1 : 1)) : (x.Key < y.Key ? -1 : 1); } } } - + private readonly System.Data.IDbCommand m_insertFileCommand; private readonly System.Data.IDbCommand m_insertFilesetEntryCommand; private readonly System.Data.IDbCommand m_insertMetadatasetCommand; @@ -71,7 +71,7 @@ namespace Duplicati.Library.Main.Database /// A lookup table that prevents multiple downloads of the same volume /// private Dictionary m_proccessedVolumes; - + // SQL that finds index and block size for all blocklist hashes, based on the temporary hash list // with vars Used: // {0} --> Blocksize @@ -124,7 +124,7 @@ namespace Duplicati.Library.Main.Database m_tempblocklist = "TempBlocklist_" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); m_tempsmalllist = "TempSmalllist_" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using(var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand()) { cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)", m_tempblocklist)); cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""Index_{0}"" ON ""{0}"" (""BlockListHash"");", m_tempblocklist)); @@ -152,19 +152,19 @@ namespace Duplicati.Library.Main.Database m_insertFileCommand.CommandText = @"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") VALUES (?,?,?,?); SELECT last_insert_rowid();"; m_insertFileCommand.AddParameters(4); - + m_insertFilesetEntryCommand.CommandText = @"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (?,?,?)"; m_insertFilesetEntryCommand.AddParameters(3); m_insertMetadatasetCommand.CommandText = @"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (?); SELECT last_insert_rowid();"; m_insertMetadatasetCommand.AddParameters(1); - + m_insertBlocksetCommand.CommandText = @"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (?,?); SELECT last_insert_rowid();"; m_insertBlocksetCommand.AddParameters(2); - + m_insertBlocklistHashCommand.CommandText = @"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (?,?,?)"; m_insertBlocklistHashCommand.AddParameters(3); - + m_updateBlockVolumeCommand.CommandText = @"UPDATE ""Block"" SET ""VolumeID"" = ? WHERE ""Hash"" = ? AND ""Size"" = ?"; m_updateBlockVolumeCommand.AddParameters(3); @@ -176,36 +176,36 @@ namespace Duplicati.Library.Main.Database m_findBlocksetCommand.CommandText = @"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" = ? AND ""FullHash"" = ? "; m_findBlocksetCommand.AddParameters(2); - + m_findMetadatasetCommand.CommandText = @"SELECT ""Metadataset"".""ID"" FROM ""Metadataset"",""Blockset"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Blockset"".""FullHash"" = ? AND ""Blockset"".""Length"" = ? "; m_findMetadatasetCommand.AddParameters(2); - + m_findFilesetCommand.CommandText = @"SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = ? AND ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? "; m_findFilesetCommand.AddParameters(4); - + m_findTempBlockListHashCommand.CommandText = string.Format(@"SELECT DISTINCT ""BlockListHash"" FROM ""{0}"" WHERE ""BlockListHash"" = ? ", m_tempblocklist); m_findTempBlockListHashCommand.AddParameters(1); - + m_findHashBlockCommand.CommandText = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ? "; m_findHashBlockCommand.AddParameters(2); - + m_insertBlockCommand.CommandText = @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") VALUES (?,?,?)"; m_insertBlockCommand.AddParameters(3); - + m_insertDuplicateBlockCommand.CommandText = @"INSERT INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") VALUES ((SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?), ?)"; m_insertDuplicateBlockCommand.AddParameters(3); } public void FindMissingBlocklistHashes(long hashsize, long blocksize, System.Data.IDbTransaction transaction) { - using(var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand()) { cmd.Transaction = transaction; - + //Update all small blocklists and matching blocks var selectSmallBlocks = string.Format(@"SELECT ""BlockHash"", ""BlockSize"" FROM ""{0}""", m_tempsmalllist); - + var selectBlockHashes = string.Format( @"SELECT ""BlockHash"" AS ""FullHash"", ""BlockSize"" AS ""Length"" FROM ( " + SELECT_BLOCKLIST_ENTRIES + @@ -215,24 +215,24 @@ namespace Duplicati.Library.Main.Database m_tempblocklist, blocksize / hashsize ); - + var selectAllBlocks = @"SELECT DISTINCT ""FullHash"", ""Length"" FROM (" + selectBlockHashes + " UNION " + selectSmallBlocks + " )"; - + var selectNewBlocks = string.Format( @"SELECT ""FullHash"" AS ""Hash"", ""Length"" AS ""Size"", -1 AS ""VolumeID"" " + - @" FROM (SELECT ""A"".""FullHash"", ""A"".""Length"", CASE WHEN ""B"".""Hash"" IS NULL THEN '' ELSE ""B"".""Hash"" END AS ""Hash"", CASE WHEN ""B"".""Size"" is NULL THEN -1 ELSE ""B"".""Size"" END AS ""Size"" FROM ({0}) A" + - @" LEFT OUTER JOIN ""Block"" B ON ""B"".""Hash"" = ""A"".""FullHash"" AND ""B"".""Size"" = ""A"".""Length"" )" + + @" FROM (SELECT ""A"".""FullHash"", ""A"".""Length"", CASE WHEN ""B"".""Hash"" IS NULL THEN '' ELSE ""B"".""Hash"" END AS ""Hash"", CASE WHEN ""B"".""Size"" is NULL THEN -1 ELSE ""B"".""Size"" END AS ""Size"" FROM ({0}) A" + + @" LEFT OUTER JOIN ""Block"" B ON ""B"".""Hash"" = ""A"".""FullHash"" AND ""B"".""Size"" = ""A"".""Length"" )" + @" WHERE ""FullHash"" != ""Hash"" AND ""Length"" != ""Size"" ", - selectAllBlocks + selectAllBlocks ); - - var insertBlocksCommand = - @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") " + + + var insertBlocksCommand = + @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") " + selectNewBlocks; - + // Insert all known blocks into block table with volumeid = -1 cmd.ExecuteNonQuery(insertBlocksCommand); - + var selectBlocklistBlocksetEntries = string.Format( @"SELECT ""E"".""BlocksetID"" AS ""BlocksetID"", ""D"".""FullIndex"" AS ""Index"", ""F"".""ID"" AS ""BlockID"" FROM ( " + SELECT_BLOCKLIST_ENTRIES + @@ -253,17 +253,17 @@ namespace Duplicati.Library.Main.Database blocksize, m_tempsmalllist ); - + var selectAllBlocksetEntries = selectBlocklistBlocksetEntries + @" UNION " + selectBlocksetEntries; - + var selectFiltered = @"SELECT DISTINCT ""H"".""BlocksetID"", ""H"".""Index"", ""H"".""BlockID"" FROM (" + selectAllBlocksetEntries + @") H WHERE (""H"".""BlocksetID"" || ':' || ""H"".""Index"") NOT IN (SELECT (""ExistingBlocksetEntries"".""BlocksetID"" || ':' || ""ExistingBlocksetEntries"".""Index"") FROM ""BlocksetEntry"" ""ExistingBlocksetEntries"" )"; - + var insertBlocksetEntriesCommand = @"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") " + selectFiltered; @@ -286,13 +286,13 @@ namespace Duplicati.Library.Main.Database } } - /// + /// /// From the temporary tables 1) insert new blocks into Block (VolumeID to be set at a later stage) /// and 2) add missing BlocksetEntry lines - /// - /// hashsize and blocksize: global database parameters - /// hashOnly: do not take in account small blocks - these have been added at the - /// end of the index handling and are not changed in the dblock handling so we can ignore them + /// + /// hashsize and blocksize: global database parameters + /// hashOnly: do not take in account small blocks - these have been added at the + /// end of the index handling and are not changed in the dblock handling so we can ignore them /// /// Notes: /// @@ -305,7 +305,7 @@ namespace Duplicati.Library.Main.Database public void AddBlockAndBlockSetEntryFromTemp(long hashsize, long blocksize, System.Data.IDbTransaction transaction, bool hashOnly = false) { - using(var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand()) { cmd.Transaction = transaction; @@ -326,19 +326,20 @@ namespace Duplicati.Library.Main.Database blocksize, blocksize / hashsize, m_tempblocklist - ); - if (!hashOnly) { - insertBlocksCommand += string.Format( - @" UNION " + - @"" + - @"SELECT TS.BlockHash, TS.BlockSize FROM " + - @"{0} TS " + - @"WHERE NOT EXISTS (SELECT ""X"" FROM Block AS B WHERE " + - @" B.Hash = TS.BlockHash AND " + - @" B.Size = TS.BlockSize) " + - @")", - m_tempsmalllist - ); + ); + if (!hashOnly) + { + insertBlocksCommand += string.Format( + @" UNION " + + @"" + + @"SELECT TS.BlockHash, TS.BlockSize FROM " + + @"{0} TS " + + @"WHERE NOT EXISTS (SELECT ""X"" FROM Block AS B WHERE " + + @" B.Hash = TS.BlockHash AND " + + @" B.Size = TS.BlockSize) " + + @")", + m_tempsmalllist + ); } var insertBlocksetEntriesCommand = string.Format( @@ -353,22 +354,23 @@ namespace Duplicati.Library.Main.Database blocksize / hashsize, m_tempblocklist ); - if (!hashOnly) { - insertBlocksetEntriesCommand += string.Format( - @" UNION " + - @"SELECT BS.ID AS BlocksetID, 0 AS ""Index"", BL.ID AS BlockID " + - @"FROM {1} TS " + - @" JOIN Blockset BS ON (BS.FullHash = TS.FileHash AND " + - @" BS.Length = TS.BlockSize AND " + - @" BS.Length <= {0}) " + - @" JOIN Block BL ON (BL.Hash = TS.BlockHash AND " + - @" BL.Size = TS.BlockSize) " + - @" LEFT OUTER JOIN BlocksetEntry BE ON (BE.BlocksetID = BS.ID AND BE.""Index"" = 0) " + - @"WHERE " + - @"BE.BlocksetID IS NULL ", - blocksize, - m_tempsmalllist - ); + if (!hashOnly) + { + insertBlocksetEntriesCommand += string.Format( + @" UNION " + + @"SELECT BS.ID AS BlocksetID, 0 AS ""Index"", BL.ID AS BlockID " + + @"FROM {1} TS " + + @" JOIN Blockset BS ON (BS.FullHash = TS.FileHash AND " + + @" BS.Length = TS.BlockSize AND " + + @" BS.Length <= {0}) " + + @" JOIN Block BL ON (BL.Hash = TS.BlockHash AND " + + @" BL.Size = TS.BlockSize) " + + @" LEFT OUTER JOIN BlocksetEntry BE ON (BE.BlocksetID = BS.ID AND BE.""Index"" = 0) " + + @"WHERE " + + @"BE.BlocksetID IS NULL ", + blocksize, + m_tempsmalllist + ); } try @@ -402,7 +404,7 @@ namespace Duplicati.Library.Main.Database { AddEntry(filesetid, pathprefixid, path, time, SYMLINK_BLOCKSET_ID, metadataid, transaction); } - + public void AddFileEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction) { AddEntry(filesetid, pathprefixid, path, time, blocksetid, metadataid, transaction); @@ -411,7 +413,7 @@ namespace Duplicati.Library.Main.Database private void AddEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction) { var fileid = -1L; - + m_findFilesetCommand.Transaction = transaction; m_findFilesetCommand.SetParameterValue(0, pathprefixid); m_findFilesetCommand.SetParameterValue(1, path); @@ -428,20 +430,20 @@ namespace Duplicati.Library.Main.Database m_insertFileCommand.SetParameterValue(3, metadataid); fileid = m_insertFileCommand.ExecuteScalarInt64(-1); } - + m_insertFilesetEntryCommand.Transaction = transaction; m_insertFilesetEntryCommand.SetParameterValue(0, filesetid); m_insertFilesetEntryCommand.SetParameterValue(1, fileid); m_insertFilesetEntryCommand.SetParameterValue(2, time.ToUniversalTime().Ticks); m_insertFilesetEntryCommand.ExecuteNonQuery(); } - + public long AddMetadataset(string metahash, long metahashsize, IEnumerable metablocklisthashes, long expectedmetablocklisthashes, System.Data.IDbTransaction transaction) { var metadataid = -1L; if (metahash == null) return metadataid; - + m_findMetadatasetCommand.Transaction = transaction; m_findMetadatasetCommand.SetParameterValue(0, metahash); m_findMetadatasetCommand.SetParameterValue(1, metahashsize); @@ -450,14 +452,14 @@ namespace Duplicati.Library.Main.Database return metadataid; var blocksetid = AddBlockset(metahash, metahashsize, metablocklisthashes, expectedmetablocklisthashes, transaction); - + m_insertMetadatasetCommand.Transaction = transaction; m_insertMetadatasetCommand.SetParameterValue(0, blocksetid); metadataid = m_insertMetadatasetCommand.ExecuteScalarInt64(-1); - + return metadataid; } - + public long AddBlockset(string fullhash, long size, IEnumerable blocklisthashes, long expectedblocklisthashes, System.Data.IDbTransaction transaction) { m_findBlocksetCommand.Transaction = transaction; @@ -465,8 +467,8 @@ namespace Duplicati.Library.Main.Database m_findBlocksetCommand.SetParameterValue(1, fullhash); var blocksetid = m_findBlocksetCommand.ExecuteScalarInt64(-1); if (blocksetid != -1) - return blocksetid; - + return blocksetid; + m_insertBlocksetCommand.Transaction = transaction; m_insertBlocksetCommand.SetParameterValue(0, size); m_insertBlocksetCommand.SetParameterValue(1, fullhash); @@ -479,7 +481,7 @@ namespace Duplicati.Library.Main.Database m_insertBlocklistHashCommand.Transaction = transaction; m_insertBlocklistHashCommand.SetParameterValue(0, blocksetid); - foreach(var hash in blocklisthashes) + foreach (var hash in blocklisthashes) { if (!string.IsNullOrEmpty(hash)) { @@ -493,10 +495,10 @@ namespace Duplicati.Library.Main.Database } } } - + if (c != expectedblocklisthashes) Logging.Log.WriteWarningMessage(LOGTAG, "MismatchInBlocklistHashCount", null, "Mismatching number of blocklist hashes detected on blockset {2}. Expected {0} blocklist hashes, but found {1}", expectedblocklisthashes, c, blocksetid); - + return blocksetid; } @@ -520,7 +522,7 @@ namespace Duplicati.Library.Main.Database m_insertBlockCommand.SetParameterValue(1, size); m_insertBlockCommand.SetParameterValue(2, volumeID); m_insertBlockCommand.ExecuteNonQuery(); - + return true; } else if (currentVolumeId == -1) @@ -532,8 +534,8 @@ namespace Duplicati.Library.Main.Database m_updateBlockVolumeCommand.SetParameterValue(2, size); var c = m_updateBlockVolumeCommand.ExecuteNonQuery(); if (c != 1) - throw new Exception(string.Format("Failed to update table, found {0} entries for key {1} with size {2}", c ,hash, size)); - + throw new Exception(string.Format("Failed to update table, found {0} entries for key {1} with size {2}", c, hash, size)); + return true; } else @@ -545,7 +547,7 @@ namespace Duplicati.Library.Main.Database m_insertDuplicateBlockCommand.ExecuteNonQuery(); return false; - } + } } public void AddSmallBlocksetLink(string filehash, string blockhash, long blocksize, System.Data.IDbTransaction transaction) @@ -564,13 +566,13 @@ namespace Duplicati.Library.Main.Database var r = m_findTempBlockListHashCommand.ExecuteScalar(); if (r != null && r != DBNull.Value) return false; - + m_insertTempBlockListHash.Transaction = transaction; m_insertTempBlockListHash.SetParameterValue(0, hash); - + var index = 0L; - - foreach(var s in blocklisthashes) + + foreach (var s in blocklisthashes) { m_insertTempBlockListHash.SetParameterValue(1, s); m_insertTempBlockListHash.SetParameterValue(2, index++); @@ -578,17 +580,17 @@ namespace Duplicati.Library.Main.Database } return true; - } + } public IEnumerable GetBlockLists(long volumeid) { - using(var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand()) { cmd.CommandText = @"SELECT DISTINCT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"", ""Block"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" AND ""Block"".""VolumeID"" = ?"; cmd.AddParameter(volumeid); - - using(var rd = cmd.ExecuteReader()) + + using (var rd = cmd.ExecuteReader()) while (rd.Read()) yield return rd.GetValue(0).ToString(); } @@ -596,11 +598,11 @@ namespace Duplicati.Library.Main.Database public IEnumerable GetMissingBlockListVolumes(int passNo, long blocksize, long hashsize, bool forceBlockUse) { - using(var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand()) { var selectCommand = @"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"", ""RemoteVolume"".""ID"" FROM ""RemoteVolume"""; - - var missingBlocklistEntries = + + var missingBlocklistEntries = string.Format( @"SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"" LEFT OUTER JOIN ""BlocksetEntry"" ON ""BlocksetEntry"".""Index"" = (""BlocklistHash"".""Index"" * {0}) AND ""BlocksetEntry"".""BlocksetID"" = ""BlocklistHash"".""BlocksetID"" WHERE ""BlocksetEntry"".""BlocksetID"" IS NULL", blocksize / hashsize @@ -608,13 +610,13 @@ namespace Duplicati.Library.Main.Database var missingBlockInfo = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""VolumeID"" < 0 AND SIZE > 0"; - + var missingBlocklistVolumes = string.Format( @"SELECT ""VolumeID"" FROM ""Block"", (" + - missingBlocklistEntries + + missingBlocklistEntries + @") A WHERE ""A"".""Hash"" = ""Block"".""Hash"" " ); - + var countMissingInformation = string.Format( @"SELECT COUNT(*) FROM (SELECT DISTINCT ""VolumeID"" FROM ({0} UNION {1}))", missingBlockInfo, @@ -625,7 +627,7 @@ namespace Duplicati.Library.Main.Database // On the first pass, we select all the volumes we know we need, // which may be an empty list cmd.CommandText = string.Format(selectCommand + @" WHERE ""ID"" IN ({0})", missingBlocklistVolumes); - + // Reset the list m_proccessedVolumes = new Dictionary(); } @@ -635,14 +637,14 @@ namespace Duplicati.Library.Main.Database var r = cmd.ExecuteScalarInt64(countMissingInformation, 0); if (r == 0 && !forceBlockUse) yield break; - + if (passNo == 1) { // On the second pass, we select all volumes that are not mentioned in the db - + var mentionedVolumes = @"SELECT DISTINCT ""VolumeID"" FROM ""Block"" "; - + cmd.CommandText = string.Format(selectCommand + @" WHERE ""ID"" NOT IN ({0}) AND ""Type"" = ? ", mentionedVolumes); cmd.AddParameter(RemoteVolumeType.Blocks.ToString()); } @@ -654,19 +656,19 @@ namespace Duplicati.Library.Main.Database cmd.AddParameter(RemoteVolumeType.Blocks.ToString()); } } - - using(var rd = cmd.ExecuteReader()) + + using (var rd = cmd.ExecuteReader()) { while (rd.Read()) { var volumeID = rd.GetInt64(3); - + // Guard against multiple downloads of the same file if (!m_proccessedVolumes.ContainsKey(volumeID)) { m_proccessedVolumes.Add(volumeID, volumeID); - + yield return new RemoteVolume( rd.GetString(0), rd.ConvertValueToString(1), @@ -675,56 +677,58 @@ namespace Duplicati.Library.Main.Database } } } - - + + } } - public void CleanupMissingVolumes() - { - var tablename = "SwapBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + public void CleanupMissingVolumes() + { + var tablename = "SwapBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + + // TODO: either hardcode all string constants or none // The first part of this query swaps out blocks for non-present remote files with - // existing ones (as recorded in the DuplicateBlock table) + // existing ones (as recorded in the DuplicateBlock table) // The second part removes references to the non-present remote files, // and marks the index files that pointed to them, such that they will be removed later on - var sql = $@" + var sql = $@" CREATE TEMPORARY TABLE ""{tablename}"" AS -SELECT ""A"".""ID"" AS ""BlockID"", ""A"".""VolumeID"" AS ""SourceVolumeID"", ""A"".""State"" AS ""SourceVolumeState"", ""B"".""VolumeID"" AS ""TargetVolumeID"", ""B"".""State"" AS ""TargetVolumeState"" FROM (SELECT ""Block"".""ID"", ""Block"".""VolumeID"", ""Remotevolume"".""State"" FROM ""Block"", ""Remotevolume"" WHERE ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" and ""Remotevolume"".""State"" = ""{RemoteVolumeState.Temporary}"") A, (SELECT ""DuplicateBlock"".""BlockID"", ""DuplicateBlock"".""VolumeID"", ""Remotevolume"".""State"" FROM ""DuplicateBlock"", ""Remotevolume"" WHERE ""DuplicateBlock"".""VolumeID"" = ""Remotevolume"".""ID"" and ""Remotevolume"".""State"" = ""{RemoteVolumeState.Verified}"") B WHERE ""A"".""ID"" = ""B"".""BlockID""; +SELECT ""A"".""ID"" AS ""BlockID"", ""A"".""VolumeID"" AS ""SourceVolumeID"", ""A"".""State"" AS ""SourceVolumeState"", ""B"".""VolumeID"" AS ""TargetVolumeID"", ""B"".""State"" AS ""TargetVolumeState"" FROM (SELECT ""Block"".""ID"", ""Block"".""VolumeID"", ""Remotevolume"".""State"" FROM ""Block"", ""Remotevolume"" WHERE ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" and ""Remotevolume"".""State"" = ""{RemoteVolumeState.Temporary}"") A, (SELECT ""DuplicateBlock"".""BlockID"", ""DuplicateBlock"".""VolumeID"", ""Remotevolume"".""State"" FROM ""DuplicateBlock"", ""Remotevolume"" WHERE ""DuplicateBlock"".""VolumeID"" = ""Remotevolume"".""ID"" and ""Remotevolume"".""State"" = '{RemoteVolumeState.Verified}') B WHERE ""A"".""ID"" = ""B"".""BlockID""; UPDATE ""Block"" SET ""VolumeID"" = (SELECT ""TargetVolumeID"" FROM ""{tablename}"" WHERE ""Block"".""ID"" = ""{tablename}"".""BlockID"") WHERE ""Block"".""ID"" IN (SELECT ""BlockID"" FROM ""{tablename}""); UPDATE ""DuplicateBlock"" SET ""VolumeID"" = (SELECT ""SourceVolumeID"" FROM ""{tablename}"" WHERE ""DuplicateBlock"".""BlockID"" = ""{tablename}"".""BlockID"") WHERE ""DuplicateBlock"".""BlockID"" IN (SELECT ""BlockID"" FROM ""{tablename}""); DROP TABLE ""{tablename}""; -DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = ""{RemoteVolumeType.Blocks}"" AND ""State"" = ""{RemoteVolumeState.Temporary}"" AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"")); -DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = ""Blocks"" AND ""State"" = ""{RemoteVolumeState.Temporary}"" AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"")); -DELETE FROM ""RemoteVolume"" WHERE ""Type"" = ""{RemoteVolumeType.Blocks}"" AND ""State"" = ""{RemoteVolumeState.Temporary}"" AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block""); +DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND ""State"" = '{RemoteVolumeState.Temporary}' AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"")); +DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND ""State"" = '{RemoteVolumeState.Temporary}' AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"")); +DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND ""State"" = '{RemoteVolumeState.Temporary}' AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block""); "; -// We could delete these, but we don't have to, so we keep them around until the next compact is done -// UPDATE ""RemoteVolume"" SET ""State"" = ""{3}"" WHERE ""Type"" = ""{5}"" AND ""ID"" NOT IN (SELECT ""IndexVolumeID"" FROM ""IndexBlockLink""); + // We could delete these, but we don't have to, so we keep them around until the next compact is done + // UPDATE ""RemoteVolume"" SET ""State"" = ""{3}"" WHERE ""Type"" = ""{5}"" AND ""ID"" NOT IN (SELECT ""IndexVolumeID"" FROM ""IndexBlockLink""); - var countsql = @"SELECT COUNT(*) FROM ""RemoteVolume"" WHERE ""State"" = ""Temporary"" AND ""Type"" = ""Blocks"" "; + var countsql = $@"SELECT COUNT(*) FROM ""RemoteVolume"" WHERE ""State"" = '{RemoteVolumeState.Temporary}' AND ""Type"" = '{RemoteVolumeType.Blocks}' "; - using (var cmd = m_connection.CreateCommand()) - { - var cnt = cmd.ExecuteScalarInt64(countsql); - if (cnt > 0) - { - Logging.Log.WriteWarningMessage(LOGTAG, "MissingVolumesDetected", null, "Found {0} missing volumes; attempting to replace blocks from existing volumes", cnt); - cmd.ExecuteNonQuery(sql); - - var cnt2 = cmd.ExecuteScalarInt64(countsql); - Logging.Log.WriteVerboseMessage(LOGTAG, "ReplacedMissingVolumes", "Replaced blocks for {0} missing volumes; there are now {1} missing volumes", cnt, cnt2); - } - } - } - - public override void Dispose() - { using (var cmd = m_connection.CreateCommand()) - { + { + var cnt = cmd.ExecuteScalarInt64(countsql); + if (cnt > 0) + { + Logging.Log.WriteWarningMessage(LOGTAG, "MissingVolumesDetected", null, "Found {0} missing volumes; attempting to replace blocks from existing volumes", cnt); + cmd.ExecuteNonQuery(sql); + + var cnt2 = cmd.ExecuteScalarInt64(countsql); + Logging.Log.WriteVerboseMessage(LOGTAG, "ReplacedMissingVolumes", "Replaced blocks for {0} missing volumes; there are now {1} missing volumes", cnt, cnt2); + } + } + } + + public override void Dispose() + { + using (var cmd = m_connection.CreateCommand()) + { if (m_tempblocklist != null) try { @@ -733,7 +737,7 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = ""{RemoteVolumeType.Blocks}"" AND } catch { } finally { m_tempblocklist = null; } - + if (m_tempsmalllist != null) try {