haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix sql queries that use double quoted strings.

Browse Source 
Closes #5202
This commit is contained in:
Jojo-1000
2024-05-30 00:00:48 +02:00
parent d9174d1488
commit 55284410c7
3 changed files with 147 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Main/Database/LocalBugReportDatabase.cs
+3 -3
View File
@@ -66,10 +66,10 @@ namespace Duplicati.Library.Main.Database
*/
}
cmd.ExecuteNonQuery(@"UPDATE ""LogData"" SET ""Message"" = ""ERASED!"" WHERE ""Message"" LIKE ""%/%"" OR ""Message"" LIKE ""%:\%"" ");
cmd.ExecuteNonQuery(@"UPDATE ""LogData"" SET ""Exception"" = ""ERASED!"" WHERE ""Exception"" LIKE ""%/%"" OR ""Exception"" LIKE ""%:\%"" ");
cmd.ExecuteNonQuery(@"UPDATE ""LogData"" SET ""Message"" = 'ERASED!' WHERE ""Message"" LIKE '%/%' OR ""Message"" LIKE '%:\%' ");
cmd.ExecuteNonQuery(@"UPDATE ""LogData"" SET ""Exception"" = 'ERASED!' WHERE ""Exception"" LIKE '%/%' OR ""Exception"" LIKE '%:\%' ");
cmd.ExecuteNonQuery(@"UPDATE ""Configuration"" SET ""Value"" = ""ERASED!"" WHERE ""Key"" = ""passphrase"" ");
cmd.ExecuteNonQuery(@"UPDATE ""Configuration"" SET ""Value"" = 'ERASED!' WHERE ""Key"" = 'passphrase' ");
cmd.ExecuteNonQuery(string.Format(@"CREATE TABLE ""FixedFile"" AS SELECT ""B"".""ID"" AS ""ID"", ""A"".""Obfuscated"" AS ""Path"", ""B"".""BlocksetID"" AS ""BlocksetID"", ""B"".""MetadataID"" AS ""MetadataID"" FROM ""{0}"" ""A"", ""File"" ""B"" WHERE ""A"".""RealPath"" = ""B"".""Path"" ", tablename));
cmd.ExecuteNonQuery(@"DROP VIEW ""File"" ");
Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs
+1 -1
View File
@@ -27,7 +27,7 @@ namespace Duplicati.Library.Main.Database
{
internal class LocalListBrokenFilesDatabase : LocalDatabase
{
private const string BLOCK_VOLUME_IDS = @"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = ""{2}""";
private const string BLOCK_VOLUME_IDS = @"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{2}'";
// Invalid blocksets include those that:
// - Have BlocksetEntries with unknown/invalid blocks (meaning the data to rebuild the blockset isn't available)
Duplicati/Library/Main/Database/LocalRecreateDatabase.cs
+143 -139
View File
@@ -39,15 +39,15 @@ namespace Duplicati.Library.Main.Database
{
public int Compare(KeyValuePair<long, long> x, KeyValuePair<long, long> y)
{
return x.Key == y.Key ?
(x.Value == y.Value ?
0
: (x.Value < y.Value ? -1 : 1))
return x.Key == y.Key ?
(x.Value == y.Value ?
0
: (x.Value < y.Value ? -1 : 1))
: (x.Key < y.Key ? -1 : 1);
}
}
}
private readonly System.Data.IDbCommand m_insertFileCommand;
private readonly System.Data.IDbCommand m_insertFilesetEntryCommand;
private readonly System.Data.IDbCommand m_insertMetadatasetCommand;
@@ -71,7 +71,7 @@ namespace Duplicati.Library.Main.Database
/// A lookup table that prevents multiple downloads of the same volume
/// </summary>
private Dictionary<long, long> m_proccessedVolumes;
// SQL that finds index and block size for all blocklist hashes, based on the temporary hash list
// with vars Used:
// {0} --> Blocksize
@@ -124,7 +124,7 @@ namespace Duplicati.Library.Main.Database
m_tempblocklist = "TempBlocklist_" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
m_tempsmalllist = "TempSmalllist_" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
using(var cmd = m_connection.CreateCommand())
using (var cmd = m_connection.CreateCommand())
{
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)", m_tempblocklist));
cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""Index_{0}"" ON ""{0}"" (""BlockListHash"");", m_tempblocklist));
@@ -152,19 +152,19 @@ namespace Duplicati.Library.Main.Database
m_insertFileCommand.CommandText = @"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") VALUES (?,?,?,?); SELECT last_insert_rowid();";
m_insertFileCommand.AddParameters(4);
m_insertFilesetEntryCommand.CommandText = @"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (?,?,?)";
m_insertFilesetEntryCommand.AddParameters(3);
m_insertMetadatasetCommand.CommandText = @"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (?); SELECT last_insert_rowid();";
m_insertMetadatasetCommand.AddParameters(1);
m_insertBlocksetCommand.CommandText = @"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (?,?); SELECT last_insert_rowid();";
m_insertBlocksetCommand.AddParameters(2);
m_insertBlocklistHashCommand.CommandText = @"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (?,?,?)";
m_insertBlocklistHashCommand.AddParameters(3);
m_updateBlockVolumeCommand.CommandText = @"UPDATE ""Block"" SET ""VolumeID"" = ? WHERE ""Hash"" = ? AND ""Size"" = ?";
m_updateBlockVolumeCommand.AddParameters(3);
@@ -176,36 +176,36 @@ namespace Duplicati.Library.Main.Database
m_findBlocksetCommand.CommandText = @"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" = ? AND ""FullHash"" = ? ";
m_findBlocksetCommand.AddParameters(2);
m_findMetadatasetCommand.CommandText = @"SELECT ""Metadataset"".""ID"" FROM ""Metadataset"",""Blockset"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Blockset"".""FullHash"" = ? AND ""Blockset"".""Length"" = ? ";
m_findMetadatasetCommand.AddParameters(2);
m_findFilesetCommand.CommandText = @"SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = ? AND ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? ";
m_findFilesetCommand.AddParameters(4);
m_findTempBlockListHashCommand.CommandText = string.Format(@"SELECT DISTINCT ""BlockListHash"" FROM ""{0}"" WHERE ""BlockListHash"" = ? ", m_tempblocklist);
m_findTempBlockListHashCommand.AddParameters(1);
m_findHashBlockCommand.CommandText = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ? ";
m_findHashBlockCommand.AddParameters(2);
m_insertBlockCommand.CommandText = @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") VALUES (?,?,?)";
m_insertBlockCommand.AddParameters(3);
m_insertDuplicateBlockCommand.CommandText = @"INSERT INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") VALUES ((SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?), ?)";
m_insertDuplicateBlockCommand.AddParameters(3);
}
public void FindMissingBlocklistHashes(long hashsize, long blocksize, System.Data.IDbTransaction transaction)
{
using(var cmd = m_connection.CreateCommand())
using (var cmd = m_connection.CreateCommand())
{
cmd.Transaction = transaction;
//Update all small blocklists and matching blocks
var selectSmallBlocks = string.Format(@"SELECT ""BlockHash"", ""BlockSize"" FROM ""{0}""", m_tempsmalllist);
var selectBlockHashes = string.Format(
@"SELECT ""BlockHash"" AS ""FullHash"", ""BlockSize"" AS ""Length"" FROM ( " +
SELECT_BLOCKLIST_ENTRIES +
@@ -215,24 +215,24 @@ namespace Duplicati.Library.Main.Database
m_tempblocklist,
blocksize / hashsize
);
var selectAllBlocks = @"SELECT DISTINCT ""FullHash"", ""Length"" FROM (" + selectBlockHashes + " UNION " + selectSmallBlocks + " )";
var selectNewBlocks = string.Format(
@"SELECT ""FullHash"" AS ""Hash"", ""Length"" AS ""Size"", -1 AS ""VolumeID"" " +
@" FROM (SELECT ""A"".""FullHash"", ""A"".""Length"", CASE WHEN ""B"".""Hash"" IS NULL THEN '' ELSE ""B"".""Hash"" END AS ""Hash"", CASE WHEN ""B"".""Size"" is NULL THEN -1 ELSE ""B"".""Size"" END AS ""Size"" FROM ({0}) A" +
@" LEFT OUTER JOIN ""Block"" B ON ""B"".""Hash"" = ""A"".""FullHash"" AND ""B"".""Size"" = ""A"".""Length"" )" +
@" FROM (SELECT ""A"".""FullHash"", ""A"".""Length"", CASE WHEN ""B"".""Hash"" IS NULL THEN '' ELSE ""B"".""Hash"" END AS ""Hash"", CASE WHEN ""B"".""Size"" is NULL THEN -1 ELSE ""B"".""Size"" END AS ""Size"" FROM ({0}) A" +
@" LEFT OUTER JOIN ""Block"" B ON ""B"".""Hash"" = ""A"".""FullHash"" AND ""B"".""Size"" = ""A"".""Length"" )" +
@" WHERE ""FullHash"" != ""Hash"" AND ""Length"" != ""Size"" ",
selectAllBlocks
selectAllBlocks
);
var insertBlocksCommand =
@"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") " +
var insertBlocksCommand =
@"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") " +
selectNewBlocks;
// Insert all known blocks into block table with volumeid = -1
cmd.ExecuteNonQuery(insertBlocksCommand);
var selectBlocklistBlocksetEntries = string.Format(
@"SELECT ""E"".""BlocksetID"" AS ""BlocksetID"", ""D"".""FullIndex"" AS ""Index"", ""F"".""ID"" AS ""BlockID"" FROM ( " +
SELECT_BLOCKLIST_ENTRIES +
@@ -253,17 +253,17 @@ namespace Duplicati.Library.Main.Database
blocksize,
m_tempsmalllist
);
var selectAllBlocksetEntries =
selectBlocklistBlocksetEntries +
@" UNION " +
selectBlocksetEntries;
var selectFiltered =
@"SELECT DISTINCT ""H"".""BlocksetID"", ""H"".""Index"", ""H"".""BlockID"" FROM (" +
selectAllBlocksetEntries +
@") H WHERE (""H"".""BlocksetID"" || ':' || ""H"".""Index"") NOT IN (SELECT (""ExistingBlocksetEntries"".""BlocksetID"" || ':' || ""ExistingBlocksetEntries"".""Index"") FROM ""BlocksetEntry"" ""ExistingBlocksetEntries"" )";
var insertBlocksetEntriesCommand =
@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") " + selectFiltered;
@@ -286,13 +286,13 @@ namespace Duplicati.Library.Main.Database
}
}
/// <summary>
/// <summary>
/// From the temporary tables 1) insert new blocks into Block (VolumeID to be set at a later stage)
/// and 2) add missing BlocksetEntry lines
///
/// hashsize and blocksize: global database parameters
/// hashOnly: do not take in account small blocks - these have been added at the
/// end of the index handling and are not changed in the dblock handling so we can ignore them
///
/// hashsize and blocksize: global database parameters
/// hashOnly: do not take in account small blocks - these have been added at the
/// end of the index handling and are not changed in the dblock handling so we can ignore them
/// </summary>
/// Notes:
///
@@ -305,7 +305,7 @@ namespace Duplicati.Library.Main.Database
public void AddBlockAndBlockSetEntryFromTemp(long hashsize, long blocksize, System.Data.IDbTransaction transaction, bool hashOnly = false)
{
using(var cmd = m_connection.CreateCommand())
using (var cmd = m_connection.CreateCommand())
{
cmd.Transaction = transaction;
@@ -326,19 +326,20 @@ namespace Duplicati.Library.Main.Database
blocksize,
blocksize / hashsize,
m_tempblocklist
);
if (!hashOnly) {
insertBlocksCommand += string.Format(
@" UNION " +
@"" +
@"SELECT TS.BlockHash, TS.BlockSize FROM " +
@"{0} TS " +
@"WHERE NOT EXISTS (SELECT ""X"" FROM Block AS B WHERE " +
@" B.Hash = TS.BlockHash AND " +
@" B.Size = TS.BlockSize) " +
@")",
m_tempsmalllist
);
);
if (!hashOnly)
{
insertBlocksCommand += string.Format(
@" UNION " +
@"" +
@"SELECT TS.BlockHash, TS.BlockSize FROM " +
@"{0} TS " +
@"WHERE NOT EXISTS (SELECT ""X"" FROM Block AS B WHERE " +
@" B.Hash = TS.BlockHash AND " +
@" B.Size = TS.BlockSize) " +
@")",
m_tempsmalllist
);
}
var insertBlocksetEntriesCommand = string.Format(
@@ -353,22 +354,23 @@ namespace Duplicati.Library.Main.Database
blocksize / hashsize,
m_tempblocklist
);
if (!hashOnly) {
insertBlocksetEntriesCommand += string.Format(
@" UNION " +
@"SELECT BS.ID AS BlocksetID, 0 AS ""Index"", BL.ID AS BlockID " +
@"FROM {1} TS " +
@" JOIN Blockset BS ON (BS.FullHash = TS.FileHash AND " +
@" BS.Length = TS.BlockSize AND " +
@" BS.Length <= {0}) " +
@" JOIN Block BL ON (BL.Hash = TS.BlockHash AND " +
@" BL.Size = TS.BlockSize) " +
@" LEFT OUTER JOIN BlocksetEntry BE ON (BE.BlocksetID = BS.ID AND BE.""Index"" = 0) " +
@"WHERE " +
@"BE.BlocksetID IS NULL ",
blocksize,
m_tempsmalllist
);
if (!hashOnly)
{
insertBlocksetEntriesCommand += string.Format(
@" UNION " +
@"SELECT BS.ID AS BlocksetID, 0 AS ""Index"", BL.ID AS BlockID " +
@"FROM {1} TS " +
@" JOIN Blockset BS ON (BS.FullHash = TS.FileHash AND " +
@" BS.Length = TS.BlockSize AND " +
@" BS.Length <= {0}) " +
@" JOIN Block BL ON (BL.Hash = TS.BlockHash AND " +
@" BL.Size = TS.BlockSize) " +
@" LEFT OUTER JOIN BlocksetEntry BE ON (BE.BlocksetID = BS.ID AND BE.""Index"" = 0) " +
@"WHERE " +
@"BE.BlocksetID IS NULL ",
blocksize,
m_tempsmalllist
);
}
try
@@ -402,7 +404,7 @@ namespace Duplicati.Library.Main.Database
{
AddEntry(filesetid, pathprefixid, path, time, SYMLINK_BLOCKSET_ID, metadataid, transaction);
}
public void AddFileEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction)
{
AddEntry(filesetid, pathprefixid, path, time, blocksetid, metadataid, transaction);
@@ -411,7 +413,7 @@ namespace Duplicati.Library.Main.Database
private void AddEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction)
{
var fileid = -1L;
m_findFilesetCommand.Transaction = transaction;
m_findFilesetCommand.SetParameterValue(0, pathprefixid);
m_findFilesetCommand.SetParameterValue(1, path);
@@ -428,20 +430,20 @@ namespace Duplicati.Library.Main.Database
m_insertFileCommand.SetParameterValue(3, metadataid);
fileid = m_insertFileCommand.ExecuteScalarInt64(-1);
}
m_insertFilesetEntryCommand.Transaction = transaction;
m_insertFilesetEntryCommand.SetParameterValue(0, filesetid);
m_insertFilesetEntryCommand.SetParameterValue(1, fileid);
m_insertFilesetEntryCommand.SetParameterValue(2, time.ToUniversalTime().Ticks);
m_insertFilesetEntryCommand.ExecuteNonQuery();
}
public long AddMetadataset(string metahash, long metahashsize, IEnumerable<string> metablocklisthashes, long expectedmetablocklisthashes, System.Data.IDbTransaction transaction)
{
var metadataid = -1L;
if (metahash == null)
return metadataid;
m_findMetadatasetCommand.Transaction = transaction;
m_findMetadatasetCommand.SetParameterValue(0, metahash);
m_findMetadatasetCommand.SetParameterValue(1, metahashsize);
@@ -450,14 +452,14 @@ namespace Duplicati.Library.Main.Database
return metadataid;
var blocksetid = AddBlockset(metahash, metahashsize, metablocklisthashes, expectedmetablocklisthashes, transaction);
m_insertMetadatasetCommand.Transaction = transaction;
m_insertMetadatasetCommand.SetParameterValue(0, blocksetid);
metadataid = m_insertMetadatasetCommand.ExecuteScalarInt64(-1);
return metadataid;
}
public long AddBlockset(string fullhash, long size, IEnumerable<string> blocklisthashes, long expectedblocklisthashes, System.Data.IDbTransaction transaction)
{
m_findBlocksetCommand.Transaction = transaction;
@@ -465,8 +467,8 @@ namespace Duplicati.Library.Main.Database
m_findBlocksetCommand.SetParameterValue(1, fullhash);
var blocksetid = m_findBlocksetCommand.ExecuteScalarInt64(-1);
if (blocksetid != -1)
return blocksetid;
return blocksetid;
m_insertBlocksetCommand.Transaction = transaction;
m_insertBlocksetCommand.SetParameterValue(0, size);
m_insertBlocksetCommand.SetParameterValue(1, fullhash);
@@ -479,7 +481,7 @@ namespace Duplicati.Library.Main.Database
m_insertBlocklistHashCommand.Transaction = transaction;
m_insertBlocklistHashCommand.SetParameterValue(0, blocksetid);
foreach(var hash in blocklisthashes)
foreach (var hash in blocklisthashes)
{
if (!string.IsNullOrEmpty(hash))
{
@@ -493,10 +495,10 @@ namespace Duplicati.Library.Main.Database
}
}
}
if (c != expectedblocklisthashes)
Logging.Log.WriteWarningMessage(LOGTAG, "MismatchInBlocklistHashCount", null, "Mismatching number of blocklist hashes detected on blockset {2}. Expected {0} blocklist hashes, but found {1}", expectedblocklisthashes, c, blocksetid);
return blocksetid;
}
@@ -520,7 +522,7 @@ namespace Duplicati.Library.Main.Database
m_insertBlockCommand.SetParameterValue(1, size);
m_insertBlockCommand.SetParameterValue(2, volumeID);
m_insertBlockCommand.ExecuteNonQuery();
return true;
}
else if (currentVolumeId == -1)
@@ -532,8 +534,8 @@ namespace Duplicati.Library.Main.Database
m_updateBlockVolumeCommand.SetParameterValue(2, size);
var c = m_updateBlockVolumeCommand.ExecuteNonQuery();
if (c != 1)
throw new Exception(string.Format("Failed to update table, found {0} entries for key {1} with size {2}", c ,hash, size));
throw new Exception(string.Format("Failed to update table, found {0} entries for key {1} with size {2}", c, hash, size));
return true;
}
else
@@ -545,7 +547,7 @@ namespace Duplicati.Library.Main.Database
m_insertDuplicateBlockCommand.ExecuteNonQuery();
return false;
}
}
}
public void AddSmallBlocksetLink(string filehash, string blockhash, long blocksize, System.Data.IDbTransaction transaction)
@@ -564,13 +566,13 @@ namespace Duplicati.Library.Main.Database
var r = m_findTempBlockListHashCommand.ExecuteScalar();
if (r != null && r != DBNull.Value)
return false;
m_insertTempBlockListHash.Transaction = transaction;
m_insertTempBlockListHash.SetParameterValue(0, hash);
var index = 0L;
foreach(var s in blocklisthashes)
foreach (var s in blocklisthashes)
{
m_insertTempBlockListHash.SetParameterValue(1, s);
m_insertTempBlockListHash.SetParameterValue(2, index++);
@@ -578,17 +580,17 @@ namespace Duplicati.Library.Main.Database
}
return true;
}
}
public IEnumerable<string> GetBlockLists(long volumeid)
{
using(var cmd = m_connection.CreateCommand())
using (var cmd = m_connection.CreateCommand())
{
cmd.CommandText = @"SELECT DISTINCT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"", ""Block"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" AND ""Block"".""VolumeID"" = ?";
cmd.AddParameter(volumeid);
using(var rd = cmd.ExecuteReader())
using (var rd = cmd.ExecuteReader())
while (rd.Read())
yield return rd.GetValue(0).ToString();
}
@@ -596,11 +598,11 @@ namespace Duplicati.Library.Main.Database
public IEnumerable<IRemoteVolume> GetMissingBlockListVolumes(int passNo, long blocksize, long hashsize, bool forceBlockUse)
{
using(var cmd = m_connection.CreateCommand())
using (var cmd = m_connection.CreateCommand())
{
var selectCommand = @"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"", ""RemoteVolume"".""ID"" FROM ""RemoteVolume""";
var missingBlocklistEntries =
var missingBlocklistEntries =
string.Format(
@"SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"" LEFT OUTER JOIN ""BlocksetEntry"" ON ""BlocksetEntry"".""Index"" = (""BlocklistHash"".""Index"" * {0}) AND ""BlocksetEntry"".""BlocksetID"" = ""BlocklistHash"".""BlocksetID"" WHERE ""BlocksetEntry"".""BlocksetID"" IS NULL",
blocksize / hashsize
@@ -608,13 +610,13 @@ namespace Duplicati.Library.Main.Database
var missingBlockInfo =
@"SELECT ""VolumeID"" FROM ""Block"" WHERE ""VolumeID"" < 0 AND SIZE > 0";
var missingBlocklistVolumes = string.Format(
@"SELECT ""VolumeID"" FROM ""Block"", (" +
missingBlocklistEntries +
missingBlocklistEntries +
@") A WHERE ""A"".""Hash"" = ""Block"".""Hash"" "
);
var countMissingInformation = string.Format(
@"SELECT COUNT(*) FROM (SELECT DISTINCT ""VolumeID"" FROM ({0} UNION {1}))",
missingBlockInfo,
@@ -625,7 +627,7 @@ namespace Duplicati.Library.Main.Database
// On the first pass, we select all the volumes we know we need,
// which may be an empty list
cmd.CommandText = string.Format(selectCommand + @" WHERE ""ID"" IN ({0})", missingBlocklistVolumes);
// Reset the list
m_proccessedVolumes = new Dictionary<long, long>();
}
@@ -635,14 +637,14 @@ namespace Duplicati.Library.Main.Database
var r = cmd.ExecuteScalarInt64(countMissingInformation, 0);
if (r == 0 && !forceBlockUse)
yield break;
if (passNo == 1)
{
// On the second pass, we select all volumes that are not mentioned in the db
var mentionedVolumes =
@"SELECT DISTINCT ""VolumeID"" FROM ""Block"" ";
cmd.CommandText = string.Format(selectCommand + @" WHERE ""ID"" NOT IN ({0}) AND ""Type"" = ? ", mentionedVolumes);
cmd.AddParameter(RemoteVolumeType.Blocks.ToString());
}
@@ -654,19 +656,19 @@ namespace Duplicati.Library.Main.Database
cmd.AddParameter(RemoteVolumeType.Blocks.ToString());
}
}
using(var rd = cmd.ExecuteReader())
using (var rd = cmd.ExecuteReader())
{
while (rd.Read())
{
var volumeID = rd.GetInt64(3);
// Guard against multiple downloads of the same file
if (!m_proccessedVolumes.ContainsKey(volumeID))
{
m_proccessedVolumes.Add(volumeID, volumeID);
yield return new RemoteVolume(
rd.GetString(0),
rd.ConvertValueToString(1),
@@ -675,56 +677,58 @@ namespace Duplicati.Library.Main.Database
}
}
}
}
}
public void CleanupMissingVolumes()
{
var tablename = "SwapBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
public void CleanupMissingVolumes()
{
var tablename = "SwapBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
// TODO: either hardcode all string constants or none
// The first part of this query swaps out blocks for non-present remote files with
// existing ones (as recorded in the DuplicateBlock table)
// existing ones (as recorded in the DuplicateBlock table)
// The second part removes references to the non-present remote files,
// and marks the index files that pointed to them, such that they will be removed later on
var sql = $@"
var sql = $@"
CREATE TEMPORARY TABLE ""{tablename}"" AS
SELECT ""A"".""ID"" AS ""BlockID"", ""A"".""VolumeID"" AS ""SourceVolumeID"", ""A"".""State"" AS ""SourceVolumeState"", ""B"".""VolumeID"" AS ""TargetVolumeID"", ""B"".""State"" AS ""TargetVolumeState"" FROM (SELECT ""Block"".""ID"", ""Block"".""VolumeID"", ""Remotevolume"".""State"" FROM ""Block"", ""Remotevolume"" WHERE ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" and ""Remotevolume"".""State"" = ""{RemoteVolumeState.Temporary}"") A, (SELECT ""DuplicateBlock"".""BlockID"", ""DuplicateBlock"".""VolumeID"", ""Remotevolume"".""State"" FROM ""DuplicateBlock"", ""Remotevolume"" WHERE ""DuplicateBlock"".""VolumeID"" = ""Remotevolume"".""ID"" and ""Remotevolume"".""State"" = ""{RemoteVolumeState.Verified}"") B WHERE ""A"".""ID"" = ""B"".""BlockID"";
SELECT ""A"".""ID"" AS ""BlockID"", ""A"".""VolumeID"" AS ""SourceVolumeID"", ""A"".""State"" AS ""SourceVolumeState"", ""B"".""VolumeID"" AS ""TargetVolumeID"", ""B"".""State"" AS ""TargetVolumeState"" FROM (SELECT ""Block"".""ID"", ""Block"".""VolumeID"", ""Remotevolume"".""State"" FROM ""Block"", ""Remotevolume"" WHERE ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" and ""Remotevolume"".""State"" = ""{RemoteVolumeState.Temporary}"") A, (SELECT ""DuplicateBlock"".""BlockID"", ""DuplicateBlock"".""VolumeID"", ""Remotevolume"".""State"" FROM ""DuplicateBlock"", ""Remotevolume"" WHERE ""DuplicateBlock"".""VolumeID"" = ""Remotevolume"".""ID"" and ""Remotevolume"".""State"" = '{RemoteVolumeState.Verified}') B WHERE ""A"".""ID"" = ""B"".""BlockID"";
UPDATE ""Block"" SET ""VolumeID"" = (SELECT ""TargetVolumeID"" FROM ""{tablename}"" WHERE ""Block"".""ID"" = ""{tablename}"".""BlockID"") WHERE ""Block"".""ID"" IN (SELECT ""BlockID"" FROM ""{tablename}"");
UPDATE ""DuplicateBlock"" SET ""VolumeID"" = (SELECT ""SourceVolumeID"" FROM ""{tablename}"" WHERE ""DuplicateBlock"".""BlockID"" = ""{tablename}"".""BlockID"") WHERE ""DuplicateBlock"".""BlockID"" IN (SELECT ""BlockID"" FROM ""{tablename}"");
DROP TABLE ""{tablename}"";
DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = ""{RemoteVolumeType.Blocks}"" AND ""State"" = ""{RemoteVolumeState.Temporary}"" AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block""));
DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = ""Blocks"" AND ""State"" = ""{RemoteVolumeState.Temporary}"" AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block""));
DELETE FROM ""RemoteVolume"" WHERE ""Type"" = ""{RemoteVolumeType.Blocks}"" AND ""State"" = ""{RemoteVolumeState.Temporary}"" AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"");
DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND ""State"" = '{RemoteVolumeState.Temporary}' AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block""));
DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND ""State"" = '{RemoteVolumeState.Temporary}' AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block""));
DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND ""State"" = '{RemoteVolumeState.Temporary}' AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"");
";
// We could delete these, but we don't have to, so we keep them around until the next compact is done
// UPDATE ""RemoteVolume"" SET ""State"" = ""{3}"" WHERE ""Type"" = ""{5}"" AND ""ID"" NOT IN (SELECT ""IndexVolumeID"" FROM ""IndexBlockLink"");
// We could delete these, but we don't have to, so we keep them around until the next compact is done
// UPDATE ""RemoteVolume"" SET ""State"" = ""{3}"" WHERE ""Type"" = ""{5}"" AND ""ID"" NOT IN (SELECT ""IndexVolumeID"" FROM ""IndexBlockLink"");
var countsql = @"SELECT COUNT(*) FROM ""RemoteVolume"" WHERE ""State"" = ""Temporary"" AND ""Type"" = ""Blocks"" ";
var countsql = $@"SELECT COUNT(*) FROM ""RemoteVolume"" WHERE ""State"" = '{RemoteVolumeState.Temporary}' AND ""Type"" = '{RemoteVolumeType.Blocks}' ";
using (var cmd = m_connection.CreateCommand())
{
var cnt = cmd.ExecuteScalarInt64(countsql);
if (cnt > 0)
{
Logging.Log.WriteWarningMessage(LOGTAG, "MissingVolumesDetected", null, "Found {0} missing volumes; attempting to replace blocks from existing volumes", cnt);
cmd.ExecuteNonQuery(sql);
var cnt2 = cmd.ExecuteScalarInt64(countsql);
Logging.Log.WriteVerboseMessage(LOGTAG, "ReplacedMissingVolumes", "Replaced blocks for {0} missing volumes; there are now {1} missing volumes", cnt, cnt2);
}
}
}
public override void Dispose()
{
using (var cmd = m_connection.CreateCommand())
{
{
var cnt = cmd.ExecuteScalarInt64(countsql);
if (cnt > 0)
{
Logging.Log.WriteWarningMessage(LOGTAG, "MissingVolumesDetected", null, "Found {0} missing volumes; attempting to replace blocks from existing volumes", cnt);
cmd.ExecuteNonQuery(sql);
var cnt2 = cmd.ExecuteScalarInt64(countsql);
Logging.Log.WriteVerboseMessage(LOGTAG, "ReplacedMissingVolumes", "Replaced blocks for {0} missing volumes; there are now {1} missing volumes", cnt, cnt2);
}
}
}
public override void Dispose()
{
using (var cmd = m_connection.CreateCommand())
{
if (m_tempblocklist != null)
try
{
@@ -733,7 +737,7 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = ""{RemoteVolumeType.Blocks}"" AND
}
catch { }
finally { m_tempblocklist = null; }
if (m_tempsmalllist != null)
try
{