haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add more broken file detection:

Browse Source 
Blocksets with either blockset entires or blocklist entries that refer to invalid / unreachable blocks are now treated as broken as well.
This commit is contained in:
Tyler Gill
2018-05-21 10:16:51 -06:00
parent b5931b2771
commit bbdf4c2747
1 changed files with 18 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs
+18 -5
View File
@@ -22,6 +22,13 @@ namespace Duplicati.Library.Main.Database
{
internal class LocalListBrokenFilesDatabase : LocalDatabase
{
private const string BLOCK_VOLUME_IDS = @"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = ""{2}""";
// Invalid blocksets include those that:
// - Have BlocksetEntries with unknown/invalid blocks (meaning the data to rebuild the blockset isn't available)
// - Invalid blocks include those that appear to be in non-Blocks volumes (e.g., are listed as being in an Index or Files volume) or that appear in an unknown volume (-1)
// - Have BlocklistHash entries with unknown/invalid blocks (meaning the data which defines the list of hashes that makes up the blockset isn't available)
// - Are defined in the Blockset table but have no entries in the BlocksetEntries table (this can happen during recreate if Files volumes reference blocksets that are not found in any Index files)
private const string BROKEN_FILE_IDS = @"
SELECT DISTINCT ""ID"" FROM (
SELECT ""ID"" AS ""ID"", ""BlocksetID"" AS ""BlocksetID"" FROM ""File"" WHERE ""BlocksetID"" != {0} AND ""BlocksetID"" != {1}
@@ -31,7 +38,13 @@ UNION
WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN
(
SELECT DISTINCT ""BlocksetID"" FROM (
SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" NOT IN (SELECT ""ID"" FROM ""Block"")
SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" NOT IN
(SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN
(" + BLOCK_VOLUME_IDS + @"))
UNION
SELECT ""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Hash"" NOT IN
(SELECT ""Hash"" FROM ""Block"" WHERE ""VolumeID"" IN
(" + BLOCK_VOLUME_IDS + @"))
UNION
SELECT ""A"".""ID"" AS ""BlocksetID"" FROM ""Blockset"" A LEFT JOIN ""BlocksetEntry"" B ON ""A"".""ID"" = ""B"".""BlocksetID"" WHERE ""B"".""BlocksetID"" IS NULL
)
@@ -44,7 +57,7 @@ SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A, ""Blockset"" B WHERE ""
+ BROKEN_FILE_IDS +
@") AND ""A"".""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ?)";
private const string INSERT_BROKEN_IDS = @"INSERT INTO ""{2}"" (""{3}"") " + BROKEN_FILE_IDS
private const string INSERT_BROKEN_IDS = @"INSERT INTO ""{3}"" (""{4}"") " + BROKEN_FILE_IDS
+ @" AND ""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = ?)";
public LocalListBrokenFilesDatabase(string path)
@@ -61,7 +74,7 @@ SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A, ""Blockset"" B WHERE ""
public IEnumerable<Tuple<DateTime, long, long>> GetBrokenFilesets(DateTime time, long[] versions, System.Data.IDbTransaction transaction)
{
var query = string.Format(BROKEN_FILE_SETS, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID);
var query = string.Format(BROKEN_FILE_SETS, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, RemoteVolumeType.Blocks.ToString());
var clause = GetFilelistWhereClause(time, versions);
if (!string.IsNullOrWhiteSpace(clause.Item1))
query += @" AND ""A"".""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" " + clause.Item1 + ")";
@@ -77,7 +90,7 @@ SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A, ""Blockset"" B WHERE ""
public IEnumerable<Tuple<string, long>> GetBrokenFilenames(long filesetid, System.Data.IDbTransaction transaction)
{
using (var cmd = Connection.CreateCommand(transaction))
foreach (var rd in cmd.ExecuteReaderEnumerable(string.Format(BROKEN_FILE_NAMES, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID), filesetid))
foreach (var rd in cmd.ExecuteReaderEnumerable(string.Format(BROKEN_FILE_NAMES, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, RemoteVolumeType.Blocks.ToString()), filesetid))
if (!rd.IsDBNull(0))
yield return new Tuple<string, long>(rd.ConvertValueToString(0), rd.ConvertValueToInt64(1));
}
@@ -85,7 +98,7 @@ SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A, ""Blockset"" B WHERE ""
public void InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname, System.Data.IDbTransaction transaction)
{
using (var cmd = Connection.CreateCommand(transaction))
cmd.ExecuteNonQuery(string.Format(INSERT_BROKEN_IDS, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, tablename, IDfieldname), filesetid);
cmd.ExecuteNonQuery(string.Format(INSERT_BROKEN_IDS, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, RemoteVolumeType.Blocks.ToString(), tablename, IDfieldname), filesetid);
}
public void RemoveMissingBlocks(IEnumerable<string> names, System.Data.IDbTransaction transaction)