haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #6237 from duplicati/feature/issue-5987

Browse Source 
Rewrote the repair for scenarios where remote files are missing.
This commit is contained in:
Kenneth Skovhede
2025-05-12 11:07:08 +02:00
committed by GitHub
parent 230d1239af b6978afb3a
commit 2f0e9dc6f1
19 changed files with 2124 additions and 515 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Interface/ResultInterfaces.cs
+1
View File
@@ -494,6 +494,7 @@ namespace Duplicati.Library.Interface
{
long RemovedFileCount { get; }
long RemovedFileSize { get; }
long UpdatedFileCount { get; }
long RewrittenFileLists { get; }
ICompactResults CompactResults { get; }
}
Duplicati/Library/Main/Database/LocalDatabase.cs
+17 -10
View File
@@ -469,17 +469,13 @@ namespace Duplicati.Library.Main.Database
// Create and fill a temp table with the volids to delete. We avoid using too many parameters that way.
deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{volidstable}"" (""ID"" INTEGER PRIMARY KEY)"));
deletecmd.SetCommandAndParameters(FormatInvariant($@"INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") VALUES (@Id)"));
foreach (var name in names)
{
var volumeid = GetRemoteVolumeID(name, tr.Parent);
deletecmd.SetParameterValue("@Id", volumeid)
.ExecuteNonQuery();
}
deletecmd.SetCommandAndParameters(FormatInvariant($@"INSERT OR IGNORE INTO ""{volidstable}"" SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@VolumeNames)"))
.ExpandInClauseParameter("@VolumeNames", names.ToArray())
.ExecuteNonQuery();
var volIdsSubQuery = FormatInvariant($@"SELECT ""ID"" FROM ""{volidstable}"" ");
deletecmd.Parameters.Clear();
var bsIdsSubQuery = FormatInvariant(@$"
SELECT DISTINCT ""BlocksetEntry"".""BlocksetID"" FROM ""BlocksetEntry"", ""Block""
WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" IN ({volIdsSubQuery})
@@ -661,9 +657,9 @@ AND Fileset.ID NOT IN
return res;
}
public bool IsFilesetFullBackup(DateTime filesetTime)
public bool IsFilesetFullBackup(DateTime filesetTime, IDbTransaction? transaction)
{
using (var cmd = m_connection.CreateCommand())
using (var cmd = m_connection.CreateCommand(transaction))
using (var rd = cmd.SetCommandAndParameters($@"SELECT ""IsFullBackup"" FROM ""Fileset"" WHERE ""Timestamp"" = @Timestamp").SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(filesetTime)).ExecuteReader())
{
if (!rd.Read())
@@ -1504,8 +1500,19 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER
}
}
/// <summary>
/// Adds a link between an index volume and a block volume.
/// </summary>
/// <param name="indexVolumeID">The ID of the index volume.</param>
/// <param name="blockVolumeID">The ID of the block volume.</param>
/// <param name="transaction">An optional transaction.</param>
public void AddIndexBlockLink(long indexVolumeID, long blockVolumeID, IDbTransaction transaction)
{
if (indexVolumeID <= 0)
throw new ArgumentOutOfRangeException(nameof(indexVolumeID), "Index volume ID must be greater than 0.");
if (blockVolumeID <= 0)
throw new ArgumentOutOfRangeException(nameof(blockVolumeID), "Block volume ID must be greater than 0.");
m_insertIndexBlockLink.SetParameterValue("@IndexVolumeId", indexVolumeID)
.SetParameterValue("@BlockVolumeId", blockVolumeID)
.ExecuteNonQuery(transaction);
Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs
+187 -98
View File
@@ -28,17 +28,17 @@ using System.Linq;
namespace Duplicati.Library.Main.Database
{
internal class LocalListBrokenFilesDatabase : LocalDatabase
{
private static readonly string BLOCK_VOLUME_IDS = FormatInvariant($@"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks.ToString()}'");
internal class LocalListBrokenFilesDatabase : LocalDatabase
{
private static readonly string BLOCK_VOLUME_IDS = FormatInvariant($@"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks.ToString()}'");
// Invalid blocksets include those that:
// - Have BlocksetEntries with unknown/invalid blocks (meaning the data to rebuild the blockset isn't available)
// - Invalid blocks include those that appear to be in non-Blocks volumes (e.g., are listed as being in an Index or Files volume) or that appear in an unknown volume (-1)
// - Have BlocklistHash entries with unknown/invalid blocks (meaning the data which defines the list of hashes that makes up the blockset isn't available)
// - Are defined in the Blockset table but have no entries in the BlocksetEntries table (this can happen during recreate if Files volumes reference blocksets that are not found in any Index files)
// However, blocksets with a length of 0 are excluded from this check, as the corresponding blocks for these are not needed.
private static readonly string BROKEN_FILE_IDS = FormatInvariant($@"
// Invalid blocksets include those that:
// - Have BlocksetEntries with unknown/invalid blocks (meaning the data to rebuild the blockset isn't available)
// - Invalid blocks include those that appear to be in non-Blocks volumes (e.g., are listed as being in an Index or Files volume) or that appear in an unknown volume (-1)
// - Have BlocklistHash entries with unknown/invalid blocks (meaning the data which defines the list of hashes that makes up the blockset isn't available)
// - Are defined in the Blockset table but have no entries in the BlocksetEntries table (this can happen during recreate if Files volumes reference blocksets that are not found in any Index files)
// However, blocksets with a length of 0 are excluded from this check, as the corresponding blocks for these are not needed.
private static readonly string BROKEN_FILE_IDS = FormatInvariant($@"
SELECT DISTINCT ""ID"" FROM (
SELECT ""ID"" AS ""ID"", ""BlocksetID"" AS ""BlocksetID"" FROM ""FileLookup"" WHERE ""BlocksetID"" != {FOLDER_BLOCKSET_ID} AND ""BlocksetID"" != {SYMLINK_BLOCKSET_ID}
UNION
@@ -61,99 +61,188 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN
WHERE ""BlocksetID"" NOT IN (SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" == 0)
)
");
private static readonly string BROKEN_FILE_SETS = FormatInvariant($@"SELECT DISTINCT ""B"".""Timestamp"", ""A"".""FilesetID"", COUNT(""A"".""FileID"") AS ""FileCount"" FROM ""FilesetEntry"" A, ""Fileset"" B WHERE ""A"".""FilesetID"" = ""B"".""ID"" AND ""A"".""FileID"" IN ({BROKEN_FILE_IDS})");
private static readonly string BROKEN_FILE_SETS = FormatInvariant($@"SELECT DISTINCT ""B"".""Timestamp"", ""A"".""FilesetID"", COUNT(""A"".""FileID"") AS ""FileCount"" FROM ""FilesetEntry"" A, ""Fileset"" B WHERE ""A"".""FilesetID"" = ""B"".""ID"" AND ""A"".""FileID"" IN ({BROKEN_FILE_IDS})");
private static readonly string BROKEN_FILE_NAMES = FormatInvariant($@"SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A LEFT JOIN ""Blockset"" B ON (""A"".""BlocksetID"" = ""B"".""ID"") WHERE ""A"".""ID"" IN ({BROKEN_FILE_IDS}) AND ""A"".""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId)");
private static readonly string BROKEN_FILE_NAMES = FormatInvariant($@"SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A LEFT JOIN ""Blockset"" B ON (""A"".""BlocksetID"" = ""B"".""ID"") WHERE ""A"".""ID"" IN ({BROKEN_FILE_IDS}) AND ""A"".""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId)");
private static string INSERT_BROKEN_IDS(string tablename, string IDfieldname) => FormatInvariant($@"INSERT INTO ""{tablename}"" (""{IDfieldname}"") {BROKEN_FILE_IDS} AND ""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId)");
private static string INSERT_BROKEN_IDS(string tablename, string IDfieldname) => FormatInvariant($@"INSERT INTO ""{tablename}"" (""{IDfieldname}"") {BROKEN_FILE_IDS} AND ""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId)");
public LocalListBrokenFilesDatabase(string path, long pagecachesize)
: base(path, "ListBrokenFiles", false, pagecachesize)
{
ShouldCloseConnection = true;
}
public LocalListBrokenFilesDatabase(LocalDatabase parent)
: base(parent)
{
ShouldCloseConnection = false;
}
public IEnumerable<Tuple<DateTime, long, long>> GetBrokenFilesets(DateTime time, long[] versions, IDbTransaction transaction)
{
var query = BROKEN_FILE_SETS;
var clause = GetFilelistWhereClause(time, versions);
if (!string.IsNullOrWhiteSpace(clause.Item1))
query += FormatInvariant($@" AND ""A"".""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" {clause.Item1})");
query += @" GROUP BY ""A"".""FilesetID""";
using (var cmd = Connection.CreateCommand(transaction))
foreach (var rd in cmd.SetCommandAndParameters(query).SetParameterValues(clause.Values).ExecuteReaderEnumerable())
if (!rd.IsDBNull(0))
yield return new Tuple<DateTime, long, long>(ParseFromEpochSeconds(rd.ConvertValueToInt64(0, 0)), rd.ConvertValueToInt64(1, -1), rd.ConvertValueToInt64(2, 0));
}
public IEnumerable<Tuple<string, long>> GetBrokenFilenames(long filesetid, IDbTransaction transaction)
{
using (var cmd = Connection.CreateCommand(transaction, BROKEN_FILE_NAMES).SetParameterValue("@FilesetId", filesetid))
foreach (var rd in cmd.ExecuteReaderEnumerable())
if (!rd.IsDBNull(0))
yield return new Tuple<string, long>(rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), rd.ConvertValueToInt64(1));
}
public void InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname, IDbTransaction transaction)
{
using var cmd = Connection.CreateCommand(transaction, INSERT_BROKEN_IDS(tablename, IDfieldname))
.SetParameterValue("@FilesetId", filesetid);
cmd.ExecuteNonQuery();
}
public void RemoveMissingBlocks(IEnumerable<string> names, IDbTransaction transaction)
{
if (names == null || !names.Any()) return;
if (transaction == null)
throw new Exception("This function cannot be called when not in a transaction, as it leaves the database in an inconsistent state");
using (var deletecmd = m_connection.CreateCommand(transaction))
{
var temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
var volidstable = "DelVolSetIds-" + temptransguid;
// Create and fill a temp table with the volids to delete. We avoid using too many parameters that way.
deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{volidstable}"" (""ID"" INTEGER PRIMARY KEY)"));
using (var tmptable = new TemporaryDbValueList(m_connection, transaction, names))
deletecmd.SetCommandAndParameters(FormatInvariant($@"INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)"))
.ExpandInClauseParameter("@Names", tmptable)
.ExecuteNonQuery();
var volIdsSubQuery = FormatInvariant($@"SELECT ""ID"" FROM ""{volidstable}"" ");
deletecmd.Parameters.Clear();
deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery})"));
deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery})"));
deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})"));
deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})"));
// Clean up temp tables for subqueries. We truncate content and then try to delete.
// Drop in try-block, as it fails in nested transactions (SQLite problem)
// SQLite.SQLiteException (0x80004005): database table is locked
deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{volidstable}"" "));
try
public LocalListBrokenFilesDatabase(string path, long pagecachesize)
: base(path, "ListBrokenFiles", false, pagecachesize)
{
deletecmd.CommandTimeout = 2;
deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{volidstable}"" "));
ShouldCloseConnection = true;
}
catch { /* Ignore, will be deleted on close anyway. */ }
}
}
public long GetFilesetFileCount(long filesetid, IDbTransaction transaction)
{
using var cmd = m_connection.CreateCommand(transaction, @"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId")
.SetParameterValue("@FilesetId", filesetid);
return cmd.ExecuteScalarInt64(0);
public LocalListBrokenFilesDatabase(LocalDatabase parent)
: base(parent)
{
ShouldCloseConnection = false;
}
public IEnumerable<(DateTime FilesetTime, long FilesetID, long RemoveFileCount)> GetBrokenFilesets(DateTime time, long[] versions, IDbTransaction transaction)
{
var query = BROKEN_FILE_SETS;
var clause = GetFilelistWhereClause(time, versions);
if (!string.IsNullOrWhiteSpace(clause.Item1))
query += FormatInvariant($@" AND ""A"".""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" {clause.Item1})");
query += @" GROUP BY ""A"".""FilesetID""";
using (var cmd = Connection.CreateCommand(transaction))
foreach (var rd in cmd.SetCommandAndParameters(query).SetParameterValues(clause.Values).ExecuteReaderEnumerable())
if (!rd.IsDBNull(0))
yield return (ParseFromEpochSeconds(rd.ConvertValueToInt64(0, 0)), rd.ConvertValueToInt64(1, -1), rd.ConvertValueToInt64(2, 0));
}
public IEnumerable<Tuple<string, long>> GetBrokenFilenames(long filesetid, IDbTransaction transaction)
{
using (var cmd = Connection.CreateCommand(transaction, BROKEN_FILE_NAMES).SetParameterValue("@FilesetId", filesetid))
foreach (var rd in cmd.ExecuteReaderEnumerable())
if (!rd.IsDBNull(0))
yield return new Tuple<string, long>(rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), rd.ConvertValueToInt64(1));
}
/// <summary>
/// Returns all index files that are orphaned, i.e., not referenced by any block files.
/// </summary>
/// <param name="transaction">Transaction to use for the query.</param>
/// <returns>>All index files that are orphaned.</returns>
public IEnumerable<RemoteVolume> GetOrphanedIndexFiles(IDbTransaction transaction)
{
using var cmd = Connection.CreateCommand(transaction, FormatInvariant($@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Index.ToString()}' AND ""ID"" NOT IN (SELECT ""IndexVolumeID"" FROM ""IndexBlockLink"")"));
foreach (var rd in cmd.ExecuteReaderEnumerable())
yield return new RemoteVolume(
rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"),
rd.ConvertValueToString(1) ?? throw new Exception("Hash was null"),
rd.ConvertValueToInt64(2, -1)
);
}
/// <summary>
/// Inserts the broken file IDs into the given table. The table must have a single column with the same name as the ID field name.
/// </summary>
/// <param name="filesetid">The filset id for the current operation</param>
/// <param name="tablename">The name of the table to insert into</param>
/// <param name="IDfieldname">The name of the ID field in the table</param>
/// <param name="transaction">The transaction to use for the query</param>
public void InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname, IDbTransaction transaction)
{
using var cmd = Connection.CreateCommand(transaction, INSERT_BROKEN_IDS(tablename, IDfieldname))
.SetParameterValue("@FilesetId", filesetid);
cmd.ExecuteNonQuery();
}
/// <summary>
/// Returns the ID of an empty metadata blockset. If no empty blockset is found, it returns the ID of the smallest blockset that is not in the given block volume IDs.
/// If no such blockset is found, it returns -1.
/// </summary>
/// <param name="blockVolumeIds">The volume ids to ignore when searching for a suitable metadata block</param>
/// <param name="emptyHash">The hash of the empty blockset</param>
/// <param name="emptyHashSize">The size of the empty blockset</param>
/// <param name="transaction">The transaction to use for the query</param>
/// <returns>The ID of the empty metadata blockset, or -1 if no suitable blockset is found</returns>
public long GetEmptyMetadataBlocksetId(IEnumerable<long> blockVolumeIds, string emptyHash, long emptyHashSize, IDbTransaction transaction)
{
using var cmd = Connection.CreateCommand(transaction, @"SELECT ""ID"" FROM ""Blockset"" WHERE ""FullHash"" = @EmptyHash AND ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))")
.ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds)
.SetParameterValue("@EmptyHash", emptyHash)
.SetParameterValue("@EmptyHashSize", emptyHashSize);
var res = cmd.ExecuteScalarInt64(-1);
// No empty block found, try to find a zero-length block instead
if (res < 0 && emptyHashSize != 0)
res = cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))")
.ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds)
.SetParameterValue("@EmptyHashSize", 0)
.ExecuteScalarInt64(-1);
// No empty block found, pick the smallest one
if (res < 0)
res = cmd.SetCommandAndParameters(@"SELECT ""Blockset"".""ID"" FROM ""BlocksetEntry"", ""Blockset"", ""Metadataset"", ""Block"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) ORDER BY ""Blockset"".""Length"" ASC LIMIT 1")
.ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds)
.ExecuteScalarInt64(-1);
return res;
}
/// <summary>
/// Replaces the metadata blockset ID in the Metadataset table with the empty blockset ID for all fileset entries that are not in any block volume.
/// This is used to clean up the metadata blocksets that are now missing.
/// </summary>
/// <param name="filesetId">The filesetId to target</param>
/// <param name="emptyBlocksetId">The empty blockset ID to replace with</param>
/// <param name="transaction">The transaction to use for the query</param>
/// <returns>The number of rows affected</returns>
public int ReplaceMetadata(long filesetId, long emptyBlocksetId, IDbTransaction transaction)
{
using var cmd = m_connection.CreateCommand(transaction, @"
UPDATE ""Metadataset""
SET ""BlocksetID"" = @EmptyBlocksetID
WHERE
""ID"" IN (
SELECT ""FileLookup"".""MetadataID""
FROM ""FileLookup"", ""FilesetEntry""
WHERE
""FilesetEntry"".""FilesetId"" = @FilesetId
AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID""
)
AND ""BlocksetID"" NOT IN (
SELECT ""BlocksetID""
FROM ""BlocksetEntry"", ""Block""
WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID""
)")
.SetParameterValue("@EmptyBlocksetId", emptyBlocksetId)
.SetParameterValue("@FilesetId", filesetId);
return cmd.ExecuteNonQuery();
}
public void RemoveMissingBlocks(IEnumerable<string> names, IDbTransaction transaction)
{
if (names == null || !names.Any()) return;
if (transaction == null)
throw new Exception("This function cannot be called when not in a transaction, as it leaves the database in an inconsistent state");
using (var deletecmd = m_connection.CreateCommand(transaction))
{
var temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
var volidstable = "DelVolSetIds-" + temptransguid;
// Create and fill a temp table with the volids to delete. We avoid using too many parameters that way.
deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{volidstable}"" (""ID"" INTEGER PRIMARY KEY)"));
using (var tmptable = new TemporaryDbValueList(m_connection, transaction, names))
deletecmd.SetCommandAndParameters(FormatInvariant($@"INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)"))
.ExpandInClauseParameter("@Names", tmptable)
.ExecuteNonQuery();
var volIdsSubQuery = FormatInvariant($@"SELECT ""ID"" FROM ""{volidstable}"" ");
deletecmd.Parameters.Clear();
deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery})"));
deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery})"));
deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})"));
deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})"));
// Clean up temp tables for subqueries. We truncate content and then try to delete.
// Drop in try-block, as it fails in nested transactions (SQLite problem)
// SQLite.SQLiteException (0x80004005): database table is locked
deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{volidstable}"" "));
try
{
deletecmd.CommandTimeout = 2;
deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{volidstable}"" "));
}
catch { /* Ignore, will be deleted on close anyway. */ }
}
}
public long GetFilesetFileCount(long filesetid, IDbTransaction transaction)
{
using var cmd = m_connection.CreateCommand(transaction, @"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId")
.SetParameterValue("@FilesetId", filesetid);
return cmd.ExecuteScalarInt64(0);
}
}
}
}
Duplicati/Library/Main/Database/LocalListChangesDatabase.cs
+1 -1
View File
@@ -204,7 +204,7 @@ namespace Duplicati.Library.Main.Database
var cmd = asNew ? m_insertCurrentElementCommand : m_insertPreviousElementCommand;
cmd.SetParameterValue("@Path", path);
cmd.SetParameterValue("@FileHash", filehash);
cmd.SetParameterValue(@"MetaHash", metahash);
cmd.SetParameterValue("@MetaHash", metahash);
cmd.SetParameterValue("@Size", size);
cmd.SetParameterValue("@Type", (int)type);
cmd.ExecuteNonQuery();
Duplicati/Library/Main/Database/LocalPurgeDatabase.cs
+19 -7
View File
@@ -72,9 +72,10 @@ namespace Duplicati.Library.Main.Database
long ParentID { get; }
long RemovedFileCount { get; }
long RemovedFileSize { get; }
long UpdatedFileCount { get; }
void ApplyFilter(Library.Utility.IFilter filter);
void ApplyFilter(Action<IDbCommand, long, string> filtercommand);
void ApplyFilter(Func<IDbCommand, long, string, int> filtercommand);
Tuple<long, long> ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup);
IEnumerable<KeyValuePair<string, long>> ListAllDeletedFiles();
}
@@ -89,6 +90,7 @@ namespace Duplicati.Library.Main.Database
public long ParentID { get; private set; }
public long RemovedFileCount { get; private set; }
public long RemovedFileSize { get; private set; }
public long UpdatedFileCount { get; private set; }
public TemporaryFileset(long parentid, LocalPurgeDatabase parentdb, IDbConnection connection, IDbTransaction transaction)
{
@@ -102,12 +104,13 @@ namespace Duplicati.Library.Main.Database
cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tablename}"" (""FileID"" INTEGER PRIMARY KEY) "));
}
public void ApplyFilter(Action<IDbCommand, long, string> filtercommand)
public void ApplyFilter(Func<IDbCommand, long, string, int> filtercommand)
{
int updated;
using (var cmd = m_connection.CreateCommand(m_transaction))
filtercommand(cmd, ParentID, m_tablename);
updated = filtercommand(cmd, ParentID, m_tablename);
PostFilterChecks();
PostFilterChecks(updated);
}
public void ApplyFilter(Library.Utility.IFilter filter)
@@ -156,15 +159,24 @@ namespace Duplicati.Library.Main.Database
}
}
PostFilterChecks();
PostFilterChecks(0);
}
private void PostFilterChecks()
private void PostFilterChecks(int updated)
{
using (var cmd = m_connection.CreateCommand(m_transaction))
{
UpdatedFileCount = updated;
RemovedFileCount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""{m_tablename}"""), 0);
RemovedFileSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""C"".""Length"") FROM ""{m_tablename}"" A, ""FileLookup"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" "), 0);
RemovedFileSize = cmd.ExecuteScalarInt64(FormatInvariant($@"
SELECT
SUM(""C"".""Length"")
FROM
""{m_tablename}"" A, ""FileLookup"" B, ""Blockset"" C, ""Metadataset"" D
WHERE
""A"".""FileID"" = ""B"".""ID""
AND(""B"".""BlocksetID"" = ""C"".""ID"" OR(""B"".""MetadataID"" = ""D"".""ID"" AND ""D"".""BlocksetID"" = ""C"".""ID""))
"), 0);
var filesetcount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = {ParentID}"), 0);
if (filesetcount == RemovedFileCount)
throw new Interface.UserInformationException($"Refusing to purge {RemovedFileCount} files from fileset with ID {ParentID}, as that would remove the entire fileset.\nTo delete a fileset, use the \"delete\" command.", "PurgeWouldRemoveEntireFileset");
Duplicati/Library/Main/Database/LocalRepairDatabase.cs
+408 -120
View File
@@ -35,27 +35,87 @@ namespace Duplicati.Library.Main.Database
/// </summary>
private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalRepairDatabase));
/// <summary>
/// Creates a new local repair database
/// </summary>
/// <param name="path">The path to the database</param>
/// <param name="pagecachesize">The page cache size</param>
public LocalRepairDatabase(string path, long pagecachesize)
: base(path, "Repair", true, pagecachesize)
{
}
public long GetFilesetIdFromRemotename(string filelist)
/// <summary>
/// Gets the fileset ID from the remote name
/// </summary>
/// <param name="filelist">The remote name of the fileset</param>
/// <param name="transaction">An optional transaction</param>
/// <returns>>The fileset ID</returns>
public (long FilesetId, DateTime Time, bool IsFullBackup) GetFilesetFromRemotename(string filelist, IDbTransaction? transaction)
{
using (var cmd = m_connection.CreateCommand())
using (var cmd = m_connection.CreateCommand(transaction))
{
var filesetid = cmd.SetCommandAndParameters(@"SELECT ""Fileset"".""ID"" FROM ""Fileset"",""RemoteVolume"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""RemoteVolume"".""Name"" = @Name")
var rd = cmd.SetCommandAndParameters(@"SELECT ""Fileset"".""ID"", ""Fileset"".""Timestamp"", ""Fileset"".""IsFullBackup"" FROM ""Fileset"",""RemoteVolume"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""RemoteVolume"".""Name"" = @Name")
.SetParameterValue("@Name", filelist)
.ExecuteScalarInt64(-1);
.ExecuteReader();
if (filesetid == -1)
if (!rd.Read())
throw new Exception($"No such remote file: {filelist}");
return filesetid;
return (rd.ConvertValueToInt64(0, -1), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime(), rd.GetInt32(2) == BackupType.FULL_BACKUP);
}
}
/// <summary>
/// Moves entries in the FilesetEntry table from previous fileset to current fileset
/// </summary>
/// <param name="filesetid">Current fileset ID</param>
/// <param name="prevFilesetId">Source fileset ID</param>
/// <param name="transaction">The transaction</param>
public void MoveFilesFromFileset(long filesetid, long prevFilesetId, IDbTransaction transaction)
{
if (filesetid <= 0)
throw new ArgumentException("filesetid must be > 0");
if (prevFilesetId <= 0)
throw new ArgumentException("prevId must be > 0");
using var cmd = m_connection.CreateCommand(transaction);
cmd.SetCommandAndParameters(@"UPDATE ""FilesetEntry"" SET ""FilesetID"" = @CurrentFilesetId WHERE ""FilesetID"" = @PreviousFilesetId ")
.SetParameterValue("@CurrentFilesetId", filesetid)
.SetParameterValue("@PreviousFilesetId", prevFilesetId)
.ExecuteNonQuery();
}
/// <summary>
/// Deletes a fileset from the database
/// </summary>
/// <param name="filesetid">The fileset ID</param>
/// <param name="transaction">The transaction</param>
public void DeleteFileset(long filesetid, IDbTransaction transaction)
{
using var cmd = m_connection.CreateCommand(transaction);
cmd.SetCommandAndParameters(@"DELETE FROM ""Fileset"" WHERE ""ID"" = @FilesetId")
.SetParameterValue("@FilesetId", filesetid)
.ExecuteNonQuery();
}
/// <summary>
/// Gets the list of index files that reference a given block file
/// </summary>
/// <param name="blockfileid">The block file ID</param>
/// <returns>>A list of index file IDs</returns>
public IEnumerable<string> GetIndexFilesReferencingBlockFile(long blockfileid, IDbTransaction? transaction)
{
using var cmd = m_connection.CreateCommand(transaction, @"SELECT ""RemoteVolume"".""Name"" FROM ""RemoteVolume"", ""IndexBlockLink"" WHERE ""IndexBlockLink"".""BlockVolumeID"" = @BlockFileId AND ""RemoteVolume"".""ID"" = ""IndexBlockLink"".""IndexVolumeID"" AND ""RemoteVolume"".""Type"" = @Type")
.SetParameterValue("@BlockFileId", blockfileid)
.SetParameterValue("@Type", RemoteVolumeType.Index.ToString());
foreach (var rd in cmd.ExecuteReaderEnumerable())
yield return rd.ConvertValueToString(0) ?? throw new Exception("RemoteVolume name was null");
}
/// <summary>
/// Gets a list of filesets that are missing files
/// </summary>
@@ -88,63 +148,6 @@ namespace Duplicati.Library.Main.Database
.ExecuteNonQuery();
}
public interface IBlockSource
{
string File { get; }
long Offset { get; }
}
public interface IBlockWithSources : IBlock
{
IEnumerable<IBlockSource> Sources { get; }
}
private class BlockWithSources : Block, IBlockWithSources
{
private class BlockSource : IBlockSource
{
public string File { get; private set; }
public long Offset { get; private set; }
public BlockSource(string file, long offset)
{
this.File = file;
this.Offset = offset;
}
}
private readonly IDataReader m_rd;
public bool Done { get; private set; }
public BlockWithSources(IDataReader rd)
: base(rd.ConvertValueToString(0) ?? throw new Exception("Hash value was null"), rd.ConvertValueToInt64(1))
{
m_rd = rd;
Done = !m_rd.Read();
}
public IEnumerable<IBlockSource> Sources
{
get
{
if (Done)
yield break;
var cur = new BlockSource(m_rd.ConvertValueToString(2) ?? "", m_rd.ConvertValueToInt64(3));
var file = cur.File;
while (!Done && cur.File == file)
{
yield return cur;
Done = m_rd.Read();
if (!Done)
cur = new BlockSource(m_rd.ConvertValueToString(2) ?? "", m_rd.ConvertValueToInt64(3));
}
}
}
}
private class RemoteVolume : IRemoteVolume
{
public string Name { get; private set; }
@@ -159,39 +162,152 @@ namespace Duplicati.Library.Main.Database
}
}
public IEnumerable<IRemoteVolume> GetBlockVolumesFromIndexName(string name)
public IEnumerable<IRemoteVolume> GetBlockVolumesFromIndexName(string indexName)
{
using var cmd = m_connection.CreateCommand(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"" WHERE ""IndexVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = @Name)) ")
.SetParameterValue("@Name", name);
.SetParameterValue("@Name", indexName);
foreach (var rd in cmd.ExecuteReaderEnumerable())
yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2));
}
/// <summary>
/// A single block with source data for repair
/// </summary>
/// <param name="Hash">The block hash</param>
/// <param name="Size">The block size</param>
/// <param name="File">The file that contains the block</param>
/// <param name="Offset">The offset of the block in the file</param>
public sealed record BlockWithSourceData(string Hash, long Size, string File, long Offset);
/// <summary>
/// A single block with metadata source data for repair
/// </summary>
/// <param name="Hash">The block hash</param>
/// <param name="Size">The block size</param>
/// <param name="Path">The path of the file or directory that contains the block</param>
public sealed record BlockWithMetadataSourceData(string Hash, long Size, string Path);
/// <summary>
/// A single blocklist hash entry
/// </summary>
/// <param name="BlocksetId">The blockset id</param>
/// <param name="BlocklistHash">The hash of the blocklist entry (when done)</param>
/// <param name="BlocklistHashLength">The total length of the blockset</param>
/// <param name="BlocklistHashIndex">The index of the blocklist hash</param>
/// <param name="Index">The index of the block in the blockset</param>
/// <param name="Hash">The hash of the entry</param>
public sealed record BlocklistHashesEntry(
long BlocksetId,
string BlocklistHash,
long BlocklistHashLength,
long BlocklistHashIndex,
int Index,
string Hash);
/// <summary>
/// Helper interface for the missing block list
/// </summary>
public interface IMissingBlockList : IDisposable
{
bool SetBlockRestored(string hash, long size);
IEnumerable<IBlockWithSources> GetSourceFilesWithBlocks(long blocksize);
IEnumerable<KeyValuePair<string, long>> GetMissingBlocks();
/// <summary>
/// Registers a block as restored
/// </summary>
/// <param name="hash">The block hash</param>
/// <param name="size">The block size</param>
/// <param name="volumeId">The volume ID of the new target volume</param>
/// <returns>The restored blocks</returns>
bool SetBlockRestored(string hash, long size, long volumeId);
/// <summary>
/// Gets the list of files that contains missing blocks
/// </summary>
/// <param name="blocksize">The blocksize setting</param>
/// <returns>A list of files with missing blocks</returns>
IEnumerable<BlockWithSourceData> GetSourceFilesWithBlocks(long blocksize);
/// <summary>
/// Gets a list for filesystem entries that contain missing blocks in metadata
/// </summary>
/// <returns>A list of filesystem entries with missing blocks</returns>
IEnumerable<BlockWithMetadataSourceData> GetSourceItemsWithMetadataBlocks();
/// <summary>
/// Gets missing blocklist hashes
/// </summary>
/// <param name="hashesPerBlock">The number of hashes for each block</param>
/// <returns>>A list of blocklist hashes</returns>
IEnumerable<BlocklistHashesEntry> GetBlocklistHashes(long hashesPerBlock);
/// <summary>
/// Gets the number of missing blocks
/// </summary>
/// <returns>>The number of missing blocks</returns>
long GetMissingBlockCount();
/// <summary>
/// Gets all the filesets that are affected by missing blocks
/// </summary>
/// <returns>>A list of filesets</returns>
IEnumerable<IRemoteVolume> GetFilesetsUsingMissingBlocks();
/// <summary>
/// Gets a list of remote files that may contain missing blocks
/// </summary>
/// <returns>>A list of remote files</returns>
IEnumerable<IRemoteVolume> GetMissingBlockSources();
}
/// <summary>
/// Implementation of the missing block list
/// </summary>
private class MissingBlockList : IMissingBlockList
{
/// <summary>
/// The connection to the database
/// </summary>
private readonly IDbConnection m_connection;
private readonly TemporaryTransactionWrapper m_transaction;
private IDbCommand m_insertCommand;
private string m_tablename;
/// <summary>
/// The transaction to use
/// </summary>
private readonly ReusableTransaction m_transaction;
/// <summary>
/// The insert command to use for restoring blocks
/// </summary>
private readonly IDbCommand m_insertCommand;
/// <summary>
/// The command to copy blocks into the duplicate block table
/// </summary>
private readonly IDbCommand m_copyIntoDuplicatedBlocks;
/// <summary>
/// The command to assign blocks to a new volume
/// </summary>
private readonly IDbCommand m_assignBlocksToNewVolume;
/// <summary>
/// The name of the temporary table
/// </summary>
private readonly string m_tablename;
/// <summary>
/// The name of the volume where blocks are missing
/// </summary>
private readonly string m_volumename;
/// <summary>
/// The query to use for getting missing blocks
/// </summary>
private readonly string m_missingBlocksQuery;
public MissingBlockList(string volumename, IDbConnection connection, IDbTransaction? transaction)
/// <summary>
/// Whether the object has been disposed
/// </summary>
private bool m_isDisposed = false;
/// <summary>
/// Creates a new missing block list
/// </summary>
/// <param name="volumename">The name of the volume with missing blocks</param>
/// <param name="connection">The connection to the database</param>
/// <param name="transaction">The transaction to use</param>
public MissingBlockList(string volumename, IDbConnection connection, ReusableTransaction transaction)
{
m_connection = connection;
m_transaction = new TemporaryTransactionWrapper(m_connection, transaction);
m_transaction = transaction;
m_volumename = volumename;
var tablename = "MissingBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
using (var cmd = m_connection.CreateCommand(m_transaction.Parent))
using (var cmd = m_connection.CreateCommand(m_transaction.Transaction))
{
cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" INTEGER NOT NULL) "));
m_tablename = tablename;
@@ -206,38 +322,209 @@ namespace Duplicati.Library.Main.Database
cmd.ExecuteNonQuery(FormatInvariant($@"CREATE UNIQUE INDEX ""{tablename}-Ix"" ON ""{tablename}"" (""Hash"", ""Size"", ""Restored"")"));
}
m_insertCommand = m_connection.CreateCommand(m_transaction.Parent, FormatInvariant($@"UPDATE ""{tablename}"" SET ""Restored"" = @NewRestoredValue WHERE ""Hash"" = @Hash AND ""Size"" = @Size AND ""Restored"" = @PreviousRestoredValue "));
m_insertCommand = m_connection.CreateCommand(FormatInvariant($@"UPDATE ""{tablename}"" SET ""Restored"" = @NewRestoredValue WHERE ""Hash"" = @Hash AND ""Size"" = @Size AND ""Restored"" = @PreviousRestoredValue "));
m_copyIntoDuplicatedBlocks = m_connection.CreateCommand(@"INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") SELECT ""Block"".""ID"", ""Block"".""VolumeID"" FROM ""Block"" WHERE ""Block"".""Hash"" = @Hash AND ""Block"".""Size"" = @Size");
m_assignBlocksToNewVolume = m_connection.CreateCommand(@"UPDATE ""Block"" SET ""VolumeID"" = @TargetVolumeId WHERE ""Hash"" = @Hash AND ""Size"" = @Size");
m_missingBlocksQuery = FormatInvariant($@"SELECT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Restored"" = @Restored ");
}
public bool SetBlockRestored(string hash, long size)
/// <inheritdoc/>
public bool SetBlockRestored(string hash, long size, long targetVolumeId)
{
return m_insertCommand.SetParameterValue("@NewRestoredValue", 1)
var restored = m_insertCommand.SetTransaction(m_transaction.Transaction).SetParameterValue("@NewRestoredValue", 1)
.SetParameterValue("@Hash", hash)
.SetParameterValue("@Size", size)
.SetParameterValue(@"PreviousRestoredValue", 0)
.SetParameterValue("@PreviousRestoredValue", 0)
.ExecuteNonQuery() == 1;
if (restored)
{
m_copyIntoDuplicatedBlocks.SetTransaction(m_transaction.Transaction)
.SetParameterValue("@Hash", hash)
.SetParameterValue("@Size", size)
.ExecuteNonQuery();
var c = m_assignBlocksToNewVolume.SetTransaction(m_transaction.Transaction)
.SetParameterValue("@TargetVolumeId", targetVolumeId)
.SetParameterValue("@Hash", hash)
.SetParameterValue("@Size", size)
.ExecuteNonQuery();
if (c != 1)
throw new Exception($"Unexpected number of updated blocks: {c} != 1");
}
return restored;
}
public IEnumerable<IBlockWithSources> GetSourceFilesWithBlocks(long blocksize)
/// <inheritdoc/>
public IEnumerable<BlockWithSourceData> GetSourceFilesWithBlocks(long blocksize)
{
using var cmd = m_connection.CreateCommand(m_transaction.Parent, FormatInvariant($@"SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {blocksize} FROM ""{m_tablename}"", ""Block"", ""BlocksetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" AND ""{m_tablename}"".""Restored"" = @Restored "))
.SetParameterValue("@Restored", 0);
using (var rd = cmd.ExecuteReader())
if (rd.Read())
{
var bs = new BlockWithSources(rd);
while (!bs.Done)
yield return bs;
}
}
public IEnumerable<KeyValuePair<string, long>> GetMissingBlocks()
{
using var cmd = m_connection.CreateCommand(m_transaction.Parent, FormatInvariant($@"SELECT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Restored"" = @Restored "))
using var cmd = m_connection.CreateCommand(m_transaction.Transaction, FormatInvariant($@"SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {blocksize} FROM ""{m_tablename}"", ""Block"", ""BlocksetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" AND ""{m_tablename}"".""Restored"" = @Restored ORDER BY ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {blocksize}"))
.SetParameterValue("@Restored", 0);
foreach (var rd in cmd.ExecuteReaderEnumerable())
yield return new KeyValuePair<string, long>(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1));
{
var hash = rd.ConvertValueToString(0) ?? throw new Exception("Hash value was null");
var size = rd.ConvertValueToInt64(1);
var file = rd.ConvertValueToString(2) ?? throw new Exception("File value was null");
var offset = rd.ConvertValueToInt64(3);
yield return new BlockWithSourceData(hash, size, file, offset);
}
}
/// <inheritdoc/>
public IEnumerable<BlockWithMetadataSourceData> GetSourceItemsWithMetadataBlocks()
{
using var cmd = m_connection.CreateCommand(m_transaction.Transaction, FormatInvariant($@"SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"" FROM ""{m_tablename}"", ""Block"", ""BlocksetEntry"", ""Metadataset"", ""File"" WHERE ""File"".""MetadataID"" == ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" AND ""{m_tablename}"".""Restored"" = @Restored ORDER BY ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"""))
.SetParameterValue("@Restored", 0);
foreach (var rd in cmd.ExecuteReaderEnumerable())
{
var hash = rd.ConvertValueToString(0) ?? throw new Exception("Hash value was null");
var size = rd.ConvertValueToInt64(1);
var path = rd.ConvertValueToString(2) ?? throw new Exception("File value was null");
yield return new BlockWithMetadataSourceData(hash, size, path);
}
}
/// <inheritdoc/>
public IEnumerable<BlocklistHashesEntry> GetBlocklistHashes(long hashesPerBlock)
{
using var cmd = m_connection.CreateCommand(m_transaction.Transaction);
var blocklistTableName = $"BlocklistHashList-{Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray())}";
try
{
// We need to create a snapshot as we will be updating the m_tablename table during enumeration
cmd.SetCommandAndParameters(FormatInvariant($@"
CREATE TEMPORARY TABLE ""{blocklistTableName}"" AS
SELECT
b.""Hash"" AS ""BlockHash"",
bs.""Id"" AS ""BlocksetId"",
bs.""Length"" AS ""BlocksetLength"",
bse.""Index"" / @HashesPerBlock AS ""BlocklistHashIndex"",
(
SELECT blh.""Hash""
FROM ""BlocklistHash"" blh
WHERE blh.""BlocksetID"" = bs.""ID""
AND blh.""Index"" == bse.""Index"" / @HashesPerBlock
LIMIT 1
) AS ""BlocklistHashHash"",
bse.""Index"" AS ""BlocksetEntryIndex""
FROM
""BlocksetEntry"" bse
JOIN ""Block"" b ON b.""ID"" = bse.""BlockID""
JOIN ""Blockset"" bs ON bs.""ID"" = bse.""BlocksetID""
WHERE
EXISTS (
SELECT 1
FROM ""BlocklistHash"" blh
JOIN ""{m_tablename}"" mt ON mt.""Hash"" = blh.""Hash""
WHERE blh.""BlocksetID"" = bs.""ID""
AND mt.""Restored"" = @Restored
)
"))
.SetParameterValue("@HashesPerBlock", hashesPerBlock)
.SetParameterValue("@Restored", 0)
.ExecuteNonQuery();
cmd.SetCommandAndParameters(FormatInvariant($@"
SELECT
""BlockHash"",
""BlocksetId"",
""BlocksetLength"",
""BlocklistHashIndex"",
""BlocklistHashHash"",
""BlocksetEntryIndex""
FROM ""{blocklistTableName}""
ORDER BY ""BlocksetId"", ""BlocklistHashIndex"", ""BlocksetEntryIndex"""));
foreach (var rd in cmd.ExecuteReaderEnumerable())
{
var hash = rd.ConvertValueToString(0) ?? throw new Exception("Block.Hash is null");
var blocksetId = rd.ConvertValueToInt64(1);
var length = rd.ConvertValueToInt64(2);
var blocklistHashIndex = rd.ConvertValueToInt64(3);
var blocklistHash = rd.ConvertValueToString(4) ?? throw new Exception("BlocklistHash is null");
var index = rd.ConvertValueToInt64(5);
yield return new BlocklistHashesEntry(blocksetId, blocklistHash, length, blocklistHashIndex, (int)index, hash);
}
}
finally
{
try { cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{blocklistTableName}"" ")); }
catch { }
}
}
/// <inheritdoc/>
public long GetMissingBlockCount()
{
using var cmd = m_connection.CreateCommand(m_transaction.Transaction, FormatInvariant($@"SELECT COUNT(*) FROM ({m_missingBlocksQuery})"))
.SetParameterValue("@Restored", 0);
return cmd.ExecuteScalarInt64(0);
}
/// <summary>
/// Gets the list of missing blocks
/// </summary>
/// <returns>>A list of missing blocks</returns>
public IEnumerable<(string Hash, long Size)> GetMissingBlocks()
{
using var cmd = m_connection.CreateCommand(m_transaction.Transaction, m_missingBlocksQuery)
.SetParameterValue("@Restored", 0);
foreach (var rd in cmd.ExecuteReaderEnumerable())
yield return (rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1));
}
/// <inheritdoc/>
public long MoveBlocksToNewVolume(long targetVolumeId, long sourceVolumeId, IDbTransaction? transaction = null)
{
if (targetVolumeId <= 0)
throw new ArgumentOutOfRangeException(nameof(targetVolumeId), "Target volume ID must be greater than 0");
// Move the source blocks into the DuplicateBlock table
using var cmd = m_connection.CreateCommand(transaction, FormatInvariant($@"
INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"")
SELECT b.""ID"", b.""VolumeID""
FROM ""Block"" b
WHERE b.""VolumeID"" = @SourceVolumeId
AND (b.""Hash"", b.""Size"") IN (
SELECT ""Hash"", ""Size""
FROM ""{m_tablename}""
WHERE ""Restored"" = @Restored
)
"))
.SetParameterValue("@SourceVolumeId", sourceVolumeId)
.SetParameterValue("@Restored", 1);
var moved = cmd.ExecuteNonQuery();
// Then update the blocks table to point to the new volume
cmd.SetCommandAndParameters(FormatInvariant($@"
UPDATE ""Block""
SET ""VolumeID"" = @TargetVolumeId
WHERE ""VolumeID"" = @SourceVolumeId
AND (""Hash"", ""Size"") IN (
SELECT ""Hash"", ""Size""
FROM ""{m_tablename}""
WHERE ""Restored"" = @Restored
)
"))
.SetParameterValue("@TargetVolumeId", targetVolumeId)
.SetParameterValue("@SourceVolumeId", sourceVolumeId)
.SetParameterValue("@Restored", 1);
var updated = cmd.ExecuteNonQuery();
if (updated != moved)
throw new Exception($"Unexpected number of updated blocks: {updated} != {moved}");
return updated;
}
public IEnumerable<IRemoteVolume> GetFilesetsUsingMissingBlocks()
@@ -246,7 +533,7 @@ namespace Duplicati.Library.Main.Database
var blocklists = FormatInvariant($@"SELECT DISTINCT ""FileLookup"".""ID"" AS ID FROM ""{m_tablename}"", ""Block"", ""Blockset"", ""BlocklistHash"", ""FileLookup"" WHERE ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""BlocklistHash"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" ");
var cmdtxt = FormatInvariant($@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""FilesetEntry"", ""Fileset"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""RemoteVolume"".""Type"" = @Type AND ""FilesetEntry"".""FileID"" IN (SELECT DISTINCT ""ID"" FROM ( {blocks} UNION {blocklists} ))");
using var cmd = m_connection.CreateCommand(m_transaction.Parent, cmdtxt)
using var cmd = m_connection.CreateCommand(m_transaction.Transaction, cmdtxt)
.SetParameterValue("@Type", RemoteVolumeType.Files.ToString());
foreach (var rd in cmd.ExecuteReaderEnumerable())
@@ -255,7 +542,8 @@ namespace Duplicati.Library.Main.Database
public IEnumerable<IRemoteVolume> GetMissingBlockSources()
{
using var cmd = m_connection.CreateCommand(m_transaction.Parent, FormatInvariant($@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""Block"", ""{m_tablename}"" WHERE ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""Block"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Remotevolume"".""Name"" != @Name "))
using var cmd = m_connection.CreateCommand(m_transaction.Transaction, FormatInvariant($@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""Block"", ""{m_tablename}"" WHERE ""{m_tablename}"".""Restored"" = @Restored AND ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""Block"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Remotevolume"".""Name"" != @Name "))
.SetParameterValue("@Restored", 0)
.SetParameterValue("@Name", m_volumename);
foreach (var rd in cmd.ExecuteReaderEnumerable())
@@ -264,25 +552,24 @@ namespace Duplicati.Library.Main.Database
public void Dispose()
{
if (m_tablename != null)
{
try
{
using (var cmd = m_connection.CreateCommand(m_transaction.Parent))
cmd.SetTransaction(m_transaction.Parent).ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tablename}"" "));
}
catch { }
finally { m_tablename = null!; }
}
if (m_isDisposed)
return;
if (m_insertCommand != null)
try { m_insertCommand.Dispose(); }
catch { }
finally { m_insertCommand = null!; }
m_isDisposed = true;
try
{
if (m_tablename != null)
using (var cmd = m_connection.CreateCommand(m_transaction.Transaction))
cmd.SetTransaction(m_transaction.Transaction).ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tablename}"" "));
}
catch { }
try { m_insertCommand?.Dispose(); }
catch { }
}
}
public IMissingBlockList CreateBlockList(string volumename, IDbTransaction? transaction = null)
public IMissingBlockList CreateBlockList(string volumename, ReusableTransaction transaction)
{
return new MissingBlockList(volumename, m_connection, transaction);
}
@@ -589,13 +876,12 @@ namespace Duplicati.Library.Main.Database
}
}
public void CheckAllBlocksAreInVolume(string filename, IEnumerable<KeyValuePair<string, long>> blocks)
public void CheckAllBlocksAreInVolume(string filename, IEnumerable<KeyValuePair<string, long>> blocks, IDbTransaction transaction)
{
using (var tr = m_connection.BeginTransactionSafe())
using (var cmd = m_connection.CreateCommand(tr))
using var cmd = m_connection.CreateCommand(transaction);
var tablename = "ProbeBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
try
{
var tablename = "ProbeBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)"));
cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{tablename}"" (""Hash"", ""Size"") VALUES (@Hash, @Size)"));
@@ -614,16 +900,18 @@ namespace Duplicati.Library.Main.Database
.SetParameterValue("@VolumeId", id)
.ExecuteScalarInt64(0);
cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tablename}"" "));
if (aliens != 0)
throw new Exception($"Not all blocks were found in {filename}");
}
finally
{
cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tablename}"" "));
}
}
public void CheckBlocklistCorrect(string hash, long length, IEnumerable<string> blocklist, long blocksize, long blockhashlength)
public void CheckBlocklistCorrect(string hash, long length, IEnumerable<string> blocklist, long blocksize, long blockhashlength, IDbTransaction? transaction)
{
using (var cmd = m_connection.CreateCommand())
using (var cmd = m_connection.CreateCommand(transaction))
{
var query = FormatInvariant($@"
SELECT
Duplicati/Library/Main/Operation/CompactHandler.cs
+5 -8
View File
@@ -325,16 +325,13 @@ namespace Duplicati.Library.Main.Operation
private async Task FinishVolumeAndUpload(LocalDeleteDatabase db, IBackendManager backendManager, BlockVolumeWriter newvol, IndexVolumeWriter newvolindex, List<KeyValuePair<string, long>> uploadedVolumes, ReusableTransaction rtr)
{
Action indexVolumeFinished = () =>
{
if (newvolindex != null && m_options.IndexfilePolicy == Options.IndexFileStrategy.Full)
Action indexVolumeFinished = null;
if (newvolindex != null && m_options.IndexfilePolicy == Options.IndexFileStrategy.Full)
indexVolumeFinished = () =>
{
foreach (var blocklist in db.GetBlocklists(newvol.VolumeID, m_options.Blocksize, m_options.BlockhashSize))
{
newvolindex.WriteBlocklist(blocklist.Item1, blocklist.Item2, 0, blocklist.Item3);
}
}
};
};
uploadedVolumes.Add(new KeyValuePair<string, long>(newvol.RemoteFilename, newvol.Filesize));
if (newvolindex != null)
@@ -342,7 +339,7 @@ namespace Duplicati.Library.Main.Operation
// TODO: The upload here does not flush the database messages,
// and this can leave the database in a state where it does not know of the remote file
// To fix it, we need thread-safe access to the daabase and transaction
// To fix it, we need thread-safe access to the database and transaction
// Once fixed, we can perhaps let he backend manager simply call the database directly
if (!m_options.Dryrun)
{
Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs
+6 -1
View File
@@ -55,7 +55,7 @@ namespace Duplicati.Library.Main.Operation
await DoRunAsync(backendManager, db, tr, filter, callbackhandler).ConfigureAwait(false);
}
public static async Task<(Tuple<DateTime, long, long>[], List<Database.RemoteVolumeEntry> Missing)> GetBrokenFilesetsFromRemote(IBackendManager backendManager, BasicResults result, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Options options)
public static async Task<((DateTime FilesetTime, long FilesetID, long RemoveCount)[], List<Database.RemoteVolumeEntry> Missing)> GetBrokenFilesetsFromRemote(IBackendManager backendManager, BasicResults result, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Options options)
{
List<Database.RemoteVolumeEntry> missing = null;
var brokensets = db.GetBrokenFilesets(options.Time, options.Version, transaction).ToArray();
@@ -86,6 +86,11 @@ namespace Duplicati.Library.Main.Operation
// Drop all content from tables
db.RemoveMissingBlocks(missing.Select(x => x.Name), transaction);
// Mark all orphaned index files as disposable after removing the missing block files
foreach (var f in db.GetOrphanedIndexFiles(transaction).ToList())
db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, transaction);
brokensets = db.GetBrokenFilesets(options.Time, options.Version, transaction).ToArray();
}
Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs
+23 -5
View File
@@ -88,14 +88,25 @@ namespace Duplicati.Library.Main.Operation
SetCount = db.GetFilesetFileCount(x.Item2, tr)
}).ToArray();
if (m_options.Dryrun)
tr.Rollback();
else
tr.Commit();
var emptyBlocksetId = -1L;
if (!m_options.DisableReplaceMissingMetadata)
{
var emptymetadata = Utility.WrapMetadata(new Dictionary<string, string>(), m_options);
emptyBlocksetId = db.GetEmptyMetadataBlocksetId(missing.Select(x => x.ID), emptymetadata.FileHash, emptymetadata.Blob.Length, null);
if (emptyBlocksetId < 0)
throw new UserInformationException($"Failed to locate an empty metadata blockset to replace missing metadata. Set the option --disable-replace-missing-metadata=true to ignore this and drop files with missing metadata.", "FailedToLocateEmptyMetadataBlockset");
}
var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray();
var to_purge = compare_list.Where(x => x.RemoveCount != x.SetCount).ToArray();
if (fully_emptied.Length == db.FilesetTimes.Count())
throw new UserInformationException("All filesets are fully broken and needs to be removed. To avoid unexpected deletions, you must manually remove the remote files and delete the database.", "AllFilesetsBroken");
if (!m_options.Dryrun)
tr.Commit();
if (fully_emptied.Length != 0)
{
if (fully_emptied.Length == 1)
@@ -104,7 +115,7 @@ namespace Duplicati.Library.Main.Operation
Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing {0} filesets where all file(s) are broken: {1}", fully_emptied.Length, string.Join(", ", fully_emptied.Select(x => x.Timestamp.ToLocalTime().ToString())));
m_result.DeleteResults = new DeleteResults(m_result);
using (var rmdb = new Database.LocalDeleteDatabase(db))
using (var rmdb = new LocalDeleteDatabase(db))
using (var deltr = new ReusableTransaction(rmdb))
{
var opts = new Options(new Dictionary<string, string>(m_options.RawOptions));
@@ -148,7 +159,14 @@ namespace Duplicati.Library.Main.Operation
{
if (filesetid != bs.FilesetID)
throw new Exception(string.Format("Unexpected filesetid: {0}, expected {1}", filesetid, bs.FilesetID));
// Update entries that would be removed because of missing metadata
var updatedEntries = 0;
if (!m_options.DisableReplaceMissingMetadata)
updatedEntries = db.ReplaceMetadata(filesetid, emptyBlocksetId, cmd.Transaction);
db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID", cmd.Transaction);
return updatedEntries;
}).ConfigureAwait(false);
}
Duplicati/Library/Main/Operation/PurgeFilesHandler.cs
+12 -32
View File
@@ -21,9 +21,9 @@
using System;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Duplicati.Library.Interface;
using Duplicati.Library.Main.Volumes;
using Duplicati.Library.Utility;
namespace Duplicati.Library.Main.Operation
@@ -58,10 +58,10 @@ namespace Duplicati.Library.Main.Operation
}
}
public Task RunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, float pgoffset, float pgspan, Action<System.Data.IDbCommand, long, string> filtercommand)
public Task RunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, float pgoffset, float pgspan, Func<System.Data.IDbCommand, long, string, int> filtercommand)
=> DoRunAsync(backendManager, db, null, filtercommand, pgoffset, pgspan);
private async Task DoRunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, IFilter filter, Action<System.Data.IDbCommand, long, string> filtercommand, float pgoffset, float pgspan)
private async Task DoRunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, IFilter filter, Func<System.Data.IDbCommand, long, string, int> filtercommand, float pgoffset, float pgspan)
{
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Begin);
Logging.Log.WriteInformationMessage(LOGTAG, "StartingPurge", "Starting purge operation");
@@ -116,34 +116,13 @@ namespace Duplicati.Library.Main.Operation
using (var tr = db.BeginTransaction())
{
var ix = -1;
for (var i = 0; i < filesets.Length; i++)
if (filesets[i].Key == versionid)
{
ix = i;
break;
}
if (ix < 0)
(var _, var tsOriginal, var ix) = filesets.Select((x, i) => (x.Key, x.Value, i)).FirstOrDefault(x => x.Key == versionid);
if (ix < 0 || tsOriginal.Ticks == 0)
throw new InvalidProgramException(string.Format("Fileset was reported with id {0}, but could not be found?", versionid));
var secs = 0;
while (secs < 60)
{
secs++;
var tfn = Volumes.VolumeBase.GenerateFilename(RemoteVolumeType.Files, m_options, null, filesets[ix].Value.AddSeconds(secs));
if (db.GetRemoteVolumeID(tfn, tr) < 0)
break;
}
var tsOriginal = filesets[ix].Value;
var ts = tsOriginal.AddSeconds(secs);
var ts = FilesetVolumeWriter.ProbeUnusedFilenameName(db, tr, m_options, tsOriginal);
var prevfilename = db.GetRemoteVolumeNameForFileset(filesets[ix].Key, tr);
if (secs >= 60)
throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is more than 60 seconds away", prevfilename, ts));
if (ix != 0 && filesets[ix - 1].Value <= ts)
throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is larger than the next timestamp {2}", prevfilename, ts, filesets[ix - 1].Value));
@@ -154,7 +133,7 @@ namespace Duplicati.Library.Main.Operation
else
tempset.ApplyFilter(filtercommand);
if (tempset.RemovedFileCount == 0)
if (tempset.RemovedFileCount + tempset.UpdatedFileCount == 0)
{
Logging.Log.WriteInformationMessage(LOGTAG, "NotWritingNewFileset", "Not writing a new fileset for {0} as it was not changed", prevfilename);
currentprogress += versionprogress;
@@ -163,10 +142,10 @@ namespace Duplicati.Library.Main.Operation
}
else
{
using (var tf = new Library.Utility.TempFile())
using (var vol = new Volumes.FilesetVolumeWriter(m_options, ts))
using (var tf = new TempFile())
using (var vol = new FilesetVolumeWriter(m_options, ts))
{
var isOriginalFilesetFullBackup = db.IsFilesetFullBackup(tsOriginal);
var isOriginalFilesetFullBackup = db.IsFilesetFullBackup(tsOriginal, tr);
var newids = tempset.ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup);
vol.VolumeID = newids.Item1;
vol.CreateFilesetFile(isOriginalFilesetFullBackup);
@@ -177,6 +156,7 @@ namespace Duplicati.Library.Main.Operation
m_result.RemovedFileSize += tempset.RemovedFileSize;
m_result.RemovedFileCount += tempset.RemovedFileCount;
m_result.UpdatedFileCount += tempset.UpdatedFileCount;
m_result.RewrittenFileLists++;
currentprogress += (versionprogress / 2);
@@ -208,7 +188,7 @@ namespace Duplicati.Library.Main.Operation
}
else
{
var lst = db.DropFilesetsFromTable(new[] { filesets[ix].Value }, tr).ToArray();
var lst = db.DropFilesetsFromTable(new[] { tsOriginal }, tr).ToArray();
foreach (var f in lst)
db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, tr);
Duplicati/Library/Main/Operation/RepairHandler.cs
+689 -219
View File
File diff suppressed because it is too large Load Diff
Duplicati/Library/Main/Options.cs
+18
View File
@@ -471,6 +471,8 @@ namespace Duplicati.Library.Main
new CommandLineArgument("ignore-filenames", CommandLineArgument.ArgumentType.Path, Strings.Options.IgnorefilenamesShort, Strings.Options.IgnorefilenamesLong, "CACHEDIR.TAG"),
new CommandLineArgument("restore-symlink-metadata", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RestoresymlinkmetadataShort, Strings.Options.RestoresymlinkmetadataLong, "false"),
new CommandLineArgument("rebuild-missing-dblock-files", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RebuildmissingdblockfilesShort, Strings.Options.RebuildmissingdblockfilesLong, "false"),
new CommandLineArgument("disable-partial-dblock-recovery", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablePartialDblockRecoveryShort, Strings.Options.DisablePartialDblockRecoveryLong, "false"),
new CommandLineArgument("disable-replace-missing-metadata", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisableReplaceMissingMetadataShort, Strings.Options.DisableReplaceMissingMetadataLong, "false"),
new CommandLineArgument("auto-compact-interval", CommandLineArgument.ArgumentType.Timespan, Strings.Options.AutoCompactIntervalShort, Strings.Options.AutoCompactIntervalLong, "0m"),
new CommandLineArgument("auto-vacuum-interval", CommandLineArgument.ArgumentType.Timespan, Strings.Options.AutoVacuumIntervalShort, Strings.Options.AutoVacuumIntervalLong, "0m"),
@@ -1927,6 +1929,22 @@ namespace Duplicati.Library.Main
get { return GetBool("rebuild-missing-dblock-files"); }
}
/// <summary>
/// Gets a value indicating if partial dblock recovery is disabled
/// </summary>
public bool DisablePartialDblockRecovery
{
get { return GetBool("disable-partial-dblock-recovery"); }
}
/// <summary>
/// Gets a value indicating if missing metadata is replaced with empty content on purge-broken-files
/// </summary>
public bool DisableReplaceMissingMetadata
{
get { return GetBool("disable-replace-missing-metadata"); }
}
/// <summary>
/// Gets the threshold for when log data should be cleaned
/// </summary>
Duplicati/Library/Main/ResultClasses.cs
+1
View File
@@ -902,6 +902,7 @@ namespace Duplicati.Library.Main
public long RemovedFileCount { get; set; }
public long RemovedFileSize { get; set; }
public long RewrittenFileLists { get; set; }
public long UpdatedFileCount { get; set; }
public ICompactResults CompactResults { get; set; }
}
Duplicati/Library/Main/Strings.cs
+4
View File
@@ -310,6 +310,10 @@ namespace Duplicati.Library.Main.Strings
public static string ProfilealldatabasequeriesShort { get { return LC.L("Activate logging of all database queries"); } }
public static string RebuildmissingdblockfilesLong { get { return LC.L("If dblock files are missing from the destination, you can attempt to rebuild them using local source data. However, since the local data may have changed, it may not be possible to retrieve all the required data and the process may be slow. Use this option to attempt to rebuild missing dblock files."); } }
public static string RebuildmissingdblockfilesShort { get { return LC.L("Rebuild dblock files when missing"); } }
public static string DisablePartialDblockRecoveryLong { get { return LC.L("If rebuilding dblock files, the process will attempt to use local data to fill in missing blocks. If it is not possible to fill in all missing blocks, Duplicati will attempt to create a dblock with as much data as possible. Use this option to disable this behavior and only create dblocks that are complete."); } }
public static string DisablePartialDblockRecoveryShort { get { return LC.L("Disable partial dblock recovery"); } }
public static string DisableReplaceMissingMetadataLong { get { return LC.L($"If metadata is missing during the {"list-broken-files"} or {"purge-broken-files"} command, Duplicati will attempt to replace the missing metadata with a new copy. Use this option to disable this behavior and drop entries that are missing metadata."); } }
public static string DisableReplaceMissingMetadataShort { get { return LC.L("Disable replacement of missing metadata"); } }
public static string AutoCompactIntervalLong { get { return LC.L("The minimum amount of time that must elapse after the last compaction before another will be automatically triggered at the end of a backup job. Automatic compaction can be a long-running process and may not be desirable to run after every single backup."); } }
public static string AutoCompactIntervalShort { get { return LC.L("Minimum time between auto compactions"); } }
Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs
+4 -2
View File
@@ -27,6 +27,7 @@ using Duplicati.Library.Main.Database;
using Duplicati.Library.Main.Operation.Common;
using System.Threading.Tasks;
using Duplicati.Library.Utility;
using System.Data;
namespace Duplicati.Library.Main.Volumes
{
@@ -244,12 +245,13 @@ namespace Duplicati.Library.Main.Volumes
/// </summary>
/// <param name="database">The database to check for clashes</param>
/// <param name="options">The options to use for the filename</param>
/// <param name="transaction">The transaction to use</param>
/// <param name="start">The starting time to probe from</param>
/// <param name="increment">The time to increment by each probe</param>
/// <param name="maxTries">The maximum number of tries to probe</param>
/// <returns>The first unused filename</returns>
internal static DateTime ProbeUnusedFilenameName(LocalDatabase database, Options options, DateTime start, TimeSpan increment = default, int maxTries = 60)
=> ProbeUnusedFilenameName((name) => Task.FromResult(database.GetRemoteVolumeID(name)), options, start, increment, maxTries).Await();
internal static DateTime ProbeUnusedFilenameName(LocalDatabase database, IDbTransaction transaction, Options options, DateTime start, TimeSpan increment = default, int maxTries = 60)
=> ProbeUnusedFilenameName((name) => Task.FromResult(database.GetRemoteVolumeID(name, transaction)), options, start, increment, maxTries).Await();
/// <summary>
/// Probes for an unused filename, using the current time as a starting point
Duplicati/UnitTest/Issue5987.cs
+65
View File
@@ -0,0 +1,65 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.IO;
using System.Linq;
using NUnit.Framework;
namespace Duplicati.UnitTest
{
public class Issue5987 : BasicSetupHelper
{
[Test]
[Category("Targeted")]
[TestCase(true)]
[TestCase(false)]
public void TestRestoreWithMissingDblocks(bool deleteIndexFiles)
{
var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true });
// 1. Make a backup of a single file
File.WriteAllText(Path.Combine(DATAFOLDER, "a"), "abc");
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
TestUtils.AssertResults(c.Backup([DATAFOLDER]));
// 2. Delete the dblock file
var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dblockFiles)
File.Delete(file);
if (deleteIndexFiles)
{
var dindexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dindexFiles)
File.Delete(file);
}
// 3. Try to repair the remote destination
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
var res = c.Repair();
Assert.That(res.Errors.Count(), Is.EqualTo(0), "Repair should not have errors");
TestUtils.AssertResults(c.Test());
}
}
}
}
Duplicati/UnitTest/RepairHandlerTests.cs
+6 -10
View File
@@ -164,7 +164,7 @@ namespace Duplicati.UnitTest
Assert.AreEqual(0, backupResults.Warnings.Count());
}
var dindexFiles = Directory.EnumerateFiles(this.TARGETFOLDER, "*dindex*").ToArray();
var dindexFiles = Directory.EnumerateFiles(this.TARGETFOLDER, "*.dindex.*").ToArray();
Assert.Greater(dindexFiles.Length, 0);
foreach (var f in dindexFiles)
{
@@ -178,10 +178,8 @@ namespace Duplicati.UnitTest
Assert.AreEqual(0, repairResults.Warnings.Count());
}
foreach (var file in dindexFiles)
{
Assert.IsTrue(File.Exists(Path.Combine(this.TARGETFOLDER, file)));
}
var recreatedIndexFiles = Directory.EnumerateFiles(this.TARGETFOLDER, "*dindex*").ToArray();
Assert.AreEqual(dindexFiles.Length, recreatedIndexFiles.Length);
}
[Test]
@@ -216,7 +214,7 @@ namespace Duplicati.UnitTest
Assert.AreEqual(0, backupResults.Warnings.Count());
}
var dindexFiles = Directory.EnumerateFiles(this.TARGETFOLDER, "*dindex*").ToArray();
var dindexFiles = Directory.EnumerateFiles(this.TARGETFOLDER, "*.dindex.*").ToArray();
Assert.Greater(dindexFiles.Length, 0);
foreach (var f in dindexFiles)
{
@@ -230,10 +228,8 @@ namespace Duplicati.UnitTest
Assert.AreEqual(0, repairResults.Warnings.Count());
}
foreach (var file in dindexFiles)
{
Assert.IsTrue(File.Exists(Path.Combine(this.TARGETFOLDER, file)));
}
var recreatedIndexFiles = Directory.EnumerateFiles(this.TARGETFOLDER, "*dindex*").ToArray();
Assert.AreEqual(dindexFiles.Length, recreatedIndexFiles.Length);
}
[Test]
Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs
+640
View File
@@ -0,0 +1,640 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Duplicati.Library.Interface;
using Duplicati.Library.Main.Database;
using Duplicati.Library.SQLiteHelper;
using NUnit.Framework;
namespace Duplicati.UnitTest
{
public class RepairWithMissingRemoteFiles : BasicSetupHelper
{
// TODO: Add more tests to cover the following scenarios:
// Test where a single block in a large file is missing + run purge-broken-files
// Test that fetching remote sources for blocks is working
[Test]
[Category("Targeted")]
[TestCase(false, true, false)]
[TestCase(true, true, false)]
[TestCase(true, false, false)]
public void TestRepairWithMissingDblocks(bool deleteDblockFiles, bool deleteIndexFiles, bool deleteDlistFiles)
{
var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true, number_of_retries = 0 });
// 1. Make a backup of a single file (1 block + 3 bytes for a block with less data)
var data = new byte[1024 + 3];
data.AsSpan().Fill((byte)'a');
File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data);
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
TestUtils.AssertResults(c.Backup([DATAFOLDER]));
// 2. Delete files
if (deleteDblockFiles)
{
var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dblockFiles)
File.Delete(file);
}
if (deleteIndexFiles)
{
var dindexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dindexFiles)
File.Delete(file);
}
if (deleteDlistFiles)
{
var dlistFiles = Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dlistFiles)
File.Delete(file);
}
// 3. Try to repair the remote destination
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
TestUtils.AssertResults(c.Repair());
// 4. Test that the database is now valid
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
TestUtils.AssertResults(c.Test());
}
[Test]
[Category("Targeted")]
[TestCase(false, true, false)]
[TestCase(true, true, false)]
[TestCase(true, false, false)]
public void TestRepairWithMissingDblocksAndMultipleSources(bool deleteDblockFiles, bool deleteIndexFiles, bool deleteDlistFiles)
{
var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true });
// 1. Make some shared data
var data = new byte[1024 + 3];
data.AsSpan().Fill((byte)'a');
File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data);
File.WriteAllBytes(Path.Combine(DATAFOLDER, "b"), data.AsSpan().Slice(0, 1024).ToArray());
File.WriteAllBytes(Path.Combine(DATAFOLDER, "c"), data.AsSpan().Slice(0, 1024).ToArray());
File.WriteAllBytes(Path.Combine(DATAFOLDER, "d"), data.AsSpan().Slice(1024, 3).ToArray());
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
TestUtils.AssertResults(c.Backup([DATAFOLDER]));
// 2. Delete files
if (deleteDblockFiles)
{
var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dblockFiles)
File.Delete(file);
}
if (deleteIndexFiles)
{
var dindexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dindexFiles)
File.Delete(file);
}
if (deleteDlistFiles)
{
var dlistFiles = Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dlistFiles)
File.Delete(file);
}
// 3. Try to repair the remote destination
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
TestUtils.AssertResults(c.Repair());
TestUtils.AssertResults(c.Test());
}
}
[Test]
[Category("Targeted")]
[TestCase(false, true, false)]
[TestCase(true, true, false)]
[TestCase(true, false, false)]
public void TestRepairWithDataFillingMultipleBlocklists(bool deleteDblockFiles, bool deleteIndexFiles, bool deleteDlistFiles)
{
var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true });
// 1. Make some shared data 1kb blocks, 32 bytes hash, 3 blocklists, and 5 bytes extra
var data = new byte[1024 * 32 * 3 + 5];
data.AsSpan().Fill((byte)'a');
File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data);
File.WriteAllBytes(Path.Combine(DATAFOLDER, "b"), data.AsSpan().Slice(1024, 3).ToArray());
Random.Shared.NextBytes(data);
File.WriteAllBytes(Path.Combine(DATAFOLDER, "c"), data);
Random.Shared.NextBytes(data);
File.WriteAllBytes(Path.Combine(DATAFOLDER, "d"), data.AsSpan().Slice(0, 1028).ToArray());
File.WriteAllBytes(Path.Combine(DATAFOLDER, "e"), data.AsSpan().Slice(4098, 67).ToArray());
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
TestUtils.AssertResults(c.Backup([DATAFOLDER]));
// 2. Delete files
if (deleteDblockFiles)
{
var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dblockFiles)
File.Delete(file);
}
if (deleteIndexFiles)
{
var dindexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dindexFiles)
File.Delete(file);
}
if (deleteDlistFiles)
{
var dlistFiles = Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dlistFiles)
File.Delete(file);
}
// 3. Try to repair the remote destination
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
TestUtils.AssertResults(c.Repair());
TestUtils.AssertResults(c.Test());
}
}
[Test]
[Category("Targeted")]
[TestCase(true, true, true)]
[TestCase(true, false, false)]
public void TestRepairWithSmallFileOnly(bool deleteDblockFiles, bool deleteIndexFiles, bool deleteDlistFiles)
{
var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true });
// 1. Make some shared data
var data = new byte[3];
data.AsSpan().Fill((byte)'a');
File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data);
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
TestUtils.AssertResults(c.Backup([DATAFOLDER]));
// 2. Delete files
if (deleteDblockFiles)
{
var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dblockFiles)
File.Delete(file);
}
if (deleteIndexFiles)
{
var dindexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dindexFiles)
File.Delete(file);
}
if (deleteDlistFiles)
{
var dlistFiles = Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dlistFiles)
File.Delete(file);
}
// 3. Try to repair the remote destination
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
TestUtils.AssertResults(c.Repair());
TestUtils.AssertResults(c.Test());
}
}
[Test]
[Category("Targeted")]
[TestCase(false, true, false)]
[TestCase(true, true, false)]
[TestCase(true, false, false)]
[TestCase(true, true, true)]
[TestCase(true, false, true)]
public void TestRepairWithDataFillingMultipleBlocklistsRandom(bool deleteDblockFiles, bool deleteIndexFiles, bool deleteDlistFiles)
{
var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true });
// 1. Make some shared data 1kb blocks, 32 bytes hash, 3 blocklists, and 1028 bytes extra
var data = new byte[1024 * 32 * 3 + 1028];
Random.Shared.NextBytes(data);
File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data);
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
TestUtils.AssertResults(c.Backup([DATAFOLDER]));
// 2. Delete files
if (deleteDblockFiles)
{
var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dblockFiles)
File.Delete(file);
}
if (deleteIndexFiles)
{
var dindexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dindexFiles)
File.Delete(file);
}
if (deleteDlistFiles)
{
var dlistFiles = Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dlistFiles)
File.Delete(file);
}
// 3. Try to repair the remote destination
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
TestUtils.AssertResults(c.Repair());
TestUtils.AssertResults(c.Test());
}
// 4. Verify that restore works
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { restore_path = RESTOREFOLDER }), null))
TestUtils.AssertResults(c.Restore(null));
TestUtils.AssertDirectoryTreesAreEquivalent(DATAFOLDER, RESTOREFOLDER, !Library.Utility.Utility.ParseBoolOption(testopts, "skip-metadata"), "Restore");
}
[Test]
[Category("Targeted")]
[TestCase(true, true, true)]
[TestCase(true, false, false)]
public void TestRepairWithDryRun(bool deleteDblockFiles, bool deleteIndexFiles, bool deleteDlistFiles)
{
var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true });
// 1. Make some random data
var data = new byte[1028];
Random.Shared.NextBytes(data);
File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data);
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
TestUtils.AssertResults(c.Backup([DATAFOLDER]));
// 2. Delete files
if (deleteDblockFiles)
{
var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dblockFiles)
File.Delete(file);
}
if (deleteIndexFiles)
{
var dindexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dindexFiles)
File.Delete(file);
}
if (deleteDlistFiles)
{
var dlistFiles = Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dlistFiles)
File.Delete(file);
}
(long remoteVolumes, long duplicatedBlocks, List<long> blockVolumeIds, List<(long, long)> indexVolumeLinks) GetDbSignature()
{
using var db = SQLiteLoader.LoadConnection(DBFILE, 0);
using var cmd = db.CreateCommand();
var remoteVolumes = cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM RemoteVolume");
var duplicatedBlocks = cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM DuplicateBlock");
var blockVolumeIds = cmd.ExecuteReaderEnumerable("SELECT VolumeID FROM Block").Select(x => x.ConvertValueToInt64(0)).ToList();
var indexVolumeLinks = cmd.ExecuteReaderEnumerable("SELECT IndexVolumeID, BlockVolumeID FROM IndexBlockLink").Select(x => (x.ConvertValueToInt64(0), x.ConvertValueToInt64(1))).ToList();
return (remoteVolumes, duplicatedBlocks, blockVolumeIds, indexVolumeLinks);
}
var preSignature = GetDbSignature();
// 3. Try to repair the remote destination
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { dry_run = true }), null))
TestUtils.AssertResults(c.Repair());
var postSignature = GetDbSignature();
Assert.AreEqual(preSignature.remoteVolumes, postSignature.remoteVolumes, "Remote volumes count should be the same after dry run repair.");
Assert.AreEqual(preSignature.duplicatedBlocks, postSignature.duplicatedBlocks, "Duplicated blocks count should be the same after dry run repair.");
Assert.AreEqual(preSignature.blockVolumeIds.Count, postSignature.blockVolumeIds.Count, "Block volume IDs count should be the same after dry run repair.");
Assert.AreEqual(preSignature.indexVolumeLinks.Count, postSignature.indexVolumeLinks.Count, "Index volume links count should be the same after dry run repair.");
Assert.IsTrue(preSignature.blockVolumeIds.All(x => postSignature.blockVolumeIds.Contains(x)), "All block volume IDs should be present after dry run repair.");
Assert.IsTrue(preSignature.indexVolumeLinks.All(x => postSignature.indexVolumeLinks.Contains(x)), "All index volume links should be present after dry run repair.");
}
[Test]
[Category("Targeted")]
public void TestRepairWithNoBlockAvailableFails()
{
var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true });
// 1. Make some shared data 1kb blocks, 32 bytes hash, 3 blocklists, and 1028 bytes extra
var data = new byte[1024 * 32 * 3 + 1028];
Random.Shared.NextBytes(data);
File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data);
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
TestUtils.AssertResults(c.Backup([DATAFOLDER]));
// 2. Delete files
var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dblockFiles)
File.Delete(file);
// Destroy source data
File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), new byte[0]);
Directory.SetLastWriteTime(DATAFOLDER, DateTime.UtcNow.AddSeconds(1));
// 3. Try to repair the remote destination
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
var res = c.Repair();
Assert.IsTrue(res.Errors.Where(x => x.Contains("purge-broken-files")).Count() == 1, "Repair should fail with missing block data.");
}
}
[Test]
[Category("Targeted")]
public void TestPartialRepairPossibleWithMissingMetadata()
{
var testopts = TestOptions.Expand(new
{
blocksize = "1kb",
no_encryption = true,
rebuild_missing_dblock_files = true
});
// 1. Make some shared data 1kb blocks, 32 bytes hash, 3 blocklists, and 1028 bytes extra
var data = new byte[1024 * 32 * 3 + 1028];
Random.Shared.NextBytes(data);
File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data);
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
TestUtils.AssertResults(c.Backup([DATAFOLDER]));
// 2. Delete files
var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dblockFiles)
File.Delete(file);
// Destroy some source data
Directory.SetLastWriteTime(DATAFOLDER, DateTime.UtcNow.AddSeconds(1));
// 3. Try to repair the remote destination
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
var res = c.Repair();
Assert.IsTrue(res.Warnings.Where(x => x.Contains("purge-broken-files")).Count() == 1, "Repair should warn about missing block data.");
// Ensure that the partial repair did not clean up fully
Assert.Throws<RemoteListVerificationException>(() => c.Test());
// Purge broken files should be possible
TestUtils.AssertResults(c.PurgeBrokenFiles(null));
TestUtils.AssertResults(c.Test(int.MaxValue));
}
// 4. Verify that restore works
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { restore_path = RESTOREFOLDER }), null))
TestUtils.AssertResults(c.Restore(null));
TestUtils.AssertDirectoryTreesAreEquivalent(DATAFOLDER, RESTOREFOLDER, !Library.Utility.Utility.ParseBoolOption(testopts, "skip-metadata"), "Restore");
}
[Test]
[Category("Targeted")]
[TestCase(false)]
[TestCase(true)]
public void TestPartialRepairPossibleWithDeletedFiles(bool deleteLargeFile)
{
var testopts = TestOptions.Expand(new
{
blocksize = "1kb",
no_encryption = true,
rebuild_missing_dblock_files = true
});
// 1. Make some shared data 1kb blocks, 32 bytes hash, 3 blocklists, and 1028 bytes extra
var data = new byte[1024 * 32 * 3 + 1028];
Random.Shared.NextBytes(data);
File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data);
File.WriteAllBytes(Path.Combine(DATAFOLDER, "b"), data.AsSpan().Slice(0, 1024).ToArray());
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
TestUtils.AssertResults(c.Backup([DATAFOLDER]));
// 2. Delete files
var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dblockFiles)
File.Delete(file);
// Destroy some source data (also destroys the metadata)
if (deleteLargeFile)
File.Delete(Path.Combine(DATAFOLDER, "a"));
else
File.Delete(Path.Combine(DATAFOLDER, "b"));
// 3. Try to repair the remote destination
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
var res = c.Repair();
Assert.IsTrue(res.Warnings.Where(x => x.Contains("purge-broken-files")).Count() == 1, "Repair should warn about missing block data.");
// Ensure that the partial repair did not clean up fully
Assert.Throws<RemoteListVerificationException>(() => c.Test());
// Purge broken files should be possible
TestUtils.AssertResults(c.PurgeBrokenFiles(null));
TestUtils.AssertResults(c.Test(int.MaxValue));
var files = c.List("*").Files.ToList();
Console.WriteLine("Files: " + string.Join(", ", files.Select(x => x.Path)));
}
// 4. Verify that restore works
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { restore_path = RESTOREFOLDER }), null))
TestUtils.AssertResults(c.Restore(null));
Assert.That(File.Exists(Path.Combine(RESTOREFOLDER, "a")), Is.EqualTo(!deleteLargeFile), "Restored file 'a' should be restored.");
Assert.That(File.Exists(Path.Combine(RESTOREFOLDER, "b")), Is.EqualTo(true), "File 'b' should always be restored.");
if (!deleteLargeFile)
Assert.That(File.ReadAllBytes(Path.Combine(RESTOREFOLDER, "a")), Is.EqualTo(data), "Restored file 'a' should be the same as the original.");
else
Assert.That(File.ReadAllBytes(Path.Combine(RESTOREFOLDER, "b")), Is.EqualTo(data.AsSpan().Slice(0, 1024).ToArray()), "Restored file 'b' should be the same as the original.");
}
[Test]
[Category("Targeted")]
[TestCase(0)] // Large file only
[TestCase(1)] // Small file only
[TestCase(2)] // Both files
[TestCase(3)] // Metadata only
public void TestPartialRepairPossibleWithPartialData(int scenario)
{
var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true });
// 1. Make some shared data 1kb blocks, 32 bytes hash, 3 blocklists, and 1028 bytes extra
var data = new byte[1024 * 32 * 3 + 1028];
Random.Shared.NextBytes(data);
File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data);
File.WriteAllBytes(Path.Combine(DATAFOLDER, "b"), data.AsSpan().Slice(0, 1024).ToArray());
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
TestUtils.AssertResults(c.Backup([DATAFOLDER]));
// 2. Delete files
var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList();
foreach (var file in dblockFiles)
File.Delete(file);
var corruptLargeFile = scenario == 0 || scenario == 2;
var corruptSmallFile = scenario == 1 || scenario == 2;
var corruptBothFiles = corruptLargeFile && corruptSmallFile;
var destroyMetadata = scenario == 3;
// Destroy some source data (keep metadata)
var prev = data[1];
data[1] = 1;
if (corruptLargeFile)
{
var timestamp = File.GetLastWriteTime(Path.Combine(DATAFOLDER, "a"));
File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data);
File.SetLastWriteTime(Path.Combine(DATAFOLDER, "a"), timestamp);
}
if (corruptSmallFile)
{
var timestamp = File.GetLastWriteTime(Path.Combine(DATAFOLDER, "b"));
File.WriteAllBytes(Path.Combine(DATAFOLDER, "b"), data.AsSpan().Slice(0, 1024).ToArray());
File.SetLastWriteTime(Path.Combine(DATAFOLDER, "b"), timestamp);
}
data[1] = prev;
// Make sure that destroying the metadata does not destroy the files
if (destroyMetadata)
{
File.SetLastWriteTime(Path.Combine(DATAFOLDER, "a"), DateTime.UtcNow.AddSeconds(1));
File.SetLastWriteTime(Path.Combine(DATAFOLDER, "b"), DateTime.UtcNow.AddSeconds(1.1));
}
// 3. Try to repair the remote destination
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
var res = c.Repair();
if (corruptBothFiles || destroyMetadata)
{
Assert.IsTrue(res.Warnings.Where(x => x.Contains("purge-broken-files")).Count() == 1, "Repair should warn about missing block data.");
// Ensure that the partial repair did not clean up fully
Assert.Throws<RemoteListVerificationException>(() => c.Test());
}
else
{
TestUtils.AssertResults(res);
TestUtils.AssertResults(c.Test(int.MaxValue));
}
// Purge broken files should be possible
TestUtils.AssertResults(c.PurgeBrokenFiles(null));
TestUtils.AssertResults(c.Test(int.MaxValue));
}
// 4. Verify that restore works
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { restore_path = RESTOREFOLDER }), null))
{
var res = c.Restore(null);
if (corruptBothFiles)
{
// We get a warning because neither file can be restored
Assert.That(res.Errors.Count(), Is.EqualTo(0), "Restore should not fail.");
}
else
{
TestUtils.AssertResults(res);
}
}
Assert.That(File.Exists(Path.Combine(RESTOREFOLDER, "a")), Is.EqualTo(!corruptBothFiles), "File 'a' should be restored.");
Assert.That(File.Exists(Path.Combine(RESTOREFOLDER, "b")), Is.EqualTo(!corruptBothFiles), "File 'b' should be restored.");
if (!corruptLargeFile)
Assert.That(Convert.ToBase64String(File.ReadAllBytes(Path.Combine(RESTOREFOLDER, "a"))), Is.EqualTo(Convert.ToBase64String(data)), "Restored file 'a' should be the same as the original.");
else if (!corruptBothFiles)
Assert.That(Convert.ToBase64String(File.ReadAllBytes(Path.Combine(RESTOREFOLDER, "b"))), Is.EqualTo(Convert.ToBase64String(data.AsSpan().Slice(0, 1024))), "Restored file 'b' should be the same as the original.");
}
[Test]
[Category("Targeted")]
[TestCase(1)]
[TestCase(5)]
[TestCase(10)]
public void TestRepairPossibleWithMultipleDblockFiles(int dblocksToDelete)
{
var testopts = TestOptions.Expand(new
{
blocksize = "1kb",
no_encryption = true,
rebuild_missing_dblock_files = true,
zip_compression_level = 0,
dblock_size = "10kb"
});
// 1. Make some shared data 1kb blocks, 32 bytes hash, 3 blocklists, and 1028 bytes extra
var data = new byte[1024 * 32 * 3 + 1028];
Random.Shared.NextBytes(data);
File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data);
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
TestUtils.AssertResults(c.Backup([DATAFOLDER]));
// 2. Delete files
var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToArray();
Random.Shared.Shuffle(dblockFiles);
foreach (var file in dblockFiles.Take(dblocksToDelete))
File.Delete(file);
// 3. Try to repair the remote destination
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
{
TestUtils.AssertResults(c.Repair());
TestUtils.AssertResults(c.Test());
}
// 4. Verify that restore works
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { restore_path = RESTOREFOLDER }), null))
TestUtils.AssertResults(c.Restore(null));
TestUtils.AssertDirectoryTreesAreEquivalent(DATAFOLDER, RESTOREFOLDER, !Library.Utility.Utility.ParseBoolOption(testopts, "skip-metadata"), "Restore");
}
}
}
Duplicati/UnitTest/TestUtils.cs
+18 -2
View File
@@ -296,8 +296,6 @@ namespace Duplicati.UnitTest
{
operation = ((Library.Main.OperationMode)operationProperty.GetValue(results)).ToString();
}
Assert.AreEqual(0, results.Errors.Count(), "{0} errors:\n{1}", operation, string.Join("\n", results.Errors));
Assert.AreEqual(0, results.Warnings.Count(), "{0} warnings:\n{1}", operation, string.Join("\n", results.Warnings));
if (results is ITestResults testResults)
{
@@ -322,6 +320,24 @@ namespace Duplicati.UnitTest
throw new TestVerificationException(sb.ToString());
}
}
if (results.Errors.Count() != 0)
{
var sb = new StringBuilder();
sb.AppendLine($"Errors - {operation}:");
foreach (var e in results.Errors)
sb.AppendLine(e.ToString());
throw new TestVerificationException(sb.ToString());
}
if (results.Warnings.Count() != 0)
{
var sb = new StringBuilder();
sb.AppendLine($"Warnings - {operation}:");
foreach (var w in results.Warnings)
sb.AppendLine(w.ToString());
throw new TestVerificationException(sb.ToString());
}
}
public static string DumpTable(IDbCommand cmd, string table, string where = null)