diff --git a/Duplicati/Library/Interface/ResultInterfaces.cs b/Duplicati/Library/Interface/ResultInterfaces.cs index 3c8c8c1a5..a0d304312 100644 --- a/Duplicati/Library/Interface/ResultInterfaces.cs +++ b/Duplicati/Library/Interface/ResultInterfaces.cs @@ -494,6 +494,7 @@ namespace Duplicati.Library.Interface { long RemovedFileCount { get; } long RemovedFileSize { get; } + long UpdatedFileCount { get; } long RewrittenFileLists { get; } ICompactResults CompactResults { get; } } diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index c66b5d2a0..acd07f918 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -469,17 +469,13 @@ namespace Duplicati.Library.Main.Database // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{volidstable}"" (""ID"" INTEGER PRIMARY KEY)")); - deletecmd.SetCommandAndParameters(FormatInvariant($@"INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") VALUES (@Id)")); - foreach (var name in names) - { - var volumeid = GetRemoteVolumeID(name, tr.Parent); - deletecmd.SetParameterValue("@Id", volumeid) - .ExecuteNonQuery(); - } + deletecmd.SetCommandAndParameters(FormatInvariant($@"INSERT OR IGNORE INTO ""{volidstable}"" SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@VolumeNames)")) + .ExpandInClauseParameter("@VolumeNames", names.ToArray()) + .ExecuteNonQuery(); + var volIdsSubQuery = FormatInvariant($@"SELECT ""ID"" FROM ""{volidstable}"" "); deletecmd.Parameters.Clear(); - var bsIdsSubQuery = FormatInvariant(@$" SELECT DISTINCT ""BlocksetEntry"".""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" IN ({volIdsSubQuery}) @@ -661,9 +657,9 @@ AND Fileset.ID NOT IN return res; } - public bool IsFilesetFullBackup(DateTime filesetTime) + public bool IsFilesetFullBackup(DateTime filesetTime, IDbTransaction? transaction) { - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(transaction)) using (var rd = cmd.SetCommandAndParameters($@"SELECT ""IsFullBackup"" FROM ""Fileset"" WHERE ""Timestamp"" = @Timestamp").SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(filesetTime)).ExecuteReader()) { if (!rd.Read()) @@ -1504,8 +1500,19 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER } } + /// + /// Adds a link between an index volume and a block volume. + /// + /// The ID of the index volume. + /// The ID of the block volume. + /// An optional transaction. public void AddIndexBlockLink(long indexVolumeID, long blockVolumeID, IDbTransaction transaction) { + if (indexVolumeID <= 0) + throw new ArgumentOutOfRangeException(nameof(indexVolumeID), "Index volume ID must be greater than 0."); + if (blockVolumeID <= 0) + throw new ArgumentOutOfRangeException(nameof(blockVolumeID), "Block volume ID must be greater than 0."); + m_insertIndexBlockLink.SetParameterValue("@IndexVolumeId", indexVolumeID) .SetParameterValue("@BlockVolumeId", blockVolumeID) .ExecuteNonQuery(transaction); diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index 678c8e988..8a2b6ad18 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -28,17 +28,17 @@ using System.Linq; namespace Duplicati.Library.Main.Database { - internal class LocalListBrokenFilesDatabase : LocalDatabase - { - private static readonly string BLOCK_VOLUME_IDS = FormatInvariant($@"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks.ToString()}'"); + internal class LocalListBrokenFilesDatabase : LocalDatabase + { + private static readonly string BLOCK_VOLUME_IDS = FormatInvariant($@"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks.ToString()}'"); - // Invalid blocksets include those that: - // - Have BlocksetEntries with unknown/invalid blocks (meaning the data to rebuild the blockset isn't available) - // - Invalid blocks include those that appear to be in non-Blocks volumes (e.g., are listed as being in an Index or Files volume) or that appear in an unknown volume (-1) - // - Have BlocklistHash entries with unknown/invalid blocks (meaning the data which defines the list of hashes that makes up the blockset isn't available) - // - Are defined in the Blockset table but have no entries in the BlocksetEntries table (this can happen during recreate if Files volumes reference blocksets that are not found in any Index files) - // However, blocksets with a length of 0 are excluded from this check, as the corresponding blocks for these are not needed. - private static readonly string BROKEN_FILE_IDS = FormatInvariant($@" + // Invalid blocksets include those that: + // - Have BlocksetEntries with unknown/invalid blocks (meaning the data to rebuild the blockset isn't available) + // - Invalid blocks include those that appear to be in non-Blocks volumes (e.g., are listed as being in an Index or Files volume) or that appear in an unknown volume (-1) + // - Have BlocklistHash entries with unknown/invalid blocks (meaning the data which defines the list of hashes that makes up the blockset isn't available) + // - Are defined in the Blockset table but have no entries in the BlocksetEntries table (this can happen during recreate if Files volumes reference blocksets that are not found in any Index files) + // However, blocksets with a length of 0 are excluded from this check, as the corresponding blocks for these are not needed. + private static readonly string BROKEN_FILE_IDS = FormatInvariant($@" SELECT DISTINCT ""ID"" FROM ( SELECT ""ID"" AS ""ID"", ""BlocksetID"" AS ""BlocksetID"" FROM ""FileLookup"" WHERE ""BlocksetID"" != {FOLDER_BLOCKSET_ID} AND ""BlocksetID"" != {SYMLINK_BLOCKSET_ID} UNION @@ -61,99 +61,188 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN WHERE ""BlocksetID"" NOT IN (SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" == 0) ) "); - private static readonly string BROKEN_FILE_SETS = FormatInvariant($@"SELECT DISTINCT ""B"".""Timestamp"", ""A"".""FilesetID"", COUNT(""A"".""FileID"") AS ""FileCount"" FROM ""FilesetEntry"" A, ""Fileset"" B WHERE ""A"".""FilesetID"" = ""B"".""ID"" AND ""A"".""FileID"" IN ({BROKEN_FILE_IDS})"); + private static readonly string BROKEN_FILE_SETS = FormatInvariant($@"SELECT DISTINCT ""B"".""Timestamp"", ""A"".""FilesetID"", COUNT(""A"".""FileID"") AS ""FileCount"" FROM ""FilesetEntry"" A, ""Fileset"" B WHERE ""A"".""FilesetID"" = ""B"".""ID"" AND ""A"".""FileID"" IN ({BROKEN_FILE_IDS})"); - private static readonly string BROKEN_FILE_NAMES = FormatInvariant($@"SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A LEFT JOIN ""Blockset"" B ON (""A"".""BlocksetID"" = ""B"".""ID"") WHERE ""A"".""ID"" IN ({BROKEN_FILE_IDS}) AND ""A"".""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId)"); + private static readonly string BROKEN_FILE_NAMES = FormatInvariant($@"SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A LEFT JOIN ""Blockset"" B ON (""A"".""BlocksetID"" = ""B"".""ID"") WHERE ""A"".""ID"" IN ({BROKEN_FILE_IDS}) AND ""A"".""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId)"); - private static string INSERT_BROKEN_IDS(string tablename, string IDfieldname) => FormatInvariant($@"INSERT INTO ""{tablename}"" (""{IDfieldname}"") {BROKEN_FILE_IDS} AND ""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId)"); + private static string INSERT_BROKEN_IDS(string tablename, string IDfieldname) => FormatInvariant($@"INSERT INTO ""{tablename}"" (""{IDfieldname}"") {BROKEN_FILE_IDS} AND ""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId)"); - public LocalListBrokenFilesDatabase(string path, long pagecachesize) - : base(path, "ListBrokenFiles", false, pagecachesize) - { - ShouldCloseConnection = true; - } - - public LocalListBrokenFilesDatabase(LocalDatabase parent) - : base(parent) - { - ShouldCloseConnection = false; - } - - public IEnumerable> GetBrokenFilesets(DateTime time, long[] versions, IDbTransaction transaction) - { - var query = BROKEN_FILE_SETS; - var clause = GetFilelistWhereClause(time, versions); - if (!string.IsNullOrWhiteSpace(clause.Item1)) - query += FormatInvariant($@" AND ""A"".""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" {clause.Item1})"); - - query += @" GROUP BY ""A"".""FilesetID"""; - - using (var cmd = Connection.CreateCommand(transaction)) - foreach (var rd in cmd.SetCommandAndParameters(query).SetParameterValues(clause.Values).ExecuteReaderEnumerable()) - if (!rd.IsDBNull(0)) - yield return new Tuple(ParseFromEpochSeconds(rd.ConvertValueToInt64(0, 0)), rd.ConvertValueToInt64(1, -1), rd.ConvertValueToInt64(2, 0)); - } - - public IEnumerable> GetBrokenFilenames(long filesetid, IDbTransaction transaction) - { - using (var cmd = Connection.CreateCommand(transaction, BROKEN_FILE_NAMES).SetParameterValue("@FilesetId", filesetid)) - foreach (var rd in cmd.ExecuteReaderEnumerable()) - if (!rd.IsDBNull(0)) - yield return new Tuple(rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), rd.ConvertValueToInt64(1)); - } - - public void InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname, IDbTransaction transaction) - { - using var cmd = Connection.CreateCommand(transaction, INSERT_BROKEN_IDS(tablename, IDfieldname)) - .SetParameterValue("@FilesetId", filesetid); - cmd.ExecuteNonQuery(); - } - - public void RemoveMissingBlocks(IEnumerable names, IDbTransaction transaction) - { - if (names == null || !names.Any()) return; - if (transaction == null) - throw new Exception("This function cannot be called when not in a transaction, as it leaves the database in an inconsistent state"); - - using (var deletecmd = m_connection.CreateCommand(transaction)) - { - var temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var volidstable = "DelVolSetIds-" + temptransguid; - - // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. - deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{volidstable}"" (""ID"" INTEGER PRIMARY KEY)")); - - using (var tmptable = new TemporaryDbValueList(m_connection, transaction, names)) - deletecmd.SetCommandAndParameters(FormatInvariant($@"INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)")) - .ExpandInClauseParameter("@Names", tmptable) - .ExecuteNonQuery(); - - var volIdsSubQuery = FormatInvariant($@"SELECT ""ID"" FROM ""{volidstable}"" "); - deletecmd.Parameters.Clear(); - - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); - - // Clean up temp tables for subqueries. We truncate content and then try to delete. - // Drop in try-block, as it fails in nested transactions (SQLite problem) - // SQLite.SQLiteException (0x80004005): database table is locked - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{volidstable}"" ")); - try + public LocalListBrokenFilesDatabase(string path, long pagecachesize) + : base(path, "ListBrokenFiles", false, pagecachesize) { - deletecmd.CommandTimeout = 2; - deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{volidstable}"" ")); + ShouldCloseConnection = true; } - catch { /* Ignore, will be deleted on close anyway. */ } - } - } - public long GetFilesetFileCount(long filesetid, IDbTransaction transaction) - { - using var cmd = m_connection.CreateCommand(transaction, @"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId") - .SetParameterValue("@FilesetId", filesetid); - return cmd.ExecuteScalarInt64(0); + public LocalListBrokenFilesDatabase(LocalDatabase parent) + : base(parent) + { + ShouldCloseConnection = false; + } + + public IEnumerable<(DateTime FilesetTime, long FilesetID, long RemoveFileCount)> GetBrokenFilesets(DateTime time, long[] versions, IDbTransaction transaction) + { + var query = BROKEN_FILE_SETS; + var clause = GetFilelistWhereClause(time, versions); + if (!string.IsNullOrWhiteSpace(clause.Item1)) + query += FormatInvariant($@" AND ""A"".""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" {clause.Item1})"); + + query += @" GROUP BY ""A"".""FilesetID"""; + + using (var cmd = Connection.CreateCommand(transaction)) + foreach (var rd in cmd.SetCommandAndParameters(query).SetParameterValues(clause.Values).ExecuteReaderEnumerable()) + if (!rd.IsDBNull(0)) + yield return (ParseFromEpochSeconds(rd.ConvertValueToInt64(0, 0)), rd.ConvertValueToInt64(1, -1), rd.ConvertValueToInt64(2, 0)); + } + + public IEnumerable> GetBrokenFilenames(long filesetid, IDbTransaction transaction) + { + using (var cmd = Connection.CreateCommand(transaction, BROKEN_FILE_NAMES).SetParameterValue("@FilesetId", filesetid)) + foreach (var rd in cmd.ExecuteReaderEnumerable()) + if (!rd.IsDBNull(0)) + yield return new Tuple(rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), rd.ConvertValueToInt64(1)); + } + + /// + /// Returns all index files that are orphaned, i.e., not referenced by any block files. + /// + /// Transaction to use for the query. + /// >All index files that are orphaned. + public IEnumerable GetOrphanedIndexFiles(IDbTransaction transaction) + { + using var cmd = Connection.CreateCommand(transaction, FormatInvariant($@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Index.ToString()}' AND ""ID"" NOT IN (SELECT ""IndexVolumeID"" FROM ""IndexBlockLink"")")); + + foreach (var rd in cmd.ExecuteReaderEnumerable()) + yield return new RemoteVolume( + rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), + rd.ConvertValueToString(1) ?? throw new Exception("Hash was null"), + rd.ConvertValueToInt64(2, -1) + ); + } + + /// + /// Inserts the broken file IDs into the given table. The table must have a single column with the same name as the ID field name. + /// + /// The filset id for the current operation + /// The name of the table to insert into + /// The name of the ID field in the table + /// The transaction to use for the query + public void InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname, IDbTransaction transaction) + { + using var cmd = Connection.CreateCommand(transaction, INSERT_BROKEN_IDS(tablename, IDfieldname)) + .SetParameterValue("@FilesetId", filesetid); + cmd.ExecuteNonQuery(); + } + + /// + /// Returns the ID of an empty metadata blockset. If no empty blockset is found, it returns the ID of the smallest blockset that is not in the given block volume IDs. + /// If no such blockset is found, it returns -1. + /// + /// The volume ids to ignore when searching for a suitable metadata block + /// The hash of the empty blockset + /// The size of the empty blockset + /// The transaction to use for the query + /// The ID of the empty metadata blockset, or -1 if no suitable blockset is found + public long GetEmptyMetadataBlocksetId(IEnumerable blockVolumeIds, string emptyHash, long emptyHashSize, IDbTransaction transaction) + { + using var cmd = Connection.CreateCommand(transaction, @"SELECT ""ID"" FROM ""Blockset"" WHERE ""FullHash"" = @EmptyHash AND ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))") + .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) + .SetParameterValue("@EmptyHash", emptyHash) + .SetParameterValue("@EmptyHashSize", emptyHashSize); + + var res = cmd.ExecuteScalarInt64(-1); + + // No empty block found, try to find a zero-length block instead + if (res < 0 && emptyHashSize != 0) + res = cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))") + .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) + .SetParameterValue("@EmptyHashSize", 0) + .ExecuteScalarInt64(-1); + + // No empty block found, pick the smallest one + if (res < 0) + res = cmd.SetCommandAndParameters(@"SELECT ""Blockset"".""ID"" FROM ""BlocksetEntry"", ""Blockset"", ""Metadataset"", ""Block"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) ORDER BY ""Blockset"".""Length"" ASC LIMIT 1") + .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) + .ExecuteScalarInt64(-1); + + return res; + } + + /// + /// Replaces the metadata blockset ID in the Metadataset table with the empty blockset ID for all fileset entries that are not in any block volume. + /// This is used to clean up the metadata blocksets that are now missing. + /// + /// The filesetId to target + /// The empty blockset ID to replace with + /// The transaction to use for the query + /// The number of rows affected + public int ReplaceMetadata(long filesetId, long emptyBlocksetId, IDbTransaction transaction) + { + using var cmd = m_connection.CreateCommand(transaction, @" +UPDATE ""Metadataset"" +SET ""BlocksetID"" = @EmptyBlocksetID +WHERE + ""ID"" IN ( + SELECT ""FileLookup"".""MetadataID"" + FROM ""FileLookup"", ""FilesetEntry"" + WHERE + ""FilesetEntry"".""FilesetId"" = @FilesetId + AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" + ) + AND ""BlocksetID"" NOT IN ( + SELECT ""BlocksetID"" + FROM ""BlocksetEntry"", ""Block"" + WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + )") + .SetParameterValue("@EmptyBlocksetId", emptyBlocksetId) + .SetParameterValue("@FilesetId", filesetId); + return cmd.ExecuteNonQuery(); + } + + public void RemoveMissingBlocks(IEnumerable names, IDbTransaction transaction) + { + if (names == null || !names.Any()) return; + if (transaction == null) + throw new Exception("This function cannot be called when not in a transaction, as it leaves the database in an inconsistent state"); + + using (var deletecmd = m_connection.CreateCommand(transaction)) + { + var temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + var volidstable = "DelVolSetIds-" + temptransguid; + + // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. + deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{volidstable}"" (""ID"" INTEGER PRIMARY KEY)")); + + using (var tmptable = new TemporaryDbValueList(m_connection, transaction, names)) + deletecmd.SetCommandAndParameters(FormatInvariant($@"INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)")) + .ExpandInClauseParameter("@Names", tmptable) + .ExecuteNonQuery(); + + var volIdsSubQuery = FormatInvariant($@"SELECT ""ID"" FROM ""{volidstable}"" "); + deletecmd.Parameters.Clear(); + + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery})")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); + + // Clean up temp tables for subqueries. We truncate content and then try to delete. + // Drop in try-block, as it fails in nested transactions (SQLite problem) + // SQLite.SQLiteException (0x80004005): database table is locked + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{volidstable}"" ")); + try + { + deletecmd.CommandTimeout = 2; + deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{volidstable}"" ")); + } + catch { /* Ignore, will be deleted on close anyway. */ } + } + } + + public long GetFilesetFileCount(long filesetid, IDbTransaction transaction) + { + using var cmd = m_connection.CreateCommand(transaction, @"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId") + .SetParameterValue("@FilesetId", filesetid); + return cmd.ExecuteScalarInt64(0); + } } - } } diff --git a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs index 32c24ee33..e334ffb3d 100644 --- a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs @@ -204,7 +204,7 @@ namespace Duplicati.Library.Main.Database var cmd = asNew ? m_insertCurrentElementCommand : m_insertPreviousElementCommand; cmd.SetParameterValue("@Path", path); cmd.SetParameterValue("@FileHash", filehash); - cmd.SetParameterValue(@"MetaHash", metahash); + cmd.SetParameterValue("@MetaHash", metahash); cmd.SetParameterValue("@Size", size); cmd.SetParameterValue("@Type", (int)type); cmd.ExecuteNonQuery(); diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index d80e7d590..2bdb15209 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -72,9 +72,10 @@ namespace Duplicati.Library.Main.Database long ParentID { get; } long RemovedFileCount { get; } long RemovedFileSize { get; } + long UpdatedFileCount { get; } void ApplyFilter(Library.Utility.IFilter filter); - void ApplyFilter(Action filtercommand); + void ApplyFilter(Func filtercommand); Tuple ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup); IEnumerable> ListAllDeletedFiles(); } @@ -89,6 +90,7 @@ namespace Duplicati.Library.Main.Database public long ParentID { get; private set; } public long RemovedFileCount { get; private set; } public long RemovedFileSize { get; private set; } + public long UpdatedFileCount { get; private set; } public TemporaryFileset(long parentid, LocalPurgeDatabase parentdb, IDbConnection connection, IDbTransaction transaction) { @@ -102,12 +104,13 @@ namespace Duplicati.Library.Main.Database cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tablename}"" (""FileID"" INTEGER PRIMARY KEY) ")); } - public void ApplyFilter(Action filtercommand) + public void ApplyFilter(Func filtercommand) { + int updated; using (var cmd = m_connection.CreateCommand(m_transaction)) - filtercommand(cmd, ParentID, m_tablename); + updated = filtercommand(cmd, ParentID, m_tablename); - PostFilterChecks(); + PostFilterChecks(updated); } public void ApplyFilter(Library.Utility.IFilter filter) @@ -156,15 +159,24 @@ namespace Duplicati.Library.Main.Database } } - PostFilterChecks(); + PostFilterChecks(0); } - private void PostFilterChecks() + private void PostFilterChecks(int updated) { using (var cmd = m_connection.CreateCommand(m_transaction)) { + UpdatedFileCount = updated; RemovedFileCount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""{m_tablename}"""), 0); - RemovedFileSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""C"".""Length"") FROM ""{m_tablename}"" A, ""FileLookup"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" "), 0); + RemovedFileSize = cmd.ExecuteScalarInt64(FormatInvariant($@" + SELECT + SUM(""C"".""Length"") + FROM + ""{m_tablename}"" A, ""FileLookup"" B, ""Blockset"" C, ""Metadataset"" D + WHERE + ""A"".""FileID"" = ""B"".""ID"" + AND(""B"".""BlocksetID"" = ""C"".""ID"" OR(""B"".""MetadataID"" = ""D"".""ID"" AND ""D"".""BlocksetID"" = ""C"".""ID"")) + "), 0); var filesetcount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = {ParentID}"), 0); if (filesetcount == RemovedFileCount) throw new Interface.UserInformationException($"Refusing to purge {RemovedFileCount} files from fileset with ID {ParentID}, as that would remove the entire fileset.\nTo delete a fileset, use the \"delete\" command.", "PurgeWouldRemoveEntireFileset"); diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index f898a1ab3..758c084b1 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -35,27 +35,87 @@ namespace Duplicati.Library.Main.Database /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalRepairDatabase)); + /// + /// Creates a new local repair database + /// + /// The path to the database + /// The page cache size public LocalRepairDatabase(string path, long pagecachesize) : base(path, "Repair", true, pagecachesize) { } - public long GetFilesetIdFromRemotename(string filelist) + /// + /// Gets the fileset ID from the remote name + /// + /// The remote name of the fileset + /// An optional transaction + /// >The fileset ID + public (long FilesetId, DateTime Time, bool IsFullBackup) GetFilesetFromRemotename(string filelist, IDbTransaction? transaction) { - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(transaction)) { - var filesetid = cmd.SetCommandAndParameters(@"SELECT ""Fileset"".""ID"" FROM ""Fileset"",""RemoteVolume"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""RemoteVolume"".""Name"" = @Name") + var rd = cmd.SetCommandAndParameters(@"SELECT ""Fileset"".""ID"", ""Fileset"".""Timestamp"", ""Fileset"".""IsFullBackup"" FROM ""Fileset"",""RemoteVolume"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""RemoteVolume"".""Name"" = @Name") .SetParameterValue("@Name", filelist) - .ExecuteScalarInt64(-1); + .ExecuteReader(); - if (filesetid == -1) + if (!rd.Read()) throw new Exception($"No such remote file: {filelist}"); - return filesetid; + return (rd.ConvertValueToInt64(0, -1), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime(), rd.GetInt32(2) == BackupType.FULL_BACKUP); } } + /// + /// Moves entries in the FilesetEntry table from previous fileset to current fileset + /// + /// Current fileset ID + /// Source fileset ID + /// The transaction + public void MoveFilesFromFileset(long filesetid, long prevFilesetId, IDbTransaction transaction) + { + if (filesetid <= 0) + throw new ArgumentException("filesetid must be > 0"); + + if (prevFilesetId <= 0) + throw new ArgumentException("prevId must be > 0"); + + using var cmd = m_connection.CreateCommand(transaction); + cmd.SetCommandAndParameters(@"UPDATE ""FilesetEntry"" SET ""FilesetID"" = @CurrentFilesetId WHERE ""FilesetID"" = @PreviousFilesetId ") + .SetParameterValue("@CurrentFilesetId", filesetid) + .SetParameterValue("@PreviousFilesetId", prevFilesetId) + .ExecuteNonQuery(); + } + + /// + /// Deletes a fileset from the database + /// + /// The fileset ID + /// The transaction + public void DeleteFileset(long filesetid, IDbTransaction transaction) + { + using var cmd = m_connection.CreateCommand(transaction); + cmd.SetCommandAndParameters(@"DELETE FROM ""Fileset"" WHERE ""ID"" = @FilesetId") + .SetParameterValue("@FilesetId", filesetid) + .ExecuteNonQuery(); + } + + /// + /// Gets the list of index files that reference a given block file + /// + /// The block file ID + /// >A list of index file IDs + public IEnumerable GetIndexFilesReferencingBlockFile(long blockfileid, IDbTransaction? transaction) + { + using var cmd = m_connection.CreateCommand(transaction, @"SELECT ""RemoteVolume"".""Name"" FROM ""RemoteVolume"", ""IndexBlockLink"" WHERE ""IndexBlockLink"".""BlockVolumeID"" = @BlockFileId AND ""RemoteVolume"".""ID"" = ""IndexBlockLink"".""IndexVolumeID"" AND ""RemoteVolume"".""Type"" = @Type") + .SetParameterValue("@BlockFileId", blockfileid) + .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()); + + foreach (var rd in cmd.ExecuteReaderEnumerable()) + yield return rd.ConvertValueToString(0) ?? throw new Exception("RemoteVolume name was null"); + } + /// /// Gets a list of filesets that are missing files /// @@ -88,63 +148,6 @@ namespace Duplicati.Library.Main.Database .ExecuteNonQuery(); } - public interface IBlockSource - { - string File { get; } - long Offset { get; } - } - - public interface IBlockWithSources : IBlock - { - IEnumerable Sources { get; } - } - - private class BlockWithSources : Block, IBlockWithSources - { - private class BlockSource : IBlockSource - { - public string File { get; private set; } - public long Offset { get; private set; } - - public BlockSource(string file, long offset) - { - this.File = file; - this.Offset = offset; - - } - } - - private readonly IDataReader m_rd; - public bool Done { get; private set; } - - public BlockWithSources(IDataReader rd) - : base(rd.ConvertValueToString(0) ?? throw new Exception("Hash value was null"), rd.ConvertValueToInt64(1)) - { - m_rd = rd; - Done = !m_rd.Read(); - } - - public IEnumerable Sources - { - get - { - if (Done) - yield break; - - var cur = new BlockSource(m_rd.ConvertValueToString(2) ?? "", m_rd.ConvertValueToInt64(3)); - var file = cur.File; - - while (!Done && cur.File == file) - { - yield return cur; - Done = m_rd.Read(); - if (!Done) - cur = new BlockSource(m_rd.ConvertValueToString(2) ?? "", m_rd.ConvertValueToInt64(3)); - } - } - } - } - private class RemoteVolume : IRemoteVolume { public string Name { get; private set; } @@ -159,39 +162,152 @@ namespace Duplicati.Library.Main.Database } } - public IEnumerable GetBlockVolumesFromIndexName(string name) + public IEnumerable GetBlockVolumesFromIndexName(string indexName) { using var cmd = m_connection.CreateCommand(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"" WHERE ""IndexVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = @Name)) ") - .SetParameterValue("@Name", name); + .SetParameterValue("@Name", indexName); foreach (var rd in cmd.ExecuteReaderEnumerable()) yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); } + /// + /// A single block with source data for repair + /// + /// The block hash + /// The block size + /// The file that contains the block + /// The offset of the block in the file + public sealed record BlockWithSourceData(string Hash, long Size, string File, long Offset); + + /// + /// A single block with metadata source data for repair + /// + /// The block hash + /// The block size + /// The path of the file or directory that contains the block + public sealed record BlockWithMetadataSourceData(string Hash, long Size, string Path); + + /// + /// A single blocklist hash entry + /// + /// The blockset id + /// The hash of the blocklist entry (when done) + /// The total length of the blockset + /// The index of the blocklist hash + /// The index of the block in the blockset + /// The hash of the entry + public sealed record BlocklistHashesEntry( + long BlocksetId, + string BlocklistHash, + long BlocklistHashLength, + long BlocklistHashIndex, + int Index, + string Hash); + + /// + /// Helper interface for the missing block list + /// public interface IMissingBlockList : IDisposable { - bool SetBlockRestored(string hash, long size); - IEnumerable GetSourceFilesWithBlocks(long blocksize); - IEnumerable> GetMissingBlocks(); + /// + /// Registers a block as restored + /// + /// The block hash + /// The block size + /// The volume ID of the new target volume + /// The restored blocks + bool SetBlockRestored(string hash, long size, long volumeId); + /// + /// Gets the list of files that contains missing blocks + /// + /// The blocksize setting + /// A list of files with missing blocks + IEnumerable GetSourceFilesWithBlocks(long blocksize); + /// + /// Gets a list for filesystem entries that contain missing blocks in metadata + /// + /// A list of filesystem entries with missing blocks + IEnumerable GetSourceItemsWithMetadataBlocks(); + /// + /// Gets missing blocklist hashes + /// + /// The number of hashes for each block + /// >A list of blocklist hashes + IEnumerable GetBlocklistHashes(long hashesPerBlock); + /// + /// Gets the number of missing blocks + /// + /// >The number of missing blocks + long GetMissingBlockCount(); + /// + /// Gets all the filesets that are affected by missing blocks + /// + /// >A list of filesets IEnumerable GetFilesetsUsingMissingBlocks(); + /// + /// Gets a list of remote files that may contain missing blocks + /// + /// >A list of remote files IEnumerable GetMissingBlockSources(); } + /// + /// Implementation of the missing block list + /// private class MissingBlockList : IMissingBlockList { + /// + /// The connection to the database + /// private readonly IDbConnection m_connection; - private readonly TemporaryTransactionWrapper m_transaction; - private IDbCommand m_insertCommand; - private string m_tablename; + /// + /// The transaction to use + /// + private readonly ReusableTransaction m_transaction; + /// + /// The insert command to use for restoring blocks + /// + private readonly IDbCommand m_insertCommand; + /// + /// The command to copy blocks into the duplicate block table + /// + private readonly IDbCommand m_copyIntoDuplicatedBlocks; + /// + /// The command to assign blocks to a new volume + /// + private readonly IDbCommand m_assignBlocksToNewVolume; + /// + /// The name of the temporary table + /// + private readonly string m_tablename; + /// + /// The name of the volume where blocks are missing + /// private readonly string m_volumename; + /// + /// The query to use for getting missing blocks + /// + private readonly string m_missingBlocksQuery; - public MissingBlockList(string volumename, IDbConnection connection, IDbTransaction? transaction) + /// + /// Whether the object has been disposed + /// + private bool m_isDisposed = false; + + /// + /// Creates a new missing block list + /// + /// The name of the volume with missing blocks + /// The connection to the database + /// The transaction to use + public MissingBlockList(string volumename, IDbConnection connection, ReusableTransaction transaction) { m_connection = connection; - m_transaction = new TemporaryTransactionWrapper(m_connection, transaction); + m_transaction = transaction; m_volumename = volumename; var tablename = "MissingBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using (var cmd = m_connection.CreateCommand(m_transaction.Parent)) + using (var cmd = m_connection.CreateCommand(m_transaction.Transaction)) { cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" INTEGER NOT NULL) ")); m_tablename = tablename; @@ -206,38 +322,209 @@ namespace Duplicati.Library.Main.Database cmd.ExecuteNonQuery(FormatInvariant($@"CREATE UNIQUE INDEX ""{tablename}-Ix"" ON ""{tablename}"" (""Hash"", ""Size"", ""Restored"")")); } - m_insertCommand = m_connection.CreateCommand(m_transaction.Parent, FormatInvariant($@"UPDATE ""{tablename}"" SET ""Restored"" = @NewRestoredValue WHERE ""Hash"" = @Hash AND ""Size"" = @Size AND ""Restored"" = @PreviousRestoredValue ")); + m_insertCommand = m_connection.CreateCommand(FormatInvariant($@"UPDATE ""{tablename}"" SET ""Restored"" = @NewRestoredValue WHERE ""Hash"" = @Hash AND ""Size"" = @Size AND ""Restored"" = @PreviousRestoredValue ")); + m_copyIntoDuplicatedBlocks = m_connection.CreateCommand(@"INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") SELECT ""Block"".""ID"", ""Block"".""VolumeID"" FROM ""Block"" WHERE ""Block"".""Hash"" = @Hash AND ""Block"".""Size"" = @Size"); + m_assignBlocksToNewVolume = m_connection.CreateCommand(@"UPDATE ""Block"" SET ""VolumeID"" = @TargetVolumeId WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); + m_missingBlocksQuery = FormatInvariant($@"SELECT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Restored"" = @Restored "); } - public bool SetBlockRestored(string hash, long size) + /// + public bool SetBlockRestored(string hash, long size, long targetVolumeId) { - return m_insertCommand.SetParameterValue("@NewRestoredValue", 1) + var restored = m_insertCommand.SetTransaction(m_transaction.Transaction).SetParameterValue("@NewRestoredValue", 1) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .SetParameterValue(@"PreviousRestoredValue", 0) + .SetParameterValue("@PreviousRestoredValue", 0) .ExecuteNonQuery() == 1; + + if (restored) + { + m_copyIntoDuplicatedBlocks.SetTransaction(m_transaction.Transaction) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .ExecuteNonQuery(); + + var c = m_assignBlocksToNewVolume.SetTransaction(m_transaction.Transaction) + .SetParameterValue("@TargetVolumeId", targetVolumeId) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .ExecuteNonQuery(); + + if (c != 1) + throw new Exception($"Unexpected number of updated blocks: {c} != 1"); + } + + return restored; } - public IEnumerable GetSourceFilesWithBlocks(long blocksize) + /// + public IEnumerable GetSourceFilesWithBlocks(long blocksize) { - using var cmd = m_connection.CreateCommand(m_transaction.Parent, FormatInvariant($@"SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {blocksize} FROM ""{m_tablename}"", ""Block"", ""BlocksetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" AND ""{m_tablename}"".""Restored"" = @Restored ")) - .SetParameterValue("@Restored", 0); - using (var rd = cmd.ExecuteReader()) - if (rd.Read()) - { - var bs = new BlockWithSources(rd); - while (!bs.Done) - yield return bs; - } - } - - public IEnumerable> GetMissingBlocks() - { - using var cmd = m_connection.CreateCommand(m_transaction.Parent, FormatInvariant($@"SELECT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Restored"" = @Restored ")) + using var cmd = m_connection.CreateCommand(m_transaction.Transaction, FormatInvariant($@"SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {blocksize} FROM ""{m_tablename}"", ""Block"", ""BlocksetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" AND ""{m_tablename}"".""Restored"" = @Restored ORDER BY ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {blocksize}")) .SetParameterValue("@Restored", 0); foreach (var rd in cmd.ExecuteReaderEnumerable()) - yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); + { + var hash = rd.ConvertValueToString(0) ?? throw new Exception("Hash value was null"); + var size = rd.ConvertValueToInt64(1); + var file = rd.ConvertValueToString(2) ?? throw new Exception("File value was null"); + var offset = rd.ConvertValueToInt64(3); + + yield return new BlockWithSourceData(hash, size, file, offset); + } + } + + /// + public IEnumerable GetSourceItemsWithMetadataBlocks() + { + using var cmd = m_connection.CreateCommand(m_transaction.Transaction, FormatInvariant($@"SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"" FROM ""{m_tablename}"", ""Block"", ""BlocksetEntry"", ""Metadataset"", ""File"" WHERE ""File"".""MetadataID"" == ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" AND ""{m_tablename}"".""Restored"" = @Restored ORDER BY ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path""")) + .SetParameterValue("@Restored", 0); + + foreach (var rd in cmd.ExecuteReaderEnumerable()) + { + var hash = rd.ConvertValueToString(0) ?? throw new Exception("Hash value was null"); + var size = rd.ConvertValueToInt64(1); + var path = rd.ConvertValueToString(2) ?? throw new Exception("File value was null"); + + yield return new BlockWithMetadataSourceData(hash, size, path); + } + } + + /// + public IEnumerable GetBlocklistHashes(long hashesPerBlock) + { + using var cmd = m_connection.CreateCommand(m_transaction.Transaction); + var blocklistTableName = $"BlocklistHashList-{Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray())}"; + try + { + // We need to create a snapshot as we will be updating the m_tablename table during enumeration + cmd.SetCommandAndParameters(FormatInvariant($@" + CREATE TEMPORARY TABLE ""{blocklistTableName}"" AS + SELECT + b.""Hash"" AS ""BlockHash"", + bs.""Id"" AS ""BlocksetId"", + bs.""Length"" AS ""BlocksetLength"", + bse.""Index"" / @HashesPerBlock AS ""BlocklistHashIndex"", + ( + SELECT blh.""Hash"" + FROM ""BlocklistHash"" blh + WHERE blh.""BlocksetID"" = bs.""ID"" + AND blh.""Index"" == bse.""Index"" / @HashesPerBlock + LIMIT 1 + ) AS ""BlocklistHashHash"", + bse.""Index"" AS ""BlocksetEntryIndex"" + FROM + ""BlocksetEntry"" bse + JOIN ""Block"" b ON b.""ID"" = bse.""BlockID"" + JOIN ""Blockset"" bs ON bs.""ID"" = bse.""BlocksetID"" + WHERE + EXISTS ( + SELECT 1 + FROM ""BlocklistHash"" blh + JOIN ""{m_tablename}"" mt ON mt.""Hash"" = blh.""Hash"" + WHERE blh.""BlocksetID"" = bs.""ID"" + AND mt.""Restored"" = @Restored + ) + ")) + .SetParameterValue("@HashesPerBlock", hashesPerBlock) + .SetParameterValue("@Restored", 0) + .ExecuteNonQuery(); + + cmd.SetCommandAndParameters(FormatInvariant($@" + SELECT + ""BlockHash"", + ""BlocksetId"", + ""BlocksetLength"", + ""BlocklistHashIndex"", + ""BlocklistHashHash"", + ""BlocksetEntryIndex"" + FROM ""{blocklistTableName}"" + ORDER BY ""BlocksetId"", ""BlocklistHashIndex"", ""BlocksetEntryIndex""")); + + foreach (var rd in cmd.ExecuteReaderEnumerable()) + { + var hash = rd.ConvertValueToString(0) ?? throw new Exception("Block.Hash is null"); + var blocksetId = rd.ConvertValueToInt64(1); + var length = rd.ConvertValueToInt64(2); + var blocklistHashIndex = rd.ConvertValueToInt64(3); + var blocklistHash = rd.ConvertValueToString(4) ?? throw new Exception("BlocklistHash is null"); + var index = rd.ConvertValueToInt64(5); + + yield return new BlocklistHashesEntry(blocksetId, blocklistHash, length, blocklistHashIndex, (int)index, hash); + } + } + finally + { + try { cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{blocklistTableName}"" ")); } + catch { } + } + } + + /// + public long GetMissingBlockCount() + { + using var cmd = m_connection.CreateCommand(m_transaction.Transaction, FormatInvariant($@"SELECT COUNT(*) FROM ({m_missingBlocksQuery})")) + .SetParameterValue("@Restored", 0); + + return cmd.ExecuteScalarInt64(0); + } + + /// + /// Gets the list of missing blocks + /// + /// >A list of missing blocks + public IEnumerable<(string Hash, long Size)> GetMissingBlocks() + { + using var cmd = m_connection.CreateCommand(m_transaction.Transaction, m_missingBlocksQuery) + .SetParameterValue("@Restored", 0); + + foreach (var rd in cmd.ExecuteReaderEnumerable()) + yield return (rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); + } + + /// + public long MoveBlocksToNewVolume(long targetVolumeId, long sourceVolumeId, IDbTransaction? transaction = null) + { + if (targetVolumeId <= 0) + throw new ArgumentOutOfRangeException(nameof(targetVolumeId), "Target volume ID must be greater than 0"); + + // Move the source blocks into the DuplicateBlock table + using var cmd = m_connection.CreateCommand(transaction, FormatInvariant($@" + INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") + SELECT b.""ID"", b.""VolumeID"" + FROM ""Block"" b + WHERE b.""VolumeID"" = @SourceVolumeId + AND (b.""Hash"", b.""Size"") IN ( + SELECT ""Hash"", ""Size"" + FROM ""{m_tablename}"" + WHERE ""Restored"" = @Restored + ) + ")) + .SetParameterValue("@SourceVolumeId", sourceVolumeId) + .SetParameterValue("@Restored", 1); + + var moved = cmd.ExecuteNonQuery(); + + // Then update the blocks table to point to the new volume + cmd.SetCommandAndParameters(FormatInvariant($@" + UPDATE ""Block"" + SET ""VolumeID"" = @TargetVolumeId + WHERE ""VolumeID"" = @SourceVolumeId + AND (""Hash"", ""Size"") IN ( + SELECT ""Hash"", ""Size"" + FROM ""{m_tablename}"" + WHERE ""Restored"" = @Restored + ) + ")) + .SetParameterValue("@TargetVolumeId", targetVolumeId) + .SetParameterValue("@SourceVolumeId", sourceVolumeId) + .SetParameterValue("@Restored", 1); + + var updated = cmd.ExecuteNonQuery(); + if (updated != moved) + throw new Exception($"Unexpected number of updated blocks: {updated} != {moved}"); + + return updated; } public IEnumerable GetFilesetsUsingMissingBlocks() @@ -246,7 +533,7 @@ namespace Duplicati.Library.Main.Database var blocklists = FormatInvariant($@"SELECT DISTINCT ""FileLookup"".""ID"" AS ID FROM ""{m_tablename}"", ""Block"", ""Blockset"", ""BlocklistHash"", ""FileLookup"" WHERE ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""BlocklistHash"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" "); var cmdtxt = FormatInvariant($@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""FilesetEntry"", ""Fileset"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""RemoteVolume"".""Type"" = @Type AND ""FilesetEntry"".""FileID"" IN (SELECT DISTINCT ""ID"" FROM ( {blocks} UNION {blocklists} ))"); - using var cmd = m_connection.CreateCommand(m_transaction.Parent, cmdtxt) + using var cmd = m_connection.CreateCommand(m_transaction.Transaction, cmdtxt) .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); foreach (var rd in cmd.ExecuteReaderEnumerable()) @@ -255,7 +542,8 @@ namespace Duplicati.Library.Main.Database public IEnumerable GetMissingBlockSources() { - using var cmd = m_connection.CreateCommand(m_transaction.Parent, FormatInvariant($@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""Block"", ""{m_tablename}"" WHERE ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""Block"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Remotevolume"".""Name"" != @Name ")) + using var cmd = m_connection.CreateCommand(m_transaction.Transaction, FormatInvariant($@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""Block"", ""{m_tablename}"" WHERE ""{m_tablename}"".""Restored"" = @Restored AND ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""Block"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Remotevolume"".""Name"" != @Name ")) + .SetParameterValue("@Restored", 0) .SetParameterValue("@Name", m_volumename); foreach (var rd in cmd.ExecuteReaderEnumerable()) @@ -264,25 +552,24 @@ namespace Duplicati.Library.Main.Database public void Dispose() { - if (m_tablename != null) - { - try - { - using (var cmd = m_connection.CreateCommand(m_transaction.Parent)) - cmd.SetTransaction(m_transaction.Parent).ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tablename}"" ")); - } - catch { } - finally { m_tablename = null!; } - } + if (m_isDisposed) + return; - if (m_insertCommand != null) - try { m_insertCommand.Dispose(); } - catch { } - finally { m_insertCommand = null!; } + m_isDisposed = true; + try + { + if (m_tablename != null) + using (var cmd = m_connection.CreateCommand(m_transaction.Transaction)) + cmd.SetTransaction(m_transaction.Transaction).ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tablename}"" ")); + } + catch { } + + try { m_insertCommand?.Dispose(); } + catch { } } } - public IMissingBlockList CreateBlockList(string volumename, IDbTransaction? transaction = null) + public IMissingBlockList CreateBlockList(string volumename, ReusableTransaction transaction) { return new MissingBlockList(volumename, m_connection, transaction); } @@ -589,13 +876,12 @@ namespace Duplicati.Library.Main.Database } } - public void CheckAllBlocksAreInVolume(string filename, IEnumerable> blocks) + public void CheckAllBlocksAreInVolume(string filename, IEnumerable> blocks, IDbTransaction transaction) { - using (var tr = m_connection.BeginTransactionSafe()) - using (var cmd = m_connection.CreateCommand(tr)) + using var cmd = m_connection.CreateCommand(transaction); + var tablename = "ProbeBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + try { - var tablename = "ProbeBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)")); cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{tablename}"" (""Hash"", ""Size"") VALUES (@Hash, @Size)")); @@ -614,16 +900,18 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@VolumeId", id) .ExecuteScalarInt64(0); - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tablename}"" ")); - if (aliens != 0) throw new Exception($"Not all blocks were found in {filename}"); } + finally + { + cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tablename}"" ")); + } } - public void CheckBlocklistCorrect(string hash, long length, IEnumerable blocklist, long blocksize, long blockhashlength) + public void CheckBlocklistCorrect(string hash, long length, IEnumerable blocklist, long blocksize, long blockhashlength, IDbTransaction? transaction) { - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(transaction)) { var query = FormatInvariant($@" SELECT diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index c4af01e17..95c4f88e8 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -325,16 +325,13 @@ namespace Duplicati.Library.Main.Operation private async Task FinishVolumeAndUpload(LocalDeleteDatabase db, IBackendManager backendManager, BlockVolumeWriter newvol, IndexVolumeWriter newvolindex, List> uploadedVolumes, ReusableTransaction rtr) { - Action indexVolumeFinished = () => - { - if (newvolindex != null && m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) + Action indexVolumeFinished = null; + if (newvolindex != null && m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) + indexVolumeFinished = () => { foreach (var blocklist in db.GetBlocklists(newvol.VolumeID, m_options.Blocksize, m_options.BlockhashSize)) - { newvolindex.WriteBlocklist(blocklist.Item1, blocklist.Item2, 0, blocklist.Item3); - } - } - }; + }; uploadedVolumes.Add(new KeyValuePair(newvol.RemoteFilename, newvol.Filesize)); if (newvolindex != null) @@ -342,7 +339,7 @@ namespace Duplicati.Library.Main.Operation // TODO: The upload here does not flush the database messages, // and this can leave the database in a state where it does not know of the remote file - // To fix it, we need thread-safe access to the daabase and transaction + // To fix it, we need thread-safe access to the database and transaction // Once fixed, we can perhaps let he backend manager simply call the database directly if (!m_options.Dryrun) { diff --git a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs index 43468540f..dcca28a80 100644 --- a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs @@ -55,7 +55,7 @@ namespace Duplicati.Library.Main.Operation await DoRunAsync(backendManager, db, tr, filter, callbackhandler).ConfigureAwait(false); } - public static async Task<(Tuple[], List Missing)> GetBrokenFilesetsFromRemote(IBackendManager backendManager, BasicResults result, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Options options) + public static async Task<((DateTime FilesetTime, long FilesetID, long RemoveCount)[], List Missing)> GetBrokenFilesetsFromRemote(IBackendManager backendManager, BasicResults result, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Options options) { List missing = null; var brokensets = db.GetBrokenFilesets(options.Time, options.Version, transaction).ToArray(); @@ -86,6 +86,11 @@ namespace Duplicati.Library.Main.Operation // Drop all content from tables db.RemoveMissingBlocks(missing.Select(x => x.Name), transaction); + + // Mark all orphaned index files as disposable after removing the missing block files + foreach (var f in db.GetOrphanedIndexFiles(transaction).ToList()) + db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, transaction); + brokensets = db.GetBrokenFilesets(options.Time, options.Version, transaction).ToArray(); } diff --git a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs index 75b2323aa..491958fee 100644 --- a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs @@ -88,14 +88,25 @@ namespace Duplicati.Library.Main.Operation SetCount = db.GetFilesetFileCount(x.Item2, tr) }).ToArray(); - if (m_options.Dryrun) - tr.Rollback(); - else - tr.Commit(); + var emptyBlocksetId = -1L; + if (!m_options.DisableReplaceMissingMetadata) + { + var emptymetadata = Utility.WrapMetadata(new Dictionary(), m_options); + emptyBlocksetId = db.GetEmptyMetadataBlocksetId(missing.Select(x => x.ID), emptymetadata.FileHash, emptymetadata.Blob.Length, null); + if (emptyBlocksetId < 0) + throw new UserInformationException($"Failed to locate an empty metadata blockset to replace missing metadata. Set the option --disable-replace-missing-metadata=true to ignore this and drop files with missing metadata.", "FailedToLocateEmptyMetadataBlockset"); + } var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray(); var to_purge = compare_list.Where(x => x.RemoveCount != x.SetCount).ToArray(); + if (fully_emptied.Length == db.FilesetTimes.Count()) + throw new UserInformationException("All filesets are fully broken and needs to be removed. To avoid unexpected deletions, you must manually remove the remote files and delete the database.", "AllFilesetsBroken"); + + if (!m_options.Dryrun) + tr.Commit(); + + if (fully_emptied.Length != 0) { if (fully_emptied.Length == 1) @@ -104,7 +115,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing {0} filesets where all file(s) are broken: {1}", fully_emptied.Length, string.Join(", ", fully_emptied.Select(x => x.Timestamp.ToLocalTime().ToString()))); m_result.DeleteResults = new DeleteResults(m_result); - using (var rmdb = new Database.LocalDeleteDatabase(db)) + using (var rmdb = new LocalDeleteDatabase(db)) using (var deltr = new ReusableTransaction(rmdb)) { var opts = new Options(new Dictionary(m_options.RawOptions)); @@ -148,7 +159,14 @@ namespace Duplicati.Library.Main.Operation { if (filesetid != bs.FilesetID) throw new Exception(string.Format("Unexpected filesetid: {0}, expected {1}", filesetid, bs.FilesetID)); + + // Update entries that would be removed because of missing metadata + var updatedEntries = 0; + if (!m_options.DisableReplaceMissingMetadata) + updatedEntries = db.ReplaceMetadata(filesetid, emptyBlocksetId, cmd.Transaction); + db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID", cmd.Transaction); + return updatedEntries; }).ConfigureAwait(false); } diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index 13b766d56..99cf71b8e 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -21,9 +21,9 @@ using System; using System.Linq; -using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Interface; +using Duplicati.Library.Main.Volumes; using Duplicati.Library.Utility; namespace Duplicati.Library.Main.Operation @@ -58,10 +58,10 @@ namespace Duplicati.Library.Main.Operation } } - public Task RunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, float pgoffset, float pgspan, Action filtercommand) + public Task RunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, float pgoffset, float pgspan, Func filtercommand) => DoRunAsync(backendManager, db, null, filtercommand, pgoffset, pgspan); - private async Task DoRunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, IFilter filter, Action filtercommand, float pgoffset, float pgspan) + private async Task DoRunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, IFilter filter, Func filtercommand, float pgoffset, float pgspan) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Begin); Logging.Log.WriteInformationMessage(LOGTAG, "StartingPurge", "Starting purge operation"); @@ -116,34 +116,13 @@ namespace Duplicati.Library.Main.Operation using (var tr = db.BeginTransaction()) { - var ix = -1; - for (var i = 0; i < filesets.Length; i++) - if (filesets[i].Key == versionid) - { - ix = i; - break; - } - - if (ix < 0) + (var _, var tsOriginal, var ix) = filesets.Select((x, i) => (x.Key, x.Value, i)).FirstOrDefault(x => x.Key == versionid); + if (ix < 0 || tsOriginal.Ticks == 0) throw new InvalidProgramException(string.Format("Fileset was reported with id {0}, but could not be found?", versionid)); - var secs = 0; - while (secs < 60) - { - secs++; - var tfn = Volumes.VolumeBase.GenerateFilename(RemoteVolumeType.Files, m_options, null, filesets[ix].Value.AddSeconds(secs)); - if (db.GetRemoteVolumeID(tfn, tr) < 0) - break; - } - - var tsOriginal = filesets[ix].Value; - var ts = tsOriginal.AddSeconds(secs); - + var ts = FilesetVolumeWriter.ProbeUnusedFilenameName(db, tr, m_options, tsOriginal); var prevfilename = db.GetRemoteVolumeNameForFileset(filesets[ix].Key, tr); - if (secs >= 60) - throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is more than 60 seconds away", prevfilename, ts)); - if (ix != 0 && filesets[ix - 1].Value <= ts) throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is larger than the next timestamp {2}", prevfilename, ts, filesets[ix - 1].Value)); @@ -154,7 +133,7 @@ namespace Duplicati.Library.Main.Operation else tempset.ApplyFilter(filtercommand); - if (tempset.RemovedFileCount == 0) + if (tempset.RemovedFileCount + tempset.UpdatedFileCount == 0) { Logging.Log.WriteInformationMessage(LOGTAG, "NotWritingNewFileset", "Not writing a new fileset for {0} as it was not changed", prevfilename); currentprogress += versionprogress; @@ -163,10 +142,10 @@ namespace Duplicati.Library.Main.Operation } else { - using (var tf = new Library.Utility.TempFile()) - using (var vol = new Volumes.FilesetVolumeWriter(m_options, ts)) + using (var tf = new TempFile()) + using (var vol = new FilesetVolumeWriter(m_options, ts)) { - var isOriginalFilesetFullBackup = db.IsFilesetFullBackup(tsOriginal); + var isOriginalFilesetFullBackup = db.IsFilesetFullBackup(tsOriginal, tr); var newids = tempset.ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup); vol.VolumeID = newids.Item1; vol.CreateFilesetFile(isOriginalFilesetFullBackup); @@ -177,6 +156,7 @@ namespace Duplicati.Library.Main.Operation m_result.RemovedFileSize += tempset.RemovedFileSize; m_result.RemovedFileCount += tempset.RemovedFileCount; + m_result.UpdatedFileCount += tempset.UpdatedFileCount; m_result.RewrittenFileLists++; currentprogress += (versionprogress / 2); @@ -208,7 +188,7 @@ namespace Duplicati.Library.Main.Operation } else { - var lst = db.DropFilesetsFromTable(new[] { filesets[ix].Value }, tr).ToArray(); + var lst = db.DropFilesetsFromTable(new[] { tsOriginal }, tr).ToArray(); foreach (var f in lst) db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, tr); diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index a1ef89596..61b5d6ab5 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -20,12 +20,14 @@ // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; +using System.Data; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; +using Duplicati.Library.Main.Operation.Backup; using Duplicati.Library.Main.Volumes; using Duplicati.Library.Utility; @@ -126,6 +128,7 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(0); using (var db = new LocalRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache)) + using (var rtr = new ReusableTransaction(db)) { Utility.UpdateOptionsFromDb(db, m_options); Utility.VerifyOptionsAndUpdateDatabase(db, m_options); @@ -137,14 +140,15 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsInRepairState"); // Ensure the database is consistent before we start fixing the remote - db.VerifyConsistencyForRepair(m_options.Blocksize, m_options.BlockhashSize, true, null); + db.VerifyConsistencyForRepair(m_options.Blocksize, m_options.BlockhashSize, true, rtr.Transaction); // If the last backup failed, guard the incomplete fileset, so we can create a synthetic filelist - var lastTempFilelist = db.GetLastIncompleteFilesetVolume(null); - var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, null, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false); + var lastTempFilelist = db.GetLastIncompleteFilesetVolume(rtr.Transaction); + var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, rtr.Transaction, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false); + rtr.Commit("CommitRemoteListAnalysisTransaction"); var buffer = new byte[m_options.Blocksize]; - var hashsize = HashFactory.HashSizeBytes(m_options.BlockHashAlgorithm); + var hashsize = m_options.BlockhashSize; var missingRemoteFilesets = db.MissingRemoteFilesets().ToList(); var missingLocalFilesets = db.MissingLocalFilesets().ToList(); @@ -185,21 +189,22 @@ namespace Duplicati.Library.Main.Operation if (tp.VerificationRequiredVolumes.Any()) { using (var testdb = new LocalTestDatabase(db)) - using (var rtr = new ReusableTransaction(testdb)) { foreach (var n in tp.VerificationRequiredVolumes) try { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(testdb, null, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(testdb, rtr.Transaction, cancellationToken).ConfigureAwait(false); + if (!m_options.Dryrun) + rtr.Commit("CommitEarlyExit", false); return; } progress++; m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); - KeyValuePair>> res; + KeyValuePair>> res; (var tf, var hash, var size) = await backendManager.GetWithInfoAsync(n.Name, n.Hash, n.Size, cancellationToken).ConfigureAwait(false); using (tf) res = TestHandler.TestVolumeInternals(testdb, rtr, n, tf, m_options, 1); @@ -221,7 +226,7 @@ namespace Duplicati.Library.Main.Operation throw; } - rtr.Commit("CommitVerificationTransaction", false); + rtr.Commit("CommitVerificationTransaction"); } } @@ -231,7 +236,9 @@ namespace Duplicati.Library.Main.Operation { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + if (!m_options.Dryrun) + rtr.Commit("CommitEarlyExit", false); return; } @@ -260,33 +267,31 @@ namespace Duplicati.Library.Main.Operation if (entry.Hash != rv.Hash || entry.Size != rv.Length || !new[] { RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified }.Contains(entry.State)) throw new Exception(string.Format("Volume {0} hash/size mismatch ({1} - {2}) vs ({3} - {4})", rv.Filename, entry.Hash, entry.Size, rv.Hash, rv.Length)); - db.CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks); + db.CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks, rtr.Transaction); } var blocksize = m_options.Blocksize; foreach (var ixb in ifr.BlockLists) - db.CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize); + db.CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize, rtr.Transaction); // Register the new index file and link it to the block files - using (var tr = db.BeginTransaction()) + var selfid = db.RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, RemoteVolumeState.Uploading, size, new TimeSpan(0), rtr.Transaction); + foreach (var rv in ifr.Volumes) { - var selfid = db.RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, RemoteVolumeState.Uploading, size, new TimeSpan(0), tr); - foreach (var rv in ifr.Volumes) - { - // Guard against unknown block files - long id = db.GetRemoteVolumeID(rv.Filename, tr); - if (id == -1) - Logging.Log.WriteWarningMessage(LOGTAG, "UnknownBlockFile", null, "Index file {0} references unknown block file: {1}", n.File.Name, rv.Filename); - else - db.AddIndexBlockLink(selfid, id, tr); - } - tr.Commit(); + // Guard against unknown block files + long id = db.GetRemoteVolumeID(rv.Filename, rtr.Transaction); + if (id == -1) + Logging.Log.WriteWarningMessage(LOGTAG, "UnknownBlockFile", null, "Index file {0} references unknown block file: {1}", n.File.Name, rv.Filename); + else + db.AddIndexBlockLink(selfid, id, rtr.Transaction); } + if (!m_options.Dryrun) + rtr.Commit("CommitIndexFileTransaction"); } // All checks fine, we accept the new index file Logging.Log.WriteInformationMessage(LOGTAG, "AcceptNewIndexFile", "Accepting new index file {0}", n.File.Name); - db.UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash); + db.UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash, rtr.Transaction); continue; } catch (Exception rex) @@ -310,25 +315,24 @@ namespace Duplicati.Library.Main.Operation throw; } - if (!m_options.RebuildMissingDblockFiles) - { - var missingDblocks = tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Blocks).ToArray(); - if (missingDblocks.Length > 0) - throw new UserInformationException($"The backup storage destination is missing data files. You can either enable `--rebuild-missing-dblock-files` or run the purge command to remove these files. The following files are missing: {string.Join(", ", missingDblocks.Select(x => x.Name))}", "MissingDblockFiles"); - } + var missingDblocks = tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Blocks); + if (!m_options.RebuildMissingDblockFiles && missingDblocks.Count() > 0) + throw new UserInformationException($"The backup storage destination is missing data files. You can either enable `--rebuild-missing-dblock-files` or run the purge command to remove these files. The following files are missing: {string.Join(", ", missingDblocks.Select(x => x.Name))}", "MissingDblockFiles"); var anyDlistUploads = false; foreach (var (filesetId, timestamp, isfull) in missingRemoteFilesets) { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + if (!m_options.Dryrun) + rtr.Commit("CommitEarlyExit", false); return; } progress++; m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); - var fileTime = FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, timestamp); + var fileTime = FilesetVolumeWriter.ProbeUnusedFilenameName(db, rtr.Transaction, m_options, timestamp); var fsw = new FilesetVolumeWriter(m_options, fileTime); Logging.Log.WriteInformationMessage(LOGTAG, "ReuploadingFileset", "Re-uploading fileset {0} from {1} as remote volume registration is missing, new filename: {2}", filesetId, timestamp, fsw.RemoteFilename); @@ -341,21 +345,25 @@ namespace Duplicati.Library.Main.Operation db.WriteFileset(fsw, filesetId, null); fsw.Close(); + + fsw.VolumeID = db.RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, -1, RemoteVolumeState.Temporary); + db.LinkFilesetToVolume(filesetId, fsw.VolumeID, null); if (m_options.Dryrun) { fsw.Dispose(); Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would re-upload fileset {0}", fsw.RemoteFilename); continue; } - - fsw.VolumeID = db.RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, -1, RemoteVolumeState.Temporary); - db.LinkFilesetToVolume(filesetId, fsw.VolumeID, null); - // TODO: Rewrite to use transactions and flush db messages as needed await backendManager.PutAsync(fsw, null, null, false, null, cancellationToken).ConfigureAwait(false); } if (anyDlistUploads) - await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); + { + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + if (!m_options.Dryrun) + rtr.Commit("CommitFilesetTransaction"); + } foreach (var volumename in missingLocalFilesets) { @@ -365,15 +373,15 @@ namespace Duplicati.Library.Main.Operation var parsed = VolumeBase.ParseFilename(remoteVolume.Name); using (var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions)) - using (var transaction = db.BeginTransaction()) using (var recreatedb = new LocalRecreateDatabase(db, m_options)) { if (compressor == null) throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); - var filesetid = db.CreateFileset(remoteVolume.ID, parsed.Time, transaction); - RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, transaction, compressor, filesetid, m_options, new FilterExpression()); - transaction.Commit(); + var filesetid = db.CreateFileset(remoteVolume.ID, parsed.Time, rtr.Transaction); + RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, rtr.Transaction, compressor, filesetid, m_options, new FilterExpression()); + if (!m_options.Dryrun) + rtr.Commit("CommitRecreateFilesetTransaction"); } } } @@ -381,193 +389,69 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun && tp.MissingVolumes.Any()) db.TerminatedWithActiveUploads = true; - foreach (var n in tp.MissingVolumes) + if (tp.MissingVolumes.Any(x => x.Type != RemoteVolumeType.Index && x.Type != RemoteVolumeType.Files && x.Type != RemoteVolumeType.Blocks)) + throw new InvalidOperationException(string.Format("Unknown volume type {0} detected", tp.MissingVolumes.First(x => x.Type != RemoteVolumeType.Index && x.Type != RemoteVolumeType.Files && x.Type != RemoteVolumeType.Blocks).Type)); + + // Process each of the missing volumes in the order of blocks, files and index + // It is important that we process the blocks first, as the index files are derived from the blocks + + Action incrementProgress = () => { - IDisposable newEntry = null; + progress++; + m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); + }; + + // Blocks are recreated with the entire list of missing files to handle cases where partial recreation is needed + await RunRepairDblocks(backendManager, db, rtr, missingDblocks, incrementProgress, cancellationToken).ConfigureAwait(false); + + // Filesets are recreated one at a time + foreach (var n in tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Files)) + { + FilesetVolumeWriter newEntry = null; + incrementProgress(); try { - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - { - await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); - return; - } + var timestamp = VolumeBase.ParseFilename(n.Name).Time; + var fileTime = FilesetVolumeWriter.ProbeUnusedFilenameName(db, rtr.Transaction, m_options, timestamp); + var volumeWriter = newEntry = new FilesetVolumeWriter(m_options, fileTime); - progress++; - m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); - - if (n.Type == RemoteVolumeType.Files) - { - var filesetId = db.GetFilesetIdFromRemotename(n.Name); - - // We cannot wrap the FilesetVolumeWriter in a using statement here because a reference to it is - // retained in newEntry. - FilesetVolumeWriter volumeWriter = new FilesetVolumeWriter(m_options, DateTime.UtcNow); - newEntry = volumeWriter; - volumeWriter.SetRemoteFilename(n.Name); - - db.WriteFileset(volumeWriter, filesetId, null); - DateTime filesetTime = db.FilesetTimes.First(x => x.Key == filesetId).Value; - volumeWriter.CreateFilesetFile(db.IsFilesetFullBackup(filesetTime)); - - volumeWriter.Close(); - if (m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would re-upload fileset {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(volumeWriter.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)); - else - { - db.UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null); - // TODO: Rewrite to use transactions and flush db messages as needed - await backendManager.PutAsync(volumeWriter, null, null, false, null, cancellationToken).ConfigureAwait(false); - } - } - else if (n.Type == RemoteVolumeType.Index) - { - IndexVolumeWriter w = new IndexVolumeWriter(m_options); - newEntry = w; - w.SetRemoteFilename(n.Name); - - using (var h = HashFactory.CreateHasher(m_options.BlockHashAlgorithm)) - { - - foreach (var blockvolume in db.GetBlockVolumesFromIndexName(n.Name)) - { - w.StartVolume(blockvolume.Name); - var volumeid = db.GetRemoteVolumeID(blockvolume.Name); - - foreach (var b in db.GetBlocks(volumeid)) - w.AddBlock(b.Hash, b.Size); - - w.FinishVolume(blockvolume.Hash, blockvolume.Size); - - if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) - foreach (var b in db.GetBlocklists(volumeid, m_options.Blocksize, hashsize)) - { - var bh = Convert.ToBase64String(h.ComputeHash(b.Item2, 0, b.Item3)); - if (bh != b.Item1) - throw new Exception(string.Format("Internal consistency check failed, generated index block has wrong hash, {0} vs {1}", bh, b.Item1)); - - w.WriteBlocklist(b.Item1, b.Item2, 0, b.Item3); - } - } - } - - w.Close(); - - if (m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadIndexFile", "would re-upload index file {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)); - else - { - db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null); - // TODO: Rewrite to use transactions and flush db messages as needed - await backendManager.PutAsync(w, null, null, false, null, cancellationToken).ConfigureAwait(false); - } - } - else if (n.Type == RemoteVolumeType.Blocks) - { - BlockVolumeWriter w = new BlockVolumeWriter(m_options); - newEntry = w; - w.SetRemoteFilename(n.Name); - - using (var mbl = db.CreateBlockList(n.Name)) - { - //First we grab all known blocks from local files - foreach (var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize)) - { - var hash = block.Hash; - var size = (int)block.Size; - - foreach (var source in block.Sources) - { - var file = source.File; - var offset = source.Offset; - - try - { - if (System.IO.File.Exists(file)) - using (var f = System.IO.File.OpenRead(file)) - { - f.Position = offset; - if (size == Library.Utility.Utility.ForceStreamRead(f, buffer, size)) - { - using (var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm)) - { - var newhash = Convert.ToBase64String(blockhasher.ComputeHash(buffer, 0, size)); - if (newhash == hash) - { - if (mbl.SetBlockRestored(hash, size)) - w.AddBlock(hash, buffer, 0, size, Duplicati.Library.Interface.CompressionHint.Default); - break; - } - } - } - } - } - catch (Exception ex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", ex, "Failed to access file: {0}", file); - } - } - } - - //Then we grab all remote volumes that have the missing blocks - await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(mbl.GetMissingBlockSources().ToList(), cancellationToken).ConfigureAwait(false)) - { - try - { - using (tmpfile) - using (var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options)) - foreach (var b in f.Blocks) - if (mbl.SetBlockRestored(b.Key, b.Value)) - if (f.ReadBlock(b.Key, buffer) == b.Value) - w.AddBlock(b.Key, buffer, 0, (int)b.Value, CompressionHint.Default); - } - catch (Exception ex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileAccessError", ex, "Failed to access remote file: {0}", name); - } - } - - // If we managed to recover all blocks, NICE! - var missingBlocks = mbl.GetMissingBlocks().Count(); - if (missingBlocks > 0) - { - Logging.Log.WriteInformationMessage(LOGTAG, "RepairMissingBlocks", "Repair cannot acquire {0} required blocks for volume {1}, which are required by the following filesets: ", missingBlocks, n.Name); - foreach (var f in mbl.GetFilesetsUsingMissingBlocks()) - Logging.Log.WriteInformationMessage(LOGTAG, "AffectedFilesetName", f.Name); - - var recoverymsg = string.Format("If you want to continue working with the database, you can use the \"{0}\" and \"{1}\" commands to purge the missing data from the database and the remote storage.", "list-broken-files", "purge-broken-files"); - - if (!m_options.Dryrun) - { - Logging.Log.WriteInformationMessage(LOGTAG, "RecoverySuggestion", "This may be fixed by deleting the filesets and running repair again"); - - throw new UserInformationException(string.Format("Repair not possible, missing {0} blocks.\n" + recoverymsg, missingBlocks), "RepairIsNotPossible"); - } - else - { - Logging.Log.WriteInformationMessage(LOGTAG, "RecoverySuggestion", recoverymsg); - } - } - else - { - if (m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadBlockFile", "would re-upload block file {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)); - else - { - db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null); - // TODO: Rewrite to use transactions and flush db messages as needed - await backendManager.PutAsync(w, null, null, false, null, cancellationToken).ConfigureAwait(false); - } - } - } - } + await RunRepairDlist(backendManager, db, rtr, volumeWriter, n, fileTime, cancellationToken).ConfigureAwait(false); } catch (Exception ex) { if (newEntry != null) - try { newEntry.Dispose(); } + try { newEntry?.Dispose(); } + catch { } + + Logging.Log.WriteErrorMessage(LOGTAG, "CleanupMissingFileError", ex, "Failed to perform cleanup for missing file: {0}, message: {1}", n.Name, ex.Message); + + if (ex.IsAbortException()) + throw; + } + } + + // Index files are recreated one at a time, as they are derived from the blocks + foreach (var n in tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Index)) + { + IndexVolumeWriter newEntry = null; + incrementProgress(); + + try + { + // Check if the index file has already been deleted, beause the dblock was recreated + var currentState = db.GetRemoteVolume(n.Name, rtr.Transaction).State; + if (currentState == RemoteVolumeState.Deleted || currentState == RemoteVolumeState.Temporary || currentState == RemoteVolumeState.Deleting) + continue; + + var w = newEntry = new IndexVolumeWriter(m_options); + await RunRepairDindex(backendManager, db, rtr, w, n, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + if (newEntry != null) + try { newEntry?.Dispose(); } catch { } - finally { newEntry = null; } Logging.Log.WriteErrorMessage(LOGTAG, "CleanupMissingFileError", ex, "Failed to perform cleanup for missing file: {0}, message: {1}", n.Name, ex.Message); @@ -601,6 +485,7 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "DeletingEmptyIndexFile", "Deleting empty index file {0}", emptyIndexFile.Name); await backendManager.DeleteAsync(emptyIndexFile.Name, emptyIndexFile.Size, false, cancellationToken).ConfigureAwait(false); + await backendManager.FlushPendingMessagesAsync(db, null, cancellationToken).ConfigureAwait(false); } } } @@ -608,7 +493,7 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteErrorMessage(LOGTAG, "CleanupEmptyIndexFileError", ex, "Failed to perform cleanup for empty index file: {0}, message: {1}", emptyIndexFile.Name, ex.Message); - if (ex is System.Threading.ThreadAbortException) + if (ex.IsAbortException()) throw; } } @@ -621,7 +506,592 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(1); await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) + { db.TerminatedWithActiveUploads = false; + rtr.Commit("CommitRepairTransaction", false); + } + } + } + + /// + /// Repairs a single fileset by recreating it from the database content and uploading it + /// + /// The backend manager to use for uploading + /// The database to use for the repair + /// The transaction to use for the repair + /// The volume writer to use for the repair + /// The remote volume entry to repair + /// The time of the new fileset to create + /// Cancellation token to cancel the operation + /// A task representing the asynchronous operation + private async Task RunRepairDlist(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, FilesetVolumeWriter volumeWriter, RemoteVolumeEntry originalVolume, DateTime filesetTime, CancellationToken cancellationToken) + { + volumeWriter.VolumeID = db.RegisterRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, rtr.Transaction); + (var prevFilesetId, var _, var isPrevFull) = db.GetFilesetFromRemotename(originalVolume.Name, rtr.Transaction); + + if (!string.IsNullOrEmpty(m_options.ControlFiles)) + foreach (var p in m_options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) + volumeWriter.AddControlFile(p, m_options.GetCompressionHintFromFilename(p)); + + volumeWriter.CreateFilesetFile(isPrevFull); + var newFilesetID = db.CreateFileset(volumeWriter.VolumeID, filesetTime, rtr.Transaction); + db.UpdateFullBackupStateInFileset(newFilesetID, isPrevFull); + + db.LinkFilesetToVolume(newFilesetID, volumeWriter.VolumeID, rtr.Transaction); + db.MoveFilesFromFileset(newFilesetID, prevFilesetId, rtr.Transaction); + + db.WriteFileset(volumeWriter, newFilesetID, rtr.Transaction); + volumeWriter.Close(); + + db.UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null, rtr.Transaction); + db.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); + await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + + if (m_options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would upload fileset {0}, with size {1}, previous size {2}", originalVolume.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(volumeWriter.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(originalVolume.Size)); + else + { + rtr.Commit("CommitPriorToFilesetUpload"); + await backendManager.PutAsync(volumeWriter, null, null, false, null, cancellationToken).ConfigureAwait(false); + } + + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + if (!m_options.Dryrun) + rtr.Commit("CommitFilesetTransaction"); + } + + /// + /// Repairs a single index file by recreating it from the database content and uploading it + /// + /// The backend manager to use for uploading + /// The database to use for the repair + /// The transaction to use for the repair + /// The volume writer to use for the repair + /// The remote volume entry to repair + /// The time of the new fileset to create + /// Cancellation token to cancel the operation + /// A task representing the asynchronous operation + + private async Task RunRepairDindex(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, IndexVolumeWriter indexWriter, RemoteVolumeEntry originalVolume, CancellationToken cancellationToken) + { + indexWriter.VolumeID = db.RegisterRemoteVolume(indexWriter.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Uploading, -1, TimeSpan.Zero, rtr.Transaction); + + var blockvolumeids = new List(); + using var h = HashFactory.CreateHasher(m_options.BlockHashAlgorithm); + foreach (var blockvolume in db.GetBlockVolumesFromIndexName(originalVolume.Name)) + { + indexWriter.StartVolume(blockvolume.Name); + var volumeid = db.GetRemoteVolumeID(blockvolume.Name, rtr.Transaction); + + foreach (var b in db.GetBlocks(volumeid)) + indexWriter.AddBlock(b.Hash, b.Size); + + indexWriter.FinishVolume(blockvolume.Hash, blockvolume.Size); + blockvolumeids.Add(volumeid); + + if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) + foreach (var b in db.GetBlocklists(volumeid, m_options.Blocksize, m_options.BlockhashSize)) + { + var bh = Convert.ToBase64String(h.ComputeHash(b.Item2, 0, b.Item3)); + if (bh != b.Item1) + throw new Exception(string.Format("Internal consistency check failed, generated index block has wrong hash, {0} vs {1}", bh, b.Item1)); + + indexWriter.WriteBlocklist(b.Item1, b.Item2, 0, b.Item3); + } + } + + foreach (var blockvolumeid in blockvolumeids) + db.AddIndexBlockLink(indexWriter.VolumeID, blockvolumeid, rtr.Transaction); + + indexWriter.Close(); + + if (m_options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadIndexFile", "would re-upload index file {0}, with size {1}, previous size {2}", originalVolume.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(indexWriter.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(originalVolume.Size)); + else + { + db.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); + await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backendManager.PutAsync(indexWriter, null, null, false, null, cancellationToken).ConfigureAwait(false); + } + + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + if (!m_options.Dryrun) + rtr.Commit("CommitRepairTransaction"); + } + + /// + /// Class for holding the state of a dblock volume that is being constructed + /// + private sealed class InProgressDblockVolumes + { + /// + /// Database to use for the operation + /// + private readonly LocalDatabase m_database; + /// + /// Transaction to use for the operation + /// + private readonly ReusableTransaction m_transaction; + /// + /// List of volumes to delete after the upload is complete + /// + private readonly List m_toDelete = new List(); + /// + /// Options for the current operation + /// + private readonly Options m_options; + /// + /// Maximum size of the volume before it is considered full + /// + private readonly long m_maxVolumeSize; + /// + /// The writers that are completed and ready to be uploaded + /// + private readonly List m_completedWriters = new List(); + /// + /// Writer for the current in-progress volume + /// + private BlockVolumeWriter m_activeWriter; + /// + /// Flag indicating if any data has been added to the volume + /// + private bool m_anyData = false; + + /// + /// Constructor for the InProgressDblockVolume class + /// + /// The options for the current operation + /// The database to use for registering a volume in progress + /// The transaction to use for the operation + public InProgressDblockVolumes(Options options, LocalDatabase db, ReusableTransaction transaction) + { + m_database = db; + m_transaction = transaction; + m_options = options; + m_maxVolumeSize = options.VolumeSize - m_options.Blocksize; + m_activeWriter = new BlockVolumeWriter(options); + m_activeWriter.VolumeID = m_database.RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, m_transaction.Transaction); + } + + /// + /// Checks if any data has been added to the current volume + /// + public bool AnyData => m_anyData; + /// + /// Checks if the current volume is full + /// + private bool IsFull => m_activeWriter.Filesize > m_maxVolumeSize; + + /// + /// Gets the volume ID of the current volume + /// + public long VolumeID => m_activeWriter.VolumeID; + /// + /// Gets the remote filename of the current volume + /// + public string RemoteFilename => m_activeWriter.RemoteFilename; + /// + /// Gets the list of volumes to delete after the upload is complete + /// + public List VolumesToDelete => m_toDelete; + /// + /// Gets the list of completed writers + /// + public List CompletedWriters => m_completedWriters; + + /// + /// Adds a block to the current volume, starting a new volume if the current one is full + /// + /// The hash of the block + /// The buffer containing the block data + /// The offset in the buffer where the block data starts + /// The size of the block data + /// The compression hint for the block + public void AddBlock(string hash, byte[] buffer, int offset, int size, CompressionHint hint) + { + m_activeWriter.AddBlock(hash, buffer, offset, size, hint); + m_anyData = true; + if (IsFull) + StartNewVolume(); + } + + /// + /// Starts a new volume for the current operation + /// + public void StartNewVolume() + { + if (!m_anyData) + return; + + m_completedWriters.Add(m_activeWriter); + + m_activeWriter = new BlockVolumeWriter(m_options); + m_activeWriter.VolumeID = m_database.RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, m_transaction.Transaction); + m_anyData = false; + } + } + + /// + /// Runs the repair process for missing dblock files + /// + /// The backend manager to use for uploading + /// The database to use for the repair + /// The transaction to use for the repair + /// The list of missing dblock files + /// + /// Cancellation token to cancel the operation + /// >A task representing the asynchronous operation + private async Task RunRepairDblocks(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, IEnumerable missingDblockFiles, Action incrementProgress, CancellationToken cancellationToken) + { + var currentVolume = new InProgressDblockVolumes(m_options, db, rtr); + + foreach (var n in missingDblockFiles) + { + try + { + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + { + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + if (!m_options.Dryrun) + rtr.Commit("CommitEarlyExit", false); + return; + } + + incrementProgress(); + + await RunRepairDblock(backendManager, db, rtr, n, currentVolume, cancellationToken).ConfigureAwait(false); + if (currentVolume.CompletedWriters.Count > 0) + await UploadCompletedBlockVolumes(backendManager, currentVolume, db, rtr, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "RepairDblockError", ex, "Failed to perform repair for dblock: {0}, message: {1}", n.Name, ex.Message); + if (ex.IsAbortException()) + throw; + } + } + + // If we have any data in the volume, complete it so it will be uploaded + if (currentVolume.AnyData) + currentVolume.StartNewVolume(); + // Complete any pending uploads + await UploadCompletedBlockVolumes(backendManager, currentVolume, db, rtr, cancellationToken).ConfigureAwait(false); + + // After reset, we need to remove the temporary volume from the database + db.RemoveRemoteVolume(currentVolume.RemoteFilename, rtr.Transaction); + } + + /// + /// Uploads all completed block volumes to the backend and deletes the old files + /// + /// The backend manager to use for uploading + /// The completed volume state to upload + /// The database to use for the upload + /// >The transaction to use for the upload + /// Cancellation token to cancel the operation + /// >A task representing the asynchronous operation + async Task UploadCompletedBlockVolumes(IBackendManager backendManager, InProgressDblockVolumes activeVolume, LocalRepairDatabase db, ReusableTransaction rtr, CancellationToken cancellationToken) + { + foreach (var completedVolume in activeVolume.CompletedWriters) + { + db.UpdateRemoteVolume(completedVolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, rtr.Transaction); + + // Create a new index file that points to the new volume + IndexVolumeWriter newvolindex = null; + Action indexVolumeFinished = null; + if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None) + { + newvolindex = new IndexVolumeWriter(m_options); + newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, rtr.Transaction); + newvolindex.StartVolume(completedVolume.RemoteFilename); + foreach (var b in db.GetBlocks(completedVolume.VolumeID)) + newvolindex.AddBlock(b.Hash, b.Size); + + db.AddIndexBlockLink(newvolindex.VolumeID, completedVolume.VolumeID, rtr.Transaction); + if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) + indexVolumeFinished = () => + { + foreach (var blocklist in db.GetBlocklists(completedVolume.VolumeID, m_options.Blocksize, m_options.BlockhashSize)) + newvolindex.WriteBlocklist(blocklist.Item1, blocklist.Item2, 0, blocklist.Item3); + }; + } + + // All information is in place, we can now upload the new volume + await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + if (m_options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReplaceBlockFile", "would upload new block file {0}", completedVolume.RemoteFilename); + else + { + rtr.Commit("PostRepairPreUploadBlockVolume"); + await backendManager.PutAsync(completedVolume, newvolindex, indexVolumeFinished, false, null, cancellationToken).ConfigureAwait(false); + } + } + + // Flush the database as we have new volumes uploaded + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + + // Prepare for deleting the old stuff + foreach (var vol in activeVolume.VolumesToDelete) + db.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Deleting, vol.Size, vol.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); + + // Persist desired state prior to deleting the old files + if (!m_options.Dryrun) + rtr.Commit("PostRepairBlockVolume"); + + // Delete the old files + foreach (var vol in activeVolume.VolumesToDelete) + if (m_options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteIndexFile", "would delete index file {0}", vol.Name); + else + await backendManager.DeleteAsync(vol.Name, vol.Size, false, cancellationToken).ConfigureAwait(false); + + // All done, the new dblocks are now in place + await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); + if (!m_options.Dryrun) + rtr.Commit("PostRepairBlockVolume"); + + activeVolume.CompletedWriters.Clear(); + activeVolume.VolumesToDelete.Clear(); + } + + /// + /// Repairs a single dblock file by recreating it from available data + /// + /// The backend manager to use for uploading + /// The database to use for the repair + /// The transaction to use for the repair + /// The remote volume entry to recreate + /// The in-progress dblock volume to use for the recreate + /// Cancellation token to cancel the operation + /// >A task representing the asynchronous operation + private async Task RunRepairDblock(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, RemoteVolumeEntry originalVolume, InProgressDblockVolumes pendingVolume, CancellationToken cancellationToken) + { + // The dblock files are the most complex to recreate + // as data can be either file contents, metadata or blocklist hashes + // We attempt to recover all three source parts in the steps below + using (var mbl = db.CreateBlockList(originalVolume.Name, rtr)) + { + var originalMissingBlockCount = mbl.GetMissingBlockCount(); + var recoveredSourceBlocks = 0L; + + // First we grab all known blocks from local files + string lastRestoredHash = null; + long lastRestoredSize = -1; + foreach (var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize)) + { + if (block.Hash == lastRestoredHash && block.Size == lastRestoredSize) + continue; + + var size = (int)block.Size; + + try + { + if (!File.Exists(block.File)) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "FileNotFound", null, "File not found: {0}", block.File); + continue; + } + + using var f = File.OpenRead(block.File); + f.Position = block.Offset; + + var buffer = new byte[m_options.Blocksize]; + if (size != Library.Utility.Utility.ForceStreamRead(f, buffer, size)) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "FileLengthMismatch", null, "Block length mismatch on {0}, expected {1} but got {2}", block.File, size, f.Length); + continue; + } + + using var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm); + var newhash = Convert.ToBase64String(blockhasher.ComputeHash(buffer, 0, size)); + if (newhash != block.Hash) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "FileHashMismatch", null, "Block hash mismatch on {0}, expected {1} but got {2}", block.File, block.Hash, newhash); + continue; + } + + // Found it, no need to look again + lastRestoredHash = block.Hash; + lastRestoredSize = block.Size; + if (mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID)) + { + recoveredSourceBlocks++; + pendingVolume.AddBlock(block.Hash, buffer, 0, size, CompressionHint.Default); + } + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", ex, "Failed to access file: {0}", block.File); + } + } + + // Then grab all blocks with metadata + foreach (var block in mbl.GetSourceItemsWithMetadataBlocks()) + { + if (block.Hash == lastRestoredHash && block.Size == lastRestoredSize) + continue; + + var size = (int)block.Size; + + try + { + var isFile = File.Exists(block.Path); + var isDir = Directory.Exists(block.Path); + if (!isFile && !isDir) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "EntryNotFound", null, "Entry not found: {0}", block.Path); + continue; + } + + using var snapshot = Snapshots.SnapshotUtility.CreateNoSnapshot([block.Path], true, true); + var entry = snapshot.GetFilesystemEntry(block.Path, isDir); + if (entry == null) + { + Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", null, "Entry not found: {0}", block.Path); + continue; + } + + var metadata = MetadataGenerator.GenerateMetadata(entry, entry.Attributes, m_options); + var metahash = Utility.WrapMetadata(metadata, m_options); + if (metahash.FileHash != block.Hash || metahash.Blob.Length != size) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "MetadataHashMismatch", null, "Metadata block hash mismatch on {0}, expected {1} but got {2}", block.Path, block.Hash, metahash.FileHash); + continue; + } + + // Found it, no need to look again + lastRestoredHash = block.Hash; + lastRestoredSize = block.Size; + if (mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID)) + { + recoveredSourceBlocks++; + pendingVolume.AddBlock(block.Hash, metahash.Blob, 0, metahash.Blob.Length, CompressionHint.Default); + } + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", ex, "Failed to access file: {0}", block.Path); + } + } + + // If we did not find any source blocks, + // we only have the database contents and duplicated blocks to work with + // so no point in creating a new dblock volume for storing this data + if (recoveredSourceBlocks != 0) + { + // Then restore any blocklists that are missing + using var blocklistHash = new MemoryStream(); + LocalRepairDatabase.BlocklistHashesEntry lastBlocklist = null; + var blockhashsize = m_options.BlockhashSize; + var hashesPerBlock = m_options.Blocksize / m_options.BlockhashSize; + + // Helper function to emit a blocklist hash + void EmitBlockListBlock() + { + blocklistHash.Position = 0; + using var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm); + var resulthash = Convert.ToBase64String(blockhasher.ComputeHash(blocklistHash)); + var totalHashes = (lastBlocklist.BlocklistHashLength + m_options.Blocksize - 1) / m_options.Blocksize; + var hashesInLastBlock = totalHashes % hashesPerBlock; + var targetsize = lastBlocklist.Index >= totalHashes - 1 + ? hashesInLastBlock * blockhashsize + : hashesPerBlock * blockhashsize; + + if (resulthash == lastBlocklist.BlocklistHash && targetsize == blocklistHash.Length) + { + if (mbl.SetBlockRestored(resulthash, blocklistHash.Length, pendingVolume.VolumeID)) + pendingVolume.AddBlock(resulthash, blocklistHash.ToArray(), 0, (int)blocklistHash.Length, CompressionHint.Default); + } + else + { + Logging.Log.WriteWarningMessage(LOGTAG, "BlocklistHashMismatch", null, "Internal consistency issue: blocklist hash mismatch on {0} / {1}, expected {2} but got {3}", lastBlocklist.BlocklistHash, lastBlocklist.Index, lastBlocklist.Hash, resulthash); + } + + blocklistHash.SetLength(0); + } + + foreach (var blocklist in mbl.GetBlocklistHashes(hashesPerBlock)) + { + lastBlocklist ??= blocklist; + + if (lastBlocklist.BlocksetId != blocklist.BlocksetId || lastBlocklist.BlocklistHashIndex != blocklist.BlocklistHashIndex) + EmitBlockListBlock(); + + var data = Convert.FromBase64String(blocklist.Hash); + blocklistHash.Write(data, 0, data.Length); + lastBlocklist = blocklist; + } + + // Handle any trailing blocklist hash + if (blocklistHash.Length > 0) + EmitBlockListBlock(); + + //Then we grab all remote volumes that have the missing blocks + await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(mbl.GetMissingBlockSources().ToList(), cancellationToken).ConfigureAwait(false)) + { + try + { + var buffer = new byte[m_options.Blocksize]; + using (tmpfile) + using (var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options)) + foreach (var b in f.Blocks) + if (mbl.SetBlockRestored(b.Key, b.Value, pendingVolume.VolumeID)) + if (f.ReadBlock(b.Key, buffer) == b.Value) + pendingVolume.AddBlock(b.Key, buffer, 0, (int)b.Value, CompressionHint.Default); + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileAccessError", ex, "Failed to access remote file: {0}", name); + } + } + } + + var missingBlocks = mbl.GetMissingBlockCount(); + var recoveredBlocks = originalMissingBlockCount - missingBlocks; + if (recoveredBlocks == 0 || (m_options.DisablePartialDblockRecovery && missingBlocks > 0)) + { + Logging.Log.WriteInformationMessage(LOGTAG, "RepairMissingBlocks", "Repair cannot acquire {0} required blocks for volume {1}, which are required by the following filesets: ", missingBlocks, originalVolume.Name); + foreach (var f in mbl.GetFilesetsUsingMissingBlocks()) + Logging.Log.WriteInformationMessage(LOGTAG, "AffectedFilesetName", f.Name); + + var recoverymsg = string.Format("If you want to continue working with the database, you can use the \"{0}\" and \"{1}\" commands to purge the missing data from the database and the remote storage.", "list-broken-files", "purge-broken-files"); + var logmsg = string.Format("Repair not possible, missing {0} blocks.\n" + recoverymsg, missingBlocks); + + Logging.Log.WriteInformationMessage(LOGTAG, "RecoverySuggestion", null, logmsg); + throw new UserInformationException(logmsg, "RepairIsNotPossible"); + } + else if (recoveredBlocks > 0) + { + if (missingBlocks > 0) + Logging.Log.WriteWarningMessage(LOGTAG, "RepairMissingBlocks", null, "Repair acquired {0} blocks for volume {1}, but {2} blocks are still missing. If you want to continue working with the database, you can use the \"{3}\" and \"{4}\" commands to purge the missing data from the database and the remote storage.", recoveredBlocks, originalVolume.Name, missingBlocks, "list-broken-files", "purge-broken-files"); + + // If we cannot fully recover the blocks, we will not be able to delete the old files. + // They will instead be registered with duplicated blocks, so the purge-broken-files command + // can remove them cleanly later + if (missingBlocks == 0) + { + pendingVolume.VolumesToDelete.Add(originalVolume); + var oldIndexFiles = db.GetIndexFilesReferencingBlockFile(originalVolume.ID, rtr.Transaction).ToList(); + + // Find all index files that point to the old volume + foreach (var oldIndexFile in oldIndexFiles) + { + var oldVolume = db.GetRemoteVolume(oldIndexFile, rtr.Transaction); + if (oldVolume.State == RemoteVolumeState.Uploading || oldVolume.State == RemoteVolumeState.Uploaded || oldVolume.State == RemoteVolumeState.Verified) + { + var blockVolumesReferenced = db.GetBlockVolumesFromIndexName(oldIndexFile).ToList(); + if (blockVolumesReferenced.Any(x => x.Name != originalVolume.Name)) + Logging.Log.WriteVerboseMessage(LOGTAG, "IndexFileNotDeleted", null, "Index file {0} references multiple remote volumes, skipping", oldVolume.Name); + else + pendingVolume.VolumesToDelete.Add(oldVolume); + } + else + { + Logging.Log.WriteVerboseMessage(LOGTAG, "IndexFileNotDeleted", null, "Index file {0} is not in a state to be deleted, skipping", oldVolume.Name); + } + } + } + } } } diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index f5f7bd8fb..18312efaa 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -471,6 +471,8 @@ namespace Duplicati.Library.Main new CommandLineArgument("ignore-filenames", CommandLineArgument.ArgumentType.Path, Strings.Options.IgnorefilenamesShort, Strings.Options.IgnorefilenamesLong, "CACHEDIR.TAG"), new CommandLineArgument("restore-symlink-metadata", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RestoresymlinkmetadataShort, Strings.Options.RestoresymlinkmetadataLong, "false"), new CommandLineArgument("rebuild-missing-dblock-files", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RebuildmissingdblockfilesShort, Strings.Options.RebuildmissingdblockfilesLong, "false"), + new CommandLineArgument("disable-partial-dblock-recovery", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablePartialDblockRecoveryShort, Strings.Options.DisablePartialDblockRecoveryLong, "false"), + new CommandLineArgument("disable-replace-missing-metadata", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisableReplaceMissingMetadataShort, Strings.Options.DisableReplaceMissingMetadataLong, "false"), new CommandLineArgument("auto-compact-interval", CommandLineArgument.ArgumentType.Timespan, Strings.Options.AutoCompactIntervalShort, Strings.Options.AutoCompactIntervalLong, "0m"), new CommandLineArgument("auto-vacuum-interval", CommandLineArgument.ArgumentType.Timespan, Strings.Options.AutoVacuumIntervalShort, Strings.Options.AutoVacuumIntervalLong, "0m"), @@ -1927,6 +1929,22 @@ namespace Duplicati.Library.Main get { return GetBool("rebuild-missing-dblock-files"); } } + /// + /// Gets a value indicating if partial dblock recovery is disabled + /// + public bool DisablePartialDblockRecovery + { + get { return GetBool("disable-partial-dblock-recovery"); } + } + + /// + /// Gets a value indicating if missing metadata is replaced with empty content on purge-broken-files + /// + public bool DisableReplaceMissingMetadata + { + get { return GetBool("disable-replace-missing-metadata"); } + } + /// /// Gets the threshold for when log data should be cleaned /// diff --git a/Duplicati/Library/Main/ResultClasses.cs b/Duplicati/Library/Main/ResultClasses.cs index 14d0909df..2ab7aa2ca 100644 --- a/Duplicati/Library/Main/ResultClasses.cs +++ b/Duplicati/Library/Main/ResultClasses.cs @@ -902,6 +902,7 @@ namespace Duplicati.Library.Main public long RemovedFileCount { get; set; } public long RemovedFileSize { get; set; } public long RewrittenFileLists { get; set; } + public long UpdatedFileCount { get; set; } public ICompactResults CompactResults { get; set; } } diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index e24966af4..6db36589e 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -310,6 +310,10 @@ namespace Duplicati.Library.Main.Strings public static string ProfilealldatabasequeriesShort { get { return LC.L("Activate logging of all database queries"); } } public static string RebuildmissingdblockfilesLong { get { return LC.L("If dblock files are missing from the destination, you can attempt to rebuild them using local source data. However, since the local data may have changed, it may not be possible to retrieve all the required data and the process may be slow. Use this option to attempt to rebuild missing dblock files."); } } public static string RebuildmissingdblockfilesShort { get { return LC.L("Rebuild dblock files when missing"); } } + public static string DisablePartialDblockRecoveryLong { get { return LC.L("If rebuilding dblock files, the process will attempt to use local data to fill in missing blocks. If it is not possible to fill in all missing blocks, Duplicati will attempt to create a dblock with as much data as possible. Use this option to disable this behavior and only create dblocks that are complete."); } } + public static string DisablePartialDblockRecoveryShort { get { return LC.L("Disable partial dblock recovery"); } } + public static string DisableReplaceMissingMetadataLong { get { return LC.L($"If metadata is missing during the {"list-broken-files"} or {"purge-broken-files"} command, Duplicati will attempt to replace the missing metadata with a new copy. Use this option to disable this behavior and drop entries that are missing metadata."); } } + public static string DisableReplaceMissingMetadataShort { get { return LC.L("Disable replacement of missing metadata"); } } public static string AutoCompactIntervalLong { get { return LC.L("The minimum amount of time that must elapse after the last compaction before another will be automatically triggered at the end of a backup job. Automatic compaction can be a long-running process and may not be desirable to run after every single backup."); } } public static string AutoCompactIntervalShort { get { return LC.L("Minimum time between auto compactions"); } } diff --git a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs index 4bf79d88e..79f384494 100644 --- a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs +++ b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs @@ -27,6 +27,7 @@ using Duplicati.Library.Main.Database; using Duplicati.Library.Main.Operation.Common; using System.Threading.Tasks; using Duplicati.Library.Utility; +using System.Data; namespace Duplicati.Library.Main.Volumes { @@ -244,12 +245,13 @@ namespace Duplicati.Library.Main.Volumes /// /// The database to check for clashes /// The options to use for the filename + /// The transaction to use /// The starting time to probe from /// The time to increment by each probe /// The maximum number of tries to probe /// The first unused filename - internal static DateTime ProbeUnusedFilenameName(LocalDatabase database, Options options, DateTime start, TimeSpan increment = default, int maxTries = 60) - => ProbeUnusedFilenameName((name) => Task.FromResult(database.GetRemoteVolumeID(name)), options, start, increment, maxTries).Await(); + internal static DateTime ProbeUnusedFilenameName(LocalDatabase database, IDbTransaction transaction, Options options, DateTime start, TimeSpan increment = default, int maxTries = 60) + => ProbeUnusedFilenameName((name) => Task.FromResult(database.GetRemoteVolumeID(name, transaction)), options, start, increment, maxTries).Await(); /// /// Probes for an unused filename, using the current time as a starting point diff --git a/Duplicati/UnitTest/Issue5987.cs b/Duplicati/UnitTest/Issue5987.cs new file mode 100644 index 000000000..63073a292 --- /dev/null +++ b/Duplicati/UnitTest/Issue5987.cs @@ -0,0 +1,65 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System.IO; +using System.Linq; +using NUnit.Framework; + +namespace Duplicati.UnitTest +{ + public class Issue5987 : BasicSetupHelper + { + [Test] + [Category("Targeted")] + [TestCase(true)] + [TestCase(false)] + public void TestRestoreWithMissingDblocks(bool deleteIndexFiles) + { + var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); + + // 1. Make a backup of a single file + File.WriteAllText(Path.Combine(DATAFOLDER, "a"), "abc"); + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete the dblock file + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dblockFiles) + File.Delete(file); + + if (deleteIndexFiles) + { + var dindexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dindexFiles) + File.Delete(file); + } + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + var res = c.Repair(); + Assert.That(res.Errors.Count(), Is.EqualTo(0), "Repair should not have errors"); + TestUtils.AssertResults(c.Test()); + } + } + } +} + diff --git a/Duplicati/UnitTest/RepairHandlerTests.cs b/Duplicati/UnitTest/RepairHandlerTests.cs index 533b67d06..6b1f560b0 100644 --- a/Duplicati/UnitTest/RepairHandlerTests.cs +++ b/Duplicati/UnitTest/RepairHandlerTests.cs @@ -164,7 +164,7 @@ namespace Duplicati.UnitTest Assert.AreEqual(0, backupResults.Warnings.Count()); } - var dindexFiles = Directory.EnumerateFiles(this.TARGETFOLDER, "*dindex*").ToArray(); + var dindexFiles = Directory.EnumerateFiles(this.TARGETFOLDER, "*.dindex.*").ToArray(); Assert.Greater(dindexFiles.Length, 0); foreach (var f in dindexFiles) { @@ -178,10 +178,8 @@ namespace Duplicati.UnitTest Assert.AreEqual(0, repairResults.Warnings.Count()); } - foreach (var file in dindexFiles) - { - Assert.IsTrue(File.Exists(Path.Combine(this.TARGETFOLDER, file))); - } + var recreatedIndexFiles = Directory.EnumerateFiles(this.TARGETFOLDER, "*dindex*").ToArray(); + Assert.AreEqual(dindexFiles.Length, recreatedIndexFiles.Length); } [Test] @@ -216,7 +214,7 @@ namespace Duplicati.UnitTest Assert.AreEqual(0, backupResults.Warnings.Count()); } - var dindexFiles = Directory.EnumerateFiles(this.TARGETFOLDER, "*dindex*").ToArray(); + var dindexFiles = Directory.EnumerateFiles(this.TARGETFOLDER, "*.dindex.*").ToArray(); Assert.Greater(dindexFiles.Length, 0); foreach (var f in dindexFiles) { @@ -230,10 +228,8 @@ namespace Duplicati.UnitTest Assert.AreEqual(0, repairResults.Warnings.Count()); } - foreach (var file in dindexFiles) - { - Assert.IsTrue(File.Exists(Path.Combine(this.TARGETFOLDER, file))); - } + var recreatedIndexFiles = Directory.EnumerateFiles(this.TARGETFOLDER, "*dindex*").ToArray(); + Assert.AreEqual(dindexFiles.Length, recreatedIndexFiles.Length); } [Test] diff --git a/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs b/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs new file mode 100644 index 000000000..827f23fc3 --- /dev/null +++ b/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs @@ -0,0 +1,640 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Duplicati.Library.Interface; +using Duplicati.Library.Main.Database; +using Duplicati.Library.SQLiteHelper; +using NUnit.Framework; + +namespace Duplicati.UnitTest +{ + public class RepairWithMissingRemoteFiles : BasicSetupHelper + { + // TODO: Add more tests to cover the following scenarios: + // Test where a single block in a large file is missing + run purge-broken-files + // Test that fetching remote sources for blocks is working + + [Test] + [Category("Targeted")] + [TestCase(false, true, false)] + [TestCase(true, true, false)] + [TestCase(true, false, false)] + public void TestRepairWithMissingDblocks(bool deleteDblockFiles, bool deleteIndexFiles, bool deleteDlistFiles) + { + var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true, number_of_retries = 0 }); + + // 1. Make a backup of a single file (1 block + 3 bytes for a block with less data) + var data = new byte[1024 + 3]; + data.AsSpan().Fill((byte)'a'); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete files + if (deleteDblockFiles) + { + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dblockFiles) + File.Delete(file); + } + + if (deleteIndexFiles) + { + var dindexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dindexFiles) + File.Delete(file); + } + + if (deleteDlistFiles) + { + var dlistFiles = Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dlistFiles) + File.Delete(file); + } + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Repair()); + + // 4. Test that the database is now valid + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Test()); + } + + [Test] + [Category("Targeted")] + [TestCase(false, true, false)] + [TestCase(true, true, false)] + [TestCase(true, false, false)] + public void TestRepairWithMissingDblocksAndMultipleSources(bool deleteDblockFiles, bool deleteIndexFiles, bool deleteDlistFiles) + { + var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); + + // 1. Make some shared data + var data = new byte[1024 + 3]; + data.AsSpan().Fill((byte)'a'); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "b"), data.AsSpan().Slice(0, 1024).ToArray()); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "c"), data.AsSpan().Slice(0, 1024).ToArray()); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "d"), data.AsSpan().Slice(1024, 3).ToArray()); + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete files + if (deleteDblockFiles) + { + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dblockFiles) + File.Delete(file); + } + + if (deleteIndexFiles) + { + var dindexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dindexFiles) + File.Delete(file); + } + + if (deleteDlistFiles) + { + var dlistFiles = Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dlistFiles) + File.Delete(file); + } + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + TestUtils.AssertResults(c.Repair()); + TestUtils.AssertResults(c.Test()); + } + } + + [Test] + [Category("Targeted")] + [TestCase(false, true, false)] + [TestCase(true, true, false)] + [TestCase(true, false, false)] + public void TestRepairWithDataFillingMultipleBlocklists(bool deleteDblockFiles, bool deleteIndexFiles, bool deleteDlistFiles) + { + var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); + + // 1. Make some shared data 1kb blocks, 32 bytes hash, 3 blocklists, and 5 bytes extra + var data = new byte[1024 * 32 * 3 + 5]; + data.AsSpan().Fill((byte)'a'); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "b"), data.AsSpan().Slice(1024, 3).ToArray()); + Random.Shared.NextBytes(data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "c"), data); + Random.Shared.NextBytes(data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "d"), data.AsSpan().Slice(0, 1028).ToArray()); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "e"), data.AsSpan().Slice(4098, 67).ToArray()); + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete files + if (deleteDblockFiles) + { + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dblockFiles) + File.Delete(file); + } + + if (deleteIndexFiles) + { + var dindexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dindexFiles) + File.Delete(file); + } + + if (deleteDlistFiles) + { + var dlistFiles = Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dlistFiles) + File.Delete(file); + } + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + TestUtils.AssertResults(c.Repair()); + TestUtils.AssertResults(c.Test()); + } + } + + [Test] + [Category("Targeted")] + [TestCase(true, true, true)] + [TestCase(true, false, false)] + public void TestRepairWithSmallFileOnly(bool deleteDblockFiles, bool deleteIndexFiles, bool deleteDlistFiles) + { + var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); + + // 1. Make some shared data + var data = new byte[3]; + data.AsSpan().Fill((byte)'a'); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete files + if (deleteDblockFiles) + { + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dblockFiles) + File.Delete(file); + } + + if (deleteIndexFiles) + { + var dindexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dindexFiles) + File.Delete(file); + } + + if (deleteDlistFiles) + { + var dlistFiles = Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dlistFiles) + File.Delete(file); + } + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + TestUtils.AssertResults(c.Repair()); + TestUtils.AssertResults(c.Test()); + } + } + + [Test] + [Category("Targeted")] + [TestCase(false, true, false)] + [TestCase(true, true, false)] + [TestCase(true, false, false)] + [TestCase(true, true, true)] + [TestCase(true, false, true)] + public void TestRepairWithDataFillingMultipleBlocklistsRandom(bool deleteDblockFiles, bool deleteIndexFiles, bool deleteDlistFiles) + { + var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); + + // 1. Make some shared data 1kb blocks, 32 bytes hash, 3 blocklists, and 1028 bytes extra + var data = new byte[1024 * 32 * 3 + 1028]; + Random.Shared.NextBytes(data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete files + if (deleteDblockFiles) + { + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dblockFiles) + File.Delete(file); + } + + if (deleteIndexFiles) + { + var dindexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dindexFiles) + File.Delete(file); + } + + if (deleteDlistFiles) + { + var dlistFiles = Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dlistFiles) + File.Delete(file); + } + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + TestUtils.AssertResults(c.Repair()); + TestUtils.AssertResults(c.Test()); + } + + // 4. Verify that restore works + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { restore_path = RESTOREFOLDER }), null)) + TestUtils.AssertResults(c.Restore(null)); + + TestUtils.AssertDirectoryTreesAreEquivalent(DATAFOLDER, RESTOREFOLDER, !Library.Utility.Utility.ParseBoolOption(testopts, "skip-metadata"), "Restore"); + } + + [Test] + [Category("Targeted")] + [TestCase(true, true, true)] + [TestCase(true, false, false)] + public void TestRepairWithDryRun(bool deleteDblockFiles, bool deleteIndexFiles, bool deleteDlistFiles) + { + var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); + + // 1. Make some random data + var data = new byte[1028]; + Random.Shared.NextBytes(data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete files + if (deleteDblockFiles) + { + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dblockFiles) + File.Delete(file); + } + + if (deleteIndexFiles) + { + var dindexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dindexFiles) + File.Delete(file); + } + + if (deleteDlistFiles) + { + var dlistFiles = Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dlistFiles) + File.Delete(file); + } + + (long remoteVolumes, long duplicatedBlocks, List blockVolumeIds, List<(long, long)> indexVolumeLinks) GetDbSignature() + { + using var db = SQLiteLoader.LoadConnection(DBFILE, 0); + using var cmd = db.CreateCommand(); + + var remoteVolumes = cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM RemoteVolume"); + var duplicatedBlocks = cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM DuplicateBlock"); + var blockVolumeIds = cmd.ExecuteReaderEnumerable("SELECT VolumeID FROM Block").Select(x => x.ConvertValueToInt64(0)).ToList(); + var indexVolumeLinks = cmd.ExecuteReaderEnumerable("SELECT IndexVolumeID, BlockVolumeID FROM IndexBlockLink").Select(x => (x.ConvertValueToInt64(0), x.ConvertValueToInt64(1))).ToList(); + return (remoteVolumes, duplicatedBlocks, blockVolumeIds, indexVolumeLinks); + } + + var preSignature = GetDbSignature(); + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { dry_run = true }), null)) + TestUtils.AssertResults(c.Repair()); + + var postSignature = GetDbSignature(); + + Assert.AreEqual(preSignature.remoteVolumes, postSignature.remoteVolumes, "Remote volumes count should be the same after dry run repair."); + Assert.AreEqual(preSignature.duplicatedBlocks, postSignature.duplicatedBlocks, "Duplicated blocks count should be the same after dry run repair."); + Assert.AreEqual(preSignature.blockVolumeIds.Count, postSignature.blockVolumeIds.Count, "Block volume IDs count should be the same after dry run repair."); + Assert.AreEqual(preSignature.indexVolumeLinks.Count, postSignature.indexVolumeLinks.Count, "Index volume links count should be the same after dry run repair."); + + Assert.IsTrue(preSignature.blockVolumeIds.All(x => postSignature.blockVolumeIds.Contains(x)), "All block volume IDs should be present after dry run repair."); + Assert.IsTrue(preSignature.indexVolumeLinks.All(x => postSignature.indexVolumeLinks.Contains(x)), "All index volume links should be present after dry run repair."); + } + + [Test] + [Category("Targeted")] + public void TestRepairWithNoBlockAvailableFails() + { + var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); + + // 1. Make some shared data 1kb blocks, 32 bytes hash, 3 blocklists, and 1028 bytes extra + var data = new byte[1024 * 32 * 3 + 1028]; + Random.Shared.NextBytes(data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete files + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dblockFiles) + File.Delete(file); + + // Destroy source data + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), new byte[0]); + Directory.SetLastWriteTime(DATAFOLDER, DateTime.UtcNow.AddSeconds(1)); + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + var res = c.Repair(); + Assert.IsTrue(res.Errors.Where(x => x.Contains("purge-broken-files")).Count() == 1, "Repair should fail with missing block data."); + } + } + + [Test] + [Category("Targeted")] + public void TestPartialRepairPossibleWithMissingMetadata() + { + var testopts = TestOptions.Expand(new + { + blocksize = "1kb", + no_encryption = true, + rebuild_missing_dblock_files = true + }); + + // 1. Make some shared data 1kb blocks, 32 bytes hash, 3 blocklists, and 1028 bytes extra + var data = new byte[1024 * 32 * 3 + 1028]; + Random.Shared.NextBytes(data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete files + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dblockFiles) + File.Delete(file); + + // Destroy some source data + Directory.SetLastWriteTime(DATAFOLDER, DateTime.UtcNow.AddSeconds(1)); + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + var res = c.Repair(); + Assert.IsTrue(res.Warnings.Where(x => x.Contains("purge-broken-files")).Count() == 1, "Repair should warn about missing block data."); + + // Ensure that the partial repair did not clean up fully + Assert.Throws(() => c.Test()); + + // Purge broken files should be possible + TestUtils.AssertResults(c.PurgeBrokenFiles(null)); + TestUtils.AssertResults(c.Test(int.MaxValue)); + } + + // 4. Verify that restore works + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { restore_path = RESTOREFOLDER }), null)) + TestUtils.AssertResults(c.Restore(null)); + + TestUtils.AssertDirectoryTreesAreEquivalent(DATAFOLDER, RESTOREFOLDER, !Library.Utility.Utility.ParseBoolOption(testopts, "skip-metadata"), "Restore"); + } + + [Test] + [Category("Targeted")] + [TestCase(false)] + [TestCase(true)] + public void TestPartialRepairPossibleWithDeletedFiles(bool deleteLargeFile) + { + var testopts = TestOptions.Expand(new + { + blocksize = "1kb", + no_encryption = true, + rebuild_missing_dblock_files = true + }); + + // 1. Make some shared data 1kb blocks, 32 bytes hash, 3 blocklists, and 1028 bytes extra + var data = new byte[1024 * 32 * 3 + 1028]; + Random.Shared.NextBytes(data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "b"), data.AsSpan().Slice(0, 1024).ToArray()); + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete files + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dblockFiles) + File.Delete(file); + + // Destroy some source data (also destroys the metadata) + if (deleteLargeFile) + File.Delete(Path.Combine(DATAFOLDER, "a")); + else + File.Delete(Path.Combine(DATAFOLDER, "b")); + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + var res = c.Repair(); + Assert.IsTrue(res.Warnings.Where(x => x.Contains("purge-broken-files")).Count() == 1, "Repair should warn about missing block data."); + + // Ensure that the partial repair did not clean up fully + Assert.Throws(() => c.Test()); + + // Purge broken files should be possible + TestUtils.AssertResults(c.PurgeBrokenFiles(null)); + TestUtils.AssertResults(c.Test(int.MaxValue)); + + var files = c.List("*").Files.ToList(); + Console.WriteLine("Files: " + string.Join(", ", files.Select(x => x.Path))); + } + + // 4. Verify that restore works + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { restore_path = RESTOREFOLDER }), null)) + TestUtils.AssertResults(c.Restore(null)); + + Assert.That(File.Exists(Path.Combine(RESTOREFOLDER, "a")), Is.EqualTo(!deleteLargeFile), "Restored file 'a' should be restored."); + Assert.That(File.Exists(Path.Combine(RESTOREFOLDER, "b")), Is.EqualTo(true), "File 'b' should always be restored."); + if (!deleteLargeFile) + Assert.That(File.ReadAllBytes(Path.Combine(RESTOREFOLDER, "a")), Is.EqualTo(data), "Restored file 'a' should be the same as the original."); + else + Assert.That(File.ReadAllBytes(Path.Combine(RESTOREFOLDER, "b")), Is.EqualTo(data.AsSpan().Slice(0, 1024).ToArray()), "Restored file 'b' should be the same as the original."); + } + + [Test] + [Category("Targeted")] + [TestCase(0)] // Large file only + [TestCase(1)] // Small file only + [TestCase(2)] // Both files + [TestCase(3)] // Metadata only + public void TestPartialRepairPossibleWithPartialData(int scenario) + { + var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); + + // 1. Make some shared data 1kb blocks, 32 bytes hash, 3 blocklists, and 1028 bytes extra + var data = new byte[1024 * 32 * 3 + 1028]; + Random.Shared.NextBytes(data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "b"), data.AsSpan().Slice(0, 1024).ToArray()); + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete files + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dblockFiles) + File.Delete(file); + + var corruptLargeFile = scenario == 0 || scenario == 2; + var corruptSmallFile = scenario == 1 || scenario == 2; + var corruptBothFiles = corruptLargeFile && corruptSmallFile; + var destroyMetadata = scenario == 3; + + // Destroy some source data (keep metadata) + var prev = data[1]; + data[1] = 1; + if (corruptLargeFile) + { + var timestamp = File.GetLastWriteTime(Path.Combine(DATAFOLDER, "a")); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + File.SetLastWriteTime(Path.Combine(DATAFOLDER, "a"), timestamp); + } + + if (corruptSmallFile) + { + var timestamp = File.GetLastWriteTime(Path.Combine(DATAFOLDER, "b")); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "b"), data.AsSpan().Slice(0, 1024).ToArray()); + File.SetLastWriteTime(Path.Combine(DATAFOLDER, "b"), timestamp); + } + data[1] = prev; + + // Make sure that destroying the metadata does not destroy the files + if (destroyMetadata) + { + File.SetLastWriteTime(Path.Combine(DATAFOLDER, "a"), DateTime.UtcNow.AddSeconds(1)); + File.SetLastWriteTime(Path.Combine(DATAFOLDER, "b"), DateTime.UtcNow.AddSeconds(1.1)); + } + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + var res = c.Repair(); + + if (corruptBothFiles || destroyMetadata) + { + Assert.IsTrue(res.Warnings.Where(x => x.Contains("purge-broken-files")).Count() == 1, "Repair should warn about missing block data."); + + // Ensure that the partial repair did not clean up fully + Assert.Throws(() => c.Test()); + } + else + { + TestUtils.AssertResults(res); + TestUtils.AssertResults(c.Test(int.MaxValue)); + } + + // Purge broken files should be possible + TestUtils.AssertResults(c.PurgeBrokenFiles(null)); + TestUtils.AssertResults(c.Test(int.MaxValue)); + } + + // 4. Verify that restore works + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { restore_path = RESTOREFOLDER }), null)) + { + var res = c.Restore(null); + if (corruptBothFiles) + { + // We get a warning because neither file can be restored + Assert.That(res.Errors.Count(), Is.EqualTo(0), "Restore should not fail."); + } + else + { + TestUtils.AssertResults(res); + } + } + + Assert.That(File.Exists(Path.Combine(RESTOREFOLDER, "a")), Is.EqualTo(!corruptBothFiles), "File 'a' should be restored."); + Assert.That(File.Exists(Path.Combine(RESTOREFOLDER, "b")), Is.EqualTo(!corruptBothFiles), "File 'b' should be restored."); + if (!corruptLargeFile) + Assert.That(Convert.ToBase64String(File.ReadAllBytes(Path.Combine(RESTOREFOLDER, "a"))), Is.EqualTo(Convert.ToBase64String(data)), "Restored file 'a' should be the same as the original."); + else if (!corruptBothFiles) + Assert.That(Convert.ToBase64String(File.ReadAllBytes(Path.Combine(RESTOREFOLDER, "b"))), Is.EqualTo(Convert.ToBase64String(data.AsSpan().Slice(0, 1024))), "Restored file 'b' should be the same as the original."); + } + + [Test] + [Category("Targeted")] + [TestCase(1)] + [TestCase(5)] + [TestCase(10)] + public void TestRepairPossibleWithMultipleDblockFiles(int dblocksToDelete) + { + var testopts = TestOptions.Expand(new + { + blocksize = "1kb", + no_encryption = true, + rebuild_missing_dblock_files = true, + zip_compression_level = 0, + dblock_size = "10kb" + }); + + // 1. Make some shared data 1kb blocks, 32 bytes hash, 3 blocklists, and 1028 bytes extra + var data = new byte[1024 * 32 * 3 + 1028]; + Random.Shared.NextBytes(data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete files + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToArray(); + Random.Shared.Shuffle(dblockFiles); + foreach (var file in dblockFiles.Take(dblocksToDelete)) + File.Delete(file); + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + TestUtils.AssertResults(c.Repair()); + TestUtils.AssertResults(c.Test()); + } + + // 4. Verify that restore works + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { restore_path = RESTOREFOLDER }), null)) + TestUtils.AssertResults(c.Restore(null)); + + TestUtils.AssertDirectoryTreesAreEquivalent(DATAFOLDER, RESTOREFOLDER, !Library.Utility.Utility.ParseBoolOption(testopts, "skip-metadata"), "Restore"); + } + } +} + diff --git a/Duplicati/UnitTest/TestUtils.cs b/Duplicati/UnitTest/TestUtils.cs index 6243a31d6..1f58d7a80 100644 --- a/Duplicati/UnitTest/TestUtils.cs +++ b/Duplicati/UnitTest/TestUtils.cs @@ -296,8 +296,6 @@ namespace Duplicati.UnitTest { operation = ((Library.Main.OperationMode)operationProperty.GetValue(results)).ToString(); } - Assert.AreEqual(0, results.Errors.Count(), "{0} errors:\n{1}", operation, string.Join("\n", results.Errors)); - Assert.AreEqual(0, results.Warnings.Count(), "{0} warnings:\n{1}", operation, string.Join("\n", results.Warnings)); if (results is ITestResults testResults) { @@ -322,6 +320,24 @@ namespace Duplicati.UnitTest throw new TestVerificationException(sb.ToString()); } } + + if (results.Errors.Count() != 0) + { + var sb = new StringBuilder(); + sb.AppendLine($"Errors - {operation}:"); + foreach (var e in results.Errors) + sb.AppendLine(e.ToString()); + throw new TestVerificationException(sb.ToString()); + } + + if (results.Warnings.Count() != 0) + { + var sb = new StringBuilder(); + sb.AppendLine($"Warnings - {operation}:"); + foreach (var w in results.Warnings) + sb.AppendLine(w.ToString()); + throw new TestVerificationException(sb.ToString()); + } } public static string DumpTable(IDbCommand cmd, string table, string where = null)