From 5119b67438c6c8f606f632105c4e9c86f825e05b Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 5 May 2025 16:41:47 +0200 Subject: [PATCH 01/15] Simplified log code --- Duplicati/Library/Main/Operation/RepairHandler.cs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index a1ef89596..3e095ca93 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -536,17 +536,11 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "AffectedFilesetName", f.Name); var recoverymsg = string.Format("If you want to continue working with the database, you can use the \"{0}\" and \"{1}\" commands to purge the missing data from the database and the remote storage.", "list-broken-files", "purge-broken-files"); + var logmsg = string.Format("Repair not possible, missing {0} blocks.\n" + recoverymsg, missingBlocks); + Logging.Log.WriteInformationMessage(LOGTAG, "RecoverySuggestion", null, logmsg); if (!m_options.Dryrun) - { - Logging.Log.WriteInformationMessage(LOGTAG, "RecoverySuggestion", "This may be fixed by deleting the filesets and running repair again"); - - throw new UserInformationException(string.Format("Repair not possible, missing {0} blocks.\n" + recoverymsg, missingBlocks), "RepairIsNotPossible"); - } - else - { - Logging.Log.WriteInformationMessage(LOGTAG, "RecoverySuggestion", recoverymsg); - } + throw new UserInformationException(logmsg, "RepairIsNotPossible"); } else { From a9a662e3fee325b44c64e1b78001058f9c197236 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Mon, 5 May 2025 16:42:26 +0200 Subject: [PATCH 02/15] Added test to reproduce #5987 --- Duplicati/UnitTest/Issue5987.cs | 65 +++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 Duplicati/UnitTest/Issue5987.cs diff --git a/Duplicati/UnitTest/Issue5987.cs b/Duplicati/UnitTest/Issue5987.cs new file mode 100644 index 000000000..63073a292 --- /dev/null +++ b/Duplicati/UnitTest/Issue5987.cs @@ -0,0 +1,65 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System.IO; +using System.Linq; +using NUnit.Framework; + +namespace Duplicati.UnitTest +{ + public class Issue5987 : BasicSetupHelper + { + [Test] + [Category("Targeted")] + [TestCase(true)] + [TestCase(false)] + public void TestRestoreWithMissingDblocks(bool deleteIndexFiles) + { + var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); + + // 1. Make a backup of a single file + File.WriteAllText(Path.Combine(DATAFOLDER, "a"), "abc"); + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete the dblock file + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dblockFiles) + File.Delete(file); + + if (deleteIndexFiles) + { + var dindexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dindexFiles) + File.Delete(file); + } + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + var res = c.Repair(); + Assert.That(res.Errors.Count(), Is.EqualTo(0), "Repair should not have errors"); + TestUtils.AssertResults(c.Test()); + } + } + } +} + From d22ce5370ec95d11c86454c702efb4aa772faf96 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 8 May 2025 10:31:38 +0200 Subject: [PATCH 03/15] Rewrote the repair for scenarios where remote files are missing. This adds a number of additional features to the repair process so it can actually fully recover if all data is present. Also added a several tests to ensure that the functionality works in multiple scenarios. This fixes #5987 --- .../Library/Main/Database/LocalDatabase.cs | 15 +- .../Main/Database/LocalRepairDatabase.cs | 470 +++++++++---- .../Library/Main/Operation/CompactHandler.cs | 13 +- .../Main/Operation/PurgeFilesHandler.cs | 2 +- .../Library/Main/Operation/RepairHandler.cs | 617 ++++++++++++------ .../Main/Volumes/FilesetVolumeWriter.cs | 6 +- .../UnitTest/RepairWithMissingRemoteFiles.cs | 382 +++++++++++ Duplicati/UnitTest/TestUtils.cs | 20 +- 8 files changed, 1213 insertions(+), 312 deletions(-) create mode 100644 Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index c66b5d2a0..ac3795f4f 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -661,9 +661,9 @@ AND Fileset.ID NOT IN return res; } - public bool IsFilesetFullBackup(DateTime filesetTime) + public bool IsFilesetFullBackup(DateTime filesetTime, IDbTransaction? transaction) { - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(transaction)) using (var rd = cmd.SetCommandAndParameters($@"SELECT ""IsFullBackup"" FROM ""Fileset"" WHERE ""Timestamp"" = @Timestamp").SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(filesetTime)).ExecuteReader()) { if (!rd.Read()) @@ -1504,8 +1504,19 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER } } + /// + /// Adds a link between an index volume and a block volume. + /// + /// The ID of the index volume. + /// The ID of the block volume. + /// An optional transaction. public void AddIndexBlockLink(long indexVolumeID, long blockVolumeID, IDbTransaction transaction) { + if (indexVolumeID <= 0) + throw new ArgumentOutOfRangeException(nameof(indexVolumeID), "Index volume ID must be greater than 0."); + if (blockVolumeID <= 0) + throw new ArgumentOutOfRangeException(nameof(blockVolumeID), "Block volume ID must be greater than 0."); + m_insertIndexBlockLink.SetParameterValue("@IndexVolumeId", indexVolumeID) .SetParameterValue("@BlockVolumeId", blockVolumeID) .ExecuteNonQuery(transaction); diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index f898a1ab3..0d669ccb9 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -35,27 +35,75 @@ namespace Duplicati.Library.Main.Database /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalRepairDatabase)); + /// + /// Creates a new local repair database + /// + /// The path to the database + /// The page cache size public LocalRepairDatabase(string path, long pagecachesize) : base(path, "Repair", true, pagecachesize) { } - public long GetFilesetIdFromRemotename(string filelist) + /// + /// Gets the fileset ID from the remote name + /// + /// The remote name of the fileset + /// An optional transaction + /// >The fileset ID + public (long FilesetId, DateTime Time, bool IsFullBackup) GetFilesetFromRemotename(string filelist, IDbTransaction? transaction) { - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(transaction)) { - var filesetid = cmd.SetCommandAndParameters(@"SELECT ""Fileset"".""ID"" FROM ""Fileset"",""RemoteVolume"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""RemoteVolume"".""Name"" = @Name") + var rd = cmd.SetCommandAndParameters(@"SELECT ""Fileset"".""ID"", ""Fileset"".""Timestamp"", ""Fileset"".""IsFullBackup"" FROM ""Fileset"",""RemoteVolume"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""RemoteVolume"".""Name"" = @Name") .SetParameterValue("@Name", filelist) - .ExecuteScalarInt64(-1); + .ExecuteReader(); - if (filesetid == -1) + if (!rd.Read()) throw new Exception($"No such remote file: {filelist}"); - return filesetid; + return (rd.ConvertValueToInt64(0, -1), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime(), rd.GetInt32(0) == BackupType.FULL_BACKUP); } } + /// + /// Moves entries in the FilesetEntry table from previous fileset to current fileset + /// + /// Current fileset ID + /// Source fileset ID + /// The transaction + public void MoveFilesFromFileset(long filesetid, long prevFilesetId, IDbTransaction transaction) + { + if (filesetid <= 0) + throw new ArgumentException("filesetid must be > 0"); + + if (prevFilesetId <= 0) + throw new ArgumentException("prevId must be > 0"); + + using var cmd = m_connection.CreateCommand(transaction); + cmd.SetCommandAndParameters(@"UPDATE ""FilesetEntry"" SET ""FilesetID"" = @CurrentFilesetId WHERE ""FilesetID"" = @PreviousFilesetId ") + .SetParameterValue("@CurrentFilesetId", filesetid) + .SetParameterValue("@PreviousFilesetId", prevFilesetId) + .ExecuteNonQuery(); + + } + + /// + /// Gets the list of index files that reference a given block file + /// + /// The block file ID + /// >A list of index file IDs + public IEnumerable GetIndexFilesReferencingBlockFile(long blockfileid, IDbTransaction? transaction) + { + using var cmd = m_connection.CreateCommand(transaction, @"SELECT ""RemoteVolume"".""Name"" FROM ""RemoteVolume"", ""IndexBlockLink"" WHERE ""IndexBlockLink"".""BlockVolumeID"" = @BlockFileId AND ""RemoteVolume"".""ID"" = ""IndexBlockLink"".""IndexVolumeID"" AND ""RemoteVolume"".""Type"" = @Type") + .SetParameterValue("@BlockFileId", blockfileid) + .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()); + + foreach (var rd in cmd.ExecuteReaderEnumerable()) + yield return rd.ConvertValueToString(0) ?? throw new Exception("RemoteVolume name was null"); + } + /// /// Gets a list of filesets that are missing files /// @@ -88,63 +136,6 @@ namespace Duplicati.Library.Main.Database .ExecuteNonQuery(); } - public interface IBlockSource - { - string File { get; } - long Offset { get; } - } - - public interface IBlockWithSources : IBlock - { - IEnumerable Sources { get; } - } - - private class BlockWithSources : Block, IBlockWithSources - { - private class BlockSource : IBlockSource - { - public string File { get; private set; } - public long Offset { get; private set; } - - public BlockSource(string file, long offset) - { - this.File = file; - this.Offset = offset; - - } - } - - private readonly IDataReader m_rd; - public bool Done { get; private set; } - - public BlockWithSources(IDataReader rd) - : base(rd.ConvertValueToString(0) ?? throw new Exception("Hash value was null"), rd.ConvertValueToInt64(1)) - { - m_rd = rd; - Done = !m_rd.Read(); - } - - public IEnumerable Sources - { - get - { - if (Done) - yield break; - - var cur = new BlockSource(m_rd.ConvertValueToString(2) ?? "", m_rd.ConvertValueToInt64(3)); - var file = cur.File; - - while (!Done && cur.File == file) - { - yield return cur; - Done = m_rd.Read(); - if (!Done) - cur = new BlockSource(m_rd.ConvertValueToString(2) ?? "", m_rd.ConvertValueToInt64(3)); - } - } - } - } - private class RemoteVolume : IRemoteVolume { public string Name { get; private set; } @@ -159,39 +150,160 @@ namespace Duplicati.Library.Main.Database } } - public IEnumerable GetBlockVolumesFromIndexName(string name) + public IEnumerable GetBlockVolumesFromIndexName(string indexName) { using var cmd = m_connection.CreateCommand(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"" WHERE ""IndexVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = @Name)) ") - .SetParameterValue("@Name", name); + .SetParameterValue("@Name", indexName); foreach (var rd in cmd.ExecuteReaderEnumerable()) yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); } - public interface IMissingBlockList : IDisposable + public IEnumerable GetBlockVolumesFromBlockName(string blockName) { - bool SetBlockRestored(string hash, long size); - IEnumerable GetSourceFilesWithBlocks(long blocksize); - IEnumerable> GetMissingBlocks(); - IEnumerable GetFilesetsUsingMissingBlocks(); - IEnumerable GetMissingBlockSources(); + using var cmd = m_connection.CreateCommand(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"" WHERE ""IndexVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = @Name)) ") + .SetParameterValue("@Name", blockName); + + foreach (var rd in cmd.ExecuteReaderEnumerable()) + yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); } + /// + /// A single block with source data for repair + /// + /// The block hash + /// The block size + /// The file that contains the block + /// The offset of the block in the file + public sealed record BlockWithSourceData(string Hash, long Size, string File, long Offset); + + /// + /// A single block with metadata source data for repair + /// + /// The block hash + /// The block size + /// The path of the file or directory that contains the block + public sealed record BlockWithMetadataSourceData(string Hash, long Size, string Path); + + /// + /// A single blocklist hash entry + /// + /// The blockset id + /// The hash of the blocklist entry (when done) + /// The total length of the blockset + /// The index of the blocklist hash + /// The index of the block in the blockset + /// The hash of the entry + public sealed record BlocklistHashesEntry( + long BlocksetId, + string BlocklistHash, + long BlocklistHashLength, + long BlocklistHashIndex, + int Index, + string Hash); + + /// + /// Helper interface for the missing block list + /// + public interface IMissingBlockList : IDisposable + { + /// + /// Registers a block as restored + /// + /// The block hash + /// The block size + /// The restored blocks + bool SetBlockRestored(string hash, long size); + /// + /// Gets the list of files that contains missing blocks + /// + /// The blocksize setting + /// A list of files with missing blocks + IEnumerable GetSourceFilesWithBlocks(long blocksize); + /// + /// Gets a list for filesystem entries that contain missing blocks in metadata + /// + /// A list of filesystem entries with missing blocks + IEnumerable GetSourceItemsWithMetadataBlocks(); + /// + /// Gets missing blocklist hashes + /// + /// The number of hashes for each block + /// >A list of blocklist hashes + IEnumerable GetBlocklistHashes(long hashesPerBlock); + /// + /// Gets the number of missing blocks + /// + /// >The number of missing blocks + long GetMissingBlockCount(); + /// + /// Gets all the filesets that are affected by missing blocks + /// + /// >A list of filesets + IEnumerable GetFilesetsUsingMissingBlocks(); + /// + /// Gets a list of remote files that may contain missing blocks + /// + /// >A list of remote files + IEnumerable GetMissingBlockSources(); + /// + /// Moves restored blocks to a new volume + /// + /// The volume that contains the blocks + /// The volume that used to contain the blocks + /// The transaction to use + /// >The number of moved blocks + long MoveBlocksToNewVolume(long targetVolumeId, long sourceVolumeId, IDbTransaction? transaction); + } + + /// + /// Implementation of the missing block list + /// private class MissingBlockList : IMissingBlockList { + /// + /// The connection to the database + /// private readonly IDbConnection m_connection; - private readonly TemporaryTransactionWrapper m_transaction; - private IDbCommand m_insertCommand; - private string m_tablename; + /// + /// The transaction to use + /// + private readonly ReusableTransaction m_transaction; + /// + /// The insert command to use for restoring blocks + /// + private readonly IDbCommand m_insertCommand; + /// + /// The name of the temporary table + /// + private readonly string m_tablename; + /// + /// The name of the volume where blocks are missing + /// private readonly string m_volumename; + /// + /// The query to use for getting missing blocks + /// + private readonly string m_missingBlocksQuery; - public MissingBlockList(string volumename, IDbConnection connection, IDbTransaction? transaction) + /// + /// Whether the object has been disposed + /// + private bool m_isDisposed = false; + + /// + /// Creates a new missing block list + /// + /// The name of the volume with missing blocks + /// The connection to the database + /// The transaction to use + public MissingBlockList(string volumename, IDbConnection connection, ReusableTransaction transaction) { m_connection = connection; - m_transaction = new TemporaryTransactionWrapper(m_connection, transaction); + m_transaction = transaction; m_volumename = volumename; var tablename = "MissingBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using (var cmd = m_connection.CreateCommand(m_transaction.Parent)) + using (var cmd = m_connection.CreateCommand(m_transaction.Transaction)) { cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" INTEGER NOT NULL) ")); m_tablename = tablename; @@ -206,38 +318,166 @@ namespace Duplicati.Library.Main.Database cmd.ExecuteNonQuery(FormatInvariant($@"CREATE UNIQUE INDEX ""{tablename}-Ix"" ON ""{tablename}"" (""Hash"", ""Size"", ""Restored"")")); } - m_insertCommand = m_connection.CreateCommand(m_transaction.Parent, FormatInvariant($@"UPDATE ""{tablename}"" SET ""Restored"" = @NewRestoredValue WHERE ""Hash"" = @Hash AND ""Size"" = @Size AND ""Restored"" = @PreviousRestoredValue ")); + m_insertCommand = m_connection.CreateCommand(FormatInvariant($@"UPDATE ""{tablename}"" SET ""Restored"" = @NewRestoredValue WHERE ""Hash"" = @Hash AND ""Size"" = @Size AND ""Restored"" = @PreviousRestoredValue ")); + m_missingBlocksQuery = FormatInvariant($@"SELECT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Restored"" = @Restored "); } + /// public bool SetBlockRestored(string hash, long size) { - return m_insertCommand.SetParameterValue("@NewRestoredValue", 1) + return m_insertCommand.SetTransaction(m_transaction.Transaction).SetParameterValue("@NewRestoredValue", 1) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .SetParameterValue(@"PreviousRestoredValue", 0) + .SetParameterValue("@PreviousRestoredValue", 0) .ExecuteNonQuery() == 1; } - public IEnumerable GetSourceFilesWithBlocks(long blocksize) + /// + public IEnumerable GetSourceFilesWithBlocks(long blocksize) { - using var cmd = m_connection.CreateCommand(m_transaction.Parent, FormatInvariant($@"SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {blocksize} FROM ""{m_tablename}"", ""Block"", ""BlocksetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" AND ""{m_tablename}"".""Restored"" = @Restored ")) - .SetParameterValue("@Restored", 0); - using (var rd = cmd.ExecuteReader()) - if (rd.Read()) - { - var bs = new BlockWithSources(rd); - while (!bs.Done) - yield return bs; - } - } - - public IEnumerable> GetMissingBlocks() - { - using var cmd = m_connection.CreateCommand(m_transaction.Parent, FormatInvariant($@"SELECT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Restored"" = @Restored ")) + using var cmd = m_connection.CreateCommand(m_transaction.Transaction, FormatInvariant($@"SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {blocksize} FROM ""{m_tablename}"", ""Block"", ""BlocksetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" AND ""{m_tablename}"".""Restored"" = @Restored ORDER BY ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {blocksize}")) .SetParameterValue("@Restored", 0); foreach (var rd in cmd.ExecuteReaderEnumerable()) - yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); + { + var hash = rd.ConvertValueToString(0) ?? throw new Exception("Hash value was null"); + var size = rd.ConvertValueToInt64(1); + var file = rd.ConvertValueToString(2) ?? throw new Exception("File value was null"); + var offset = rd.ConvertValueToInt64(3); + + yield return new BlockWithSourceData(hash, size, file, offset); + } + } + + /// + public IEnumerable GetSourceItemsWithMetadataBlocks() + { + using var cmd = m_connection.CreateCommand(m_transaction.Transaction, FormatInvariant($@"SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"" FROM ""{m_tablename}"", ""Block"", ""BlocksetEntry"", ""Metadataset"", ""File"" WHERE ""File"".""MetadataID"" == ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" AND ""{m_tablename}"".""Restored"" = @Restored ORDER BY ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path""")) + .SetParameterValue("@Restored", 0); + + foreach (var rd in cmd.ExecuteReaderEnumerable()) + { + var hash = rd.ConvertValueToString(0) ?? throw new Exception("Hash value was null"); + var size = rd.ConvertValueToInt64(1); + var path = rd.ConvertValueToString(2) ?? throw new Exception("File value was null"); + + yield return new BlockWithMetadataSourceData(hash, size, path); + } + } + + /// + public IEnumerable GetBlocklistHashes(long hashesPerBlock) + { + using var cmd = m_connection.CreateCommand(m_transaction.Transaction, FormatInvariant($@" + SELECT + b.""Hash"", + bs.""Id"", + bs.""Length"", + bse.""Index"" / @HashesPerBlock AS ""BlocklistHashIndex"", + ( + SELECT blh.""Hash"" + FROM ""BlocklistHash"" blh + WHERE blh.""BlocksetID"" = bs.""ID"" + AND blh.""Index"" == bse.""Index"" / @HashesPerBlock + LIMIT 1 + ) AS ""BlocklistHashHash"", + bse.""Index"" + FROM + ""BlocksetEntry"" bse + JOIN ""Block"" b ON b.""ID"" = bse.""BlockID"" + JOIN ""Blockset"" bs ON bs.""ID"" = bse.""BlocksetID"" + WHERE + EXISTS ( + SELECT 1 + FROM ""BlocklistHash"" blh + JOIN ""{m_tablename}"" mt ON mt.""Hash"" = blh.""Hash"" + WHERE blh.""BlocksetID"" = bs.""ID"" + AND mt.""Restored"" = @Restored + ) + ORDER BY + bse.""BlocksetID"", bse.""Index"" + ")) + .SetParameterValue("@HashesPerBlock", hashesPerBlock) + .SetParameterValue("@Restored", 0); + + foreach (var rd in cmd.ExecuteReaderEnumerable()) + { + var hash = rd.ConvertValueToString(0) ?? throw new Exception("Block.Hash is null"); + var blocksetId = rd.ConvertValueToInt64(1); + var length = rd.ConvertValueToInt64(2); + var blocklistHashIndex = rd.ConvertValueToInt64(3); + var blocklistHash = rd.ConvertValueToString(4) ?? throw new Exception("BlocklistHash is null"); + var index = rd.ConvertValueToInt64(5); + + yield return new BlocklistHashesEntry(blocksetId, blocklistHash, length, blocklistHashIndex, (int)index, hash); + } + } + + /// + public long GetMissingBlockCount() + { + using var cmd = m_connection.CreateCommand(m_transaction.Transaction, FormatInvariant($@"SELECT COUNT(*) FROM ({m_missingBlocksQuery})")) + .SetParameterValue("@Restored", 0); + + return cmd.ExecuteScalarInt64(0); + } + + /// + /// Gets the list of missing blocks + /// + /// >A list of missing blocks + public IEnumerable<(string Hash, long Size)> GetMissingBlocks() + { + using var cmd = m_connection.CreateCommand(m_transaction.Transaction, m_missingBlocksQuery) + .SetParameterValue("@Restored", 0); + + foreach (var rd in cmd.ExecuteReaderEnumerable()) + yield return (rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); + } + + /// + public long MoveBlocksToNewVolume(long targetVolumeId, long sourceVolumeId, IDbTransaction? transaction = null) + { + if (targetVolumeId <= 0) + throw new ArgumentOutOfRangeException(nameof(targetVolumeId), "Target volume ID must be greater than 0"); + + // Move the source blocks into the DuplicateBlock table + using var cmd = m_connection.CreateCommand(transaction, FormatInvariant($@" + INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") + SELECT b.""ID"", b.""VolumeID"" + FROM ""Block"" b + WHERE b.""VolumeID"" = @SourceVolumeId + AND (b.""Hash"", b.""Size"") IN ( + SELECT ""Hash"", ""Size"" + FROM ""{m_tablename}"" + WHERE ""Restored"" = @Restored + ) + ")) + .SetParameterValue("@SourceVolumeId", sourceVolumeId) + .SetParameterValue("@Restored", 1); + + var moved = cmd.ExecuteNonQuery(); + + // Then update the blocks table to point to the new volume + cmd.SetCommandAndParameters(FormatInvariant($@" + UPDATE ""Block"" + SET ""VolumeID"" = @TargetVolumeId + WHERE ""VolumeID"" = @SourceVolumeId + AND (""Hash"", ""Size"") IN ( + SELECT ""Hash"", ""Size"" + FROM ""{m_tablename}"" + WHERE ""Restored"" = @Restored + ) + ")) + .SetParameterValue("@TargetVolumeId", targetVolumeId) + .SetParameterValue("@SourceVolumeId", sourceVolumeId) + .SetParameterValue("@Restored", 1); + + var updated = cmd.ExecuteNonQuery(); + if (updated != moved) + throw new Exception($"Unexpected number of updated blocks: {updated} != {moved}"); + + return updated; } public IEnumerable GetFilesetsUsingMissingBlocks() @@ -246,7 +486,7 @@ namespace Duplicati.Library.Main.Database var blocklists = FormatInvariant($@"SELECT DISTINCT ""FileLookup"".""ID"" AS ID FROM ""{m_tablename}"", ""Block"", ""Blockset"", ""BlocklistHash"", ""FileLookup"" WHERE ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""BlocklistHash"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" "); var cmdtxt = FormatInvariant($@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""FilesetEntry"", ""Fileset"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""RemoteVolume"".""Type"" = @Type AND ""FilesetEntry"".""FileID"" IN (SELECT DISTINCT ""ID"" FROM ( {blocks} UNION {blocklists} ))"); - using var cmd = m_connection.CreateCommand(m_transaction.Parent, cmdtxt) + using var cmd = m_connection.CreateCommand(m_transaction.Transaction, cmdtxt) .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); foreach (var rd in cmd.ExecuteReaderEnumerable()) @@ -255,7 +495,8 @@ namespace Duplicati.Library.Main.Database public IEnumerable GetMissingBlockSources() { - using var cmd = m_connection.CreateCommand(m_transaction.Parent, FormatInvariant($@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""Block"", ""{m_tablename}"" WHERE ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""Block"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Remotevolume"".""Name"" != @Name ")) + using var cmd = m_connection.CreateCommand(m_transaction.Transaction, FormatInvariant($@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""Block"", ""{m_tablename}"" WHERE ""{m_tablename}"".""Restored"" = @Restored AND ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""Block"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Remotevolume"".""Name"" != @Name ")) + .SetParameterValue("@Restored", 0) .SetParameterValue("@Name", m_volumename); foreach (var rd in cmd.ExecuteReaderEnumerable()) @@ -264,25 +505,24 @@ namespace Duplicati.Library.Main.Database public void Dispose() { - if (m_tablename != null) - { - try - { - using (var cmd = m_connection.CreateCommand(m_transaction.Parent)) - cmd.SetTransaction(m_transaction.Parent).ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tablename}"" ")); - } - catch { } - finally { m_tablename = null!; } - } + if (m_isDisposed) + return; - if (m_insertCommand != null) - try { m_insertCommand.Dispose(); } - catch { } - finally { m_insertCommand = null!; } + m_isDisposed = true; + try + { + if (m_tablename != null) + using (var cmd = m_connection.CreateCommand(m_transaction.Transaction)) + cmd.SetTransaction(m_transaction.Transaction).ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tablename}"" ")); + } + catch { } + + try { m_insertCommand?.Dispose(); } + catch { } } } - public IMissingBlockList CreateBlockList(string volumename, IDbTransaction? transaction = null) + public IMissingBlockList CreateBlockList(string volumename, ReusableTransaction transaction) { return new MissingBlockList(volumename, m_connection, transaction); } @@ -621,9 +861,9 @@ namespace Duplicati.Library.Main.Database } } - public void CheckBlocklistCorrect(string hash, long length, IEnumerable blocklist, long blocksize, long blockhashlength) + public void CheckBlocklistCorrect(string hash, long length, IEnumerable blocklist, long blocksize, long blockhashlength, IDbTransaction? transaction) { - using (var cmd = m_connection.CreateCommand()) + using (var cmd = m_connection.CreateCommand(transaction)) { var query = FormatInvariant($@" SELECT diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index c4af01e17..95c4f88e8 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -325,16 +325,13 @@ namespace Duplicati.Library.Main.Operation private async Task FinishVolumeAndUpload(LocalDeleteDatabase db, IBackendManager backendManager, BlockVolumeWriter newvol, IndexVolumeWriter newvolindex, List> uploadedVolumes, ReusableTransaction rtr) { - Action indexVolumeFinished = () => - { - if (newvolindex != null && m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) + Action indexVolumeFinished = null; + if (newvolindex != null && m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) + indexVolumeFinished = () => { foreach (var blocklist in db.GetBlocklists(newvol.VolumeID, m_options.Blocksize, m_options.BlockhashSize)) - { newvolindex.WriteBlocklist(blocklist.Item1, blocklist.Item2, 0, blocklist.Item3); - } - } - }; + }; uploadedVolumes.Add(new KeyValuePair(newvol.RemoteFilename, newvol.Filesize)); if (newvolindex != null) @@ -342,7 +339,7 @@ namespace Duplicati.Library.Main.Operation // TODO: The upload here does not flush the database messages, // and this can leave the database in a state where it does not know of the remote file - // To fix it, we need thread-safe access to the daabase and transaction + // To fix it, we need thread-safe access to the database and transaction // Once fixed, we can perhaps let he backend manager simply call the database directly if (!m_options.Dryrun) { diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index 13b766d56..edea73564 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -166,7 +166,7 @@ namespace Duplicati.Library.Main.Operation using (var tf = new Library.Utility.TempFile()) using (var vol = new Volumes.FilesetVolumeWriter(m_options, ts)) { - var isOriginalFilesetFullBackup = db.IsFilesetFullBackup(tsOriginal); + var isOriginalFilesetFullBackup = db.IsFilesetFullBackup(tsOriginal, tr); var newids = tempset.ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup); vol.VolumeID = newids.Item1; vol.CreateFilesetFile(isOriginalFilesetFullBackup); diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 3e095ca93..61deca2ad 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -26,6 +26,7 @@ using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; +using Duplicati.Library.Main.Operation.Backup; using Duplicati.Library.Main.Volumes; using Duplicati.Library.Utility; @@ -126,6 +127,7 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(0); using (var db = new LocalRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache)) + using (var rtr = new ReusableTransaction(db)) { Utility.UpdateOptionsFromDb(db, m_options); Utility.VerifyOptionsAndUpdateDatabase(db, m_options); @@ -137,14 +139,15 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsInRepairState"); // Ensure the database is consistent before we start fixing the remote - db.VerifyConsistencyForRepair(m_options.Blocksize, m_options.BlockhashSize, true, null); + db.VerifyConsistencyForRepair(m_options.Blocksize, m_options.BlockhashSize, true, rtr.Transaction); // If the last backup failed, guard the incomplete fileset, so we can create a synthetic filelist - var lastTempFilelist = db.GetLastIncompleteFilesetVolume(null); - var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, null, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false); + var lastTempFilelist = db.GetLastIncompleteFilesetVolume(rtr.Transaction); + var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, rtr.Transaction, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false); + rtr.Commit("CommitRemoteListAnalysisTransaction"); var buffer = new byte[m_options.Blocksize]; - var hashsize = HashFactory.HashSizeBytes(m_options.BlockHashAlgorithm); + var hashsize = m_options.BlockhashSize; var missingRemoteFilesets = db.MissingRemoteFilesets().ToList(); var missingLocalFilesets = db.MissingLocalFilesets().ToList(); @@ -185,21 +188,22 @@ namespace Duplicati.Library.Main.Operation if (tp.VerificationRequiredVolumes.Any()) { using (var testdb = new LocalTestDatabase(db)) - using (var rtr = new ReusableTransaction(testdb)) { foreach (var n in tp.VerificationRequiredVolumes) try { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(testdb, null, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(testdb, rtr.Transaction, cancellationToken).ConfigureAwait(false); + if (!m_options.Dryrun) + rtr.Commit("CommitEarlyExit", false); return; } progress++; m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); - KeyValuePair>> res; + KeyValuePair>> res; (var tf, var hash, var size) = await backendManager.GetWithInfoAsync(n.Name, n.Hash, n.Size, cancellationToken).ConfigureAwait(false); using (tf) res = TestHandler.TestVolumeInternals(testdb, rtr, n, tf, m_options, 1); @@ -221,7 +225,7 @@ namespace Duplicati.Library.Main.Operation throw; } - rtr.Commit("CommitVerificationTransaction", false); + rtr.Commit("CommitVerificationTransaction"); } } @@ -231,7 +235,9 @@ namespace Duplicati.Library.Main.Operation { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + if (!m_options.Dryrun) + rtr.Commit("CommitEarlyExit", false); return; } @@ -265,28 +271,26 @@ namespace Duplicati.Library.Main.Operation var blocksize = m_options.Blocksize; foreach (var ixb in ifr.BlockLists) - db.CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize); + db.CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize, rtr.Transaction); // Register the new index file and link it to the block files - using (var tr = db.BeginTransaction()) + var selfid = db.RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, RemoteVolumeState.Uploading, size, new TimeSpan(0), rtr.Transaction); + foreach (var rv in ifr.Volumes) { - var selfid = db.RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, RemoteVolumeState.Uploading, size, new TimeSpan(0), tr); - foreach (var rv in ifr.Volumes) - { - // Guard against unknown block files - long id = db.GetRemoteVolumeID(rv.Filename, tr); - if (id == -1) - Logging.Log.WriteWarningMessage(LOGTAG, "UnknownBlockFile", null, "Index file {0} references unknown block file: {1}", n.File.Name, rv.Filename); - else - db.AddIndexBlockLink(selfid, id, tr); - } - tr.Commit(); + // Guard against unknown block files + long id = db.GetRemoteVolumeID(rv.Filename, rtr.Transaction); + if (id == -1) + Logging.Log.WriteWarningMessage(LOGTAG, "UnknownBlockFile", null, "Index file {0} references unknown block file: {1}", n.File.Name, rv.Filename); + else + db.AddIndexBlockLink(selfid, id, rtr.Transaction); } + if (!m_options.Dryrun) + rtr.Commit("CommitIndexFileTransaction"); } // All checks fine, we accept the new index file Logging.Log.WriteInformationMessage(LOGTAG, "AcceptNewIndexFile", "Accepting new index file {0}", n.File.Name); - db.UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash); + db.UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash, rtr.Transaction); continue; } catch (Exception rex) @@ -322,13 +326,15 @@ namespace Duplicati.Library.Main.Operation { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + if (!m_options.Dryrun) + rtr.Commit("CommitEarlyExit", false); return; } progress++; m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); - var fileTime = FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, timestamp); + var fileTime = FilesetVolumeWriter.ProbeUnusedFilenameName(db, rtr.Transaction, m_options, timestamp); var fsw = new FilesetVolumeWriter(m_options, fileTime); Logging.Log.WriteInformationMessage(LOGTAG, "ReuploadingFileset", "Re-uploading fileset {0} from {1} as remote volume registration is missing, new filename: {2}", filesetId, timestamp, fsw.RemoteFilename); @@ -341,21 +347,25 @@ namespace Duplicati.Library.Main.Operation db.WriteFileset(fsw, filesetId, null); fsw.Close(); + + fsw.VolumeID = db.RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, -1, RemoteVolumeState.Temporary); + db.LinkFilesetToVolume(filesetId, fsw.VolumeID, null); if (m_options.Dryrun) { fsw.Dispose(); Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would re-upload fileset {0}", fsw.RemoteFilename); continue; } - - fsw.VolumeID = db.RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, -1, RemoteVolumeState.Temporary); - db.LinkFilesetToVolume(filesetId, fsw.VolumeID, null); - // TODO: Rewrite to use transactions and flush db messages as needed await backendManager.PutAsync(fsw, null, null, false, null, cancellationToken).ConfigureAwait(false); } if (anyDlistUploads) - await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); + { + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + if (!m_options.Dryrun) + rtr.Commit("CommitFilesetTransaction"); + } foreach (var volumename in missingLocalFilesets) { @@ -365,15 +375,15 @@ namespace Duplicati.Library.Main.Operation var parsed = VolumeBase.ParseFilename(remoteVolume.Name); using (var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read)) using (var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions)) - using (var transaction = db.BeginTransaction()) using (var recreatedb = new LocalRecreateDatabase(db, m_options)) { if (compressor == null) throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); - var filesetid = db.CreateFileset(remoteVolume.ID, parsed.Time, transaction); - RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, transaction, compressor, filesetid, m_options, new FilterExpression()); - transaction.Commit(); + var filesetid = db.CreateFileset(remoteVolume.ID, parsed.Time, rtr.Transaction); + RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, rtr.Transaction, compressor, filesetid, m_options, new FilterExpression()); + if (!m_options.Dryrun) + rtr.Commit("CommitRecreateFilesetTransaction"); } } } @@ -381,7 +391,15 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun && tp.MissingVolumes.Any()) db.TerminatedWithActiveUploads = true; - foreach (var n in tp.MissingVolumes) + foreach (var n in tp.MissingVolumes.OrderBy(x => x.Type switch + { + // Make sure we process the content first, then the fileset + // process index last, as it may have been deleted during block processing + RemoteVolumeType.Blocks => 0, + RemoteVolumeType.Files => 1, + RemoteVolumeType.Index => 2, + _ => 3 + })) { IDisposable newEntry = null; @@ -398,170 +416,37 @@ namespace Duplicati.Library.Main.Operation if (n.Type == RemoteVolumeType.Files) { - var filesetId = db.GetFilesetIdFromRemotename(n.Name); + var timestamp = VolumeBase.ParseFilename(n.Name).Time; + var fileTime = FilesetVolumeWriter.ProbeUnusedFilenameName(db, rtr.Transaction, m_options, timestamp); + var volumeWriter = new FilesetVolumeWriter(m_options, fileTime); - // We cannot wrap the FilesetVolumeWriter in a using statement here because a reference to it is - // retained in newEntry. - FilesetVolumeWriter volumeWriter = new FilesetVolumeWriter(m_options, DateTime.UtcNow); newEntry = volumeWriter; - volumeWriter.SetRemoteFilename(n.Name); - - db.WriteFileset(volumeWriter, filesetId, null); - DateTime filesetTime = db.FilesetTimes.First(x => x.Key == filesetId).Value; - volumeWriter.CreateFilesetFile(db.IsFilesetFullBackup(filesetTime)); - - volumeWriter.Close(); - if (m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would re-upload fileset {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(volumeWriter.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)); - else - { - db.UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null); - // TODO: Rewrite to use transactions and flush db messages as needed - await backendManager.PutAsync(volumeWriter, null, null, false, null, cancellationToken).ConfigureAwait(false); - } + await RunRepairDlist(backendManager, db, rtr, volumeWriter, n, fileTime, cancellationToken).ConfigureAwait(false); } else if (n.Type == RemoteVolumeType.Index) { - IndexVolumeWriter w = new IndexVolumeWriter(m_options); + var w = new IndexVolumeWriter(m_options); newEntry = w; - w.SetRemoteFilename(n.Name); - using (var h = HashFactory.CreateHasher(m_options.BlockHashAlgorithm)) - { + // Check if the index file has already been deleted, beause the dblock was recreated + var currentState = db.GetRemoteVolume(n.Name, rtr.Transaction).State; + if (currentState == RemoteVolumeState.Deleted || currentState == RemoteVolumeState.Temporary || currentState == RemoteVolumeState.Deleting) + continue; - foreach (var blockvolume in db.GetBlockVolumesFromIndexName(n.Name)) - { - w.StartVolume(blockvolume.Name); - var volumeid = db.GetRemoteVolumeID(blockvolume.Name); - - foreach (var b in db.GetBlocks(volumeid)) - w.AddBlock(b.Hash, b.Size); - - w.FinishVolume(blockvolume.Hash, blockvolume.Size); - - if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) - foreach (var b in db.GetBlocklists(volumeid, m_options.Blocksize, hashsize)) - { - var bh = Convert.ToBase64String(h.ComputeHash(b.Item2, 0, b.Item3)); - if (bh != b.Item1) - throw new Exception(string.Format("Internal consistency check failed, generated index block has wrong hash, {0} vs {1}", bh, b.Item1)); - - w.WriteBlocklist(b.Item1, b.Item2, 0, b.Item3); - } - } - } - - w.Close(); - - if (m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadIndexFile", "would re-upload index file {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)); - else - { - db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null); - // TODO: Rewrite to use transactions and flush db messages as needed - await backendManager.PutAsync(w, null, null, false, null, cancellationToken).ConfigureAwait(false); - } + await RunRepairDindex(backendManager, db, rtr, w, n, cancellationToken).ConfigureAwait(false); } else if (n.Type == RemoteVolumeType.Blocks) { - BlockVolumeWriter w = new BlockVolumeWriter(m_options); + var w = new BlockVolumeWriter(m_options); newEntry = w; - w.SetRemoteFilename(n.Name); - - using (var mbl = db.CreateBlockList(n.Name)) - { - //First we grab all known blocks from local files - foreach (var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize)) - { - var hash = block.Hash; - var size = (int)block.Size; - - foreach (var source in block.Sources) - { - var file = source.File; - var offset = source.Offset; - - try - { - if (System.IO.File.Exists(file)) - using (var f = System.IO.File.OpenRead(file)) - { - f.Position = offset; - if (size == Library.Utility.Utility.ForceStreamRead(f, buffer, size)) - { - using (var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm)) - { - var newhash = Convert.ToBase64String(blockhasher.ComputeHash(buffer, 0, size)); - if (newhash == hash) - { - if (mbl.SetBlockRestored(hash, size)) - w.AddBlock(hash, buffer, 0, size, Duplicati.Library.Interface.CompressionHint.Default); - break; - } - } - } - } - } - catch (Exception ex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", ex, "Failed to access file: {0}", file); - } - } - } - - //Then we grab all remote volumes that have the missing blocks - await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(mbl.GetMissingBlockSources().ToList(), cancellationToken).ConfigureAwait(false)) - { - try - { - using (tmpfile) - using (var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options)) - foreach (var b in f.Blocks) - if (mbl.SetBlockRestored(b.Key, b.Value)) - if (f.ReadBlock(b.Key, buffer) == b.Value) - w.AddBlock(b.Key, buffer, 0, (int)b.Value, CompressionHint.Default); - } - catch (Exception ex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileAccessError", ex, "Failed to access remote file: {0}", name); - } - } - - // If we managed to recover all blocks, NICE! - var missingBlocks = mbl.GetMissingBlocks().Count(); - if (missingBlocks > 0) - { - Logging.Log.WriteInformationMessage(LOGTAG, "RepairMissingBlocks", "Repair cannot acquire {0} required blocks for volume {1}, which are required by the following filesets: ", missingBlocks, n.Name); - foreach (var f in mbl.GetFilesetsUsingMissingBlocks()) - Logging.Log.WriteInformationMessage(LOGTAG, "AffectedFilesetName", f.Name); - - var recoverymsg = string.Format("If you want to continue working with the database, you can use the \"{0}\" and \"{1}\" commands to purge the missing data from the database and the remote storage.", "list-broken-files", "purge-broken-files"); - var logmsg = string.Format("Repair not possible, missing {0} blocks.\n" + recoverymsg, missingBlocks); - - Logging.Log.WriteInformationMessage(LOGTAG, "RecoverySuggestion", null, logmsg); - if (!m_options.Dryrun) - throw new UserInformationException(logmsg, "RepairIsNotPossible"); - } - else - { - if (m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadBlockFile", "would re-upload block file {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)); - else - { - db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null); - // TODO: Rewrite to use transactions and flush db messages as needed - await backendManager.PutAsync(w, null, null, false, null, cancellationToken).ConfigureAwait(false); - } - } - } + await RunRepairDblock(backendManager, db, rtr, w, n, cancellationToken).ConfigureAwait(false); } } catch (Exception ex) { if (newEntry != null) - try { newEntry.Dispose(); } + try { newEntry?.Dispose(); } catch { } - finally { newEntry = null; } Logging.Log.WriteErrorMessage(LOGTAG, "CleanupMissingFileError", ex, "Failed to perform cleanup for missing file: {0}, message: {1}", n.Name, ex.Message); @@ -595,6 +480,7 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "DeletingEmptyIndexFile", "Deleting empty index file {0}", emptyIndexFile.Name); await backendManager.DeleteAsync(emptyIndexFile.Name, emptyIndexFile.Size, false, cancellationToken).ConfigureAwait(false); + await backendManager.FlushPendingMessagesAsync(db, null, cancellationToken).ConfigureAwait(false); } } } @@ -602,7 +488,7 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteErrorMessage(LOGTAG, "CleanupEmptyIndexFileError", ex, "Failed to perform cleanup for empty index file: {0}, message: {1}", emptyIndexFile.Name, ex.Message); - if (ex is System.Threading.ThreadAbortException) + if (ex.IsAbortException()) throw; } } @@ -615,7 +501,374 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(1); await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) + { db.TerminatedWithActiveUploads = false; + rtr.Commit("CommitRepairTransaction", false); + } + } + } + + private async Task RunRepairDlist(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, FilesetVolumeWriter volumeWriter, RemoteVolumeEntry n, DateTime filesetTime, CancellationToken cancellationToken) + { + volumeWriter.VolumeID = db.RegisterRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, rtr.Transaction); + (var prevFilesetId, var _, var isPrevFull) = db.GetFilesetFromRemotename(n.Name, rtr.Transaction); + + if (!string.IsNullOrEmpty(m_options.ControlFiles)) + foreach (var p in m_options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) + volumeWriter.AddControlFile(p, m_options.GetCompressionHintFromFilename(p)); + + volumeWriter.CreateFilesetFile(isPrevFull); + var newFilesetID = db.CreateFileset(volumeWriter.VolumeID, filesetTime, rtr.Transaction); + db.LinkFilesetToVolume(newFilesetID, volumeWriter.VolumeID, rtr.Transaction); + db.MoveFilesFromFileset(newFilesetID, prevFilesetId, rtr.Transaction); + + db.WriteFileset(volumeWriter, newFilesetID, rtr.Transaction); + volumeWriter.Close(); + + if (m_options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would re-upload fileset {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(volumeWriter.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)); + else + { + db.UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null, rtr.Transaction); + db.UpdateRemoteVolume(n.Name, RemoteVolumeState.Deleting, n.Size, n.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); + await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + if (!m_options.Dryrun) + rtr.Commit("CommitPriorToFilesetUpload"); + await backendManager.PutAsync(volumeWriter, null, null, false, null, cancellationToken).ConfigureAwait(false); + } + + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + if (!m_options.Dryrun) + rtr.Commit("CommitFilesetTransaction"); + } + + private async Task RunRepairDindex(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, IndexVolumeWriter w, RemoteVolumeEntry n, CancellationToken cancellationToken) + { + w.VolumeID = db.RegisterRemoteVolume(w.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Uploading, -1, TimeSpan.Zero, rtr.Transaction); + + var blockvolumeids = new List(); + using var h = HashFactory.CreateHasher(m_options.BlockHashAlgorithm); + foreach (var blockvolume in db.GetBlockVolumesFromIndexName(n.Name)) + { + w.StartVolume(blockvolume.Name); + var volumeid = db.GetRemoteVolumeID(blockvolume.Name, rtr.Transaction); + + foreach (var b in db.GetBlocks(volumeid)) + w.AddBlock(b.Hash, b.Size); + + w.FinishVolume(blockvolume.Hash, blockvolume.Size); + blockvolumeids.Add(volumeid); + + if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) + foreach (var b in db.GetBlocklists(volumeid, m_options.Blocksize, m_options.BlockhashSize)) + { + var bh = Convert.ToBase64String(h.ComputeHash(b.Item2, 0, b.Item3)); + if (bh != b.Item1) + throw new Exception(string.Format("Internal consistency check failed, generated index block has wrong hash, {0} vs {1}", bh, b.Item1)); + + w.WriteBlocklist(b.Item1, b.Item2, 0, b.Item3); + } + } + + foreach (var blockvolumeid in blockvolumeids) + db.AddIndexBlockLink(w.VolumeID, blockvolumeid, rtr.Transaction); + + w.Close(); + + if (m_options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadIndexFile", "would re-upload index file {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)); + else + { + db.UpdateRemoteVolume(n.Name, RemoteVolumeState.Deleting, n.Size, n.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); + await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backendManager.PutAsync(w, null, null, false, null, cancellationToken).ConfigureAwait(false); + } + + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + if (!m_options.Dryrun) + rtr.Commit("CommitRepairTransaction"); + } + + private async Task RunRepairDblock(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, BlockVolumeWriter w, RemoteVolumeEntry n, CancellationToken cancellationToken) + { + // TODO: Keep an open volume and append to that until we reach the size threshold, + // similar to how the creation works. For now we just attempt to recreate the original volume + + // The dblock files are the most complex to recreate + // as data can be either file contents, metadata or blocklist hashes + // We attempt to recover all three source parts in the steps below + using (var mbl = db.CreateBlockList(n.Name, rtr)) + { + var originalMissingBlockCount = mbl.GetMissingBlockCount(); + + // First we grab all known blocks from local files + string lastRestoredHash = null; + long lastRestoredSize = -1; + foreach (var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize)) + { + if (block.Hash == lastRestoredHash && block.Size == lastRestoredSize) + continue; + + var size = (int)block.Size; + + try + { + if (File.Exists(block.File)) + { + using (var f = File.OpenRead(block.File)) + { + f.Position = block.Offset; + + var buffer = new byte[m_options.Blocksize]; + if (size == Library.Utility.Utility.ForceStreamRead(f, buffer, size)) + { + using (var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm)) + { + var newhash = Convert.ToBase64String(blockhasher.ComputeHash(buffer, 0, size)); + if (newhash == block.Hash) + { + // Found it, no need to look again + lastRestoredHash = block.Hash; + lastRestoredSize = block.Size; + if (mbl.SetBlockRestored(block.Hash, block.Size)) + w.AddBlock(block.Hash, buffer, 0, size, CompressionHint.Default); + } + else + { + Logging.Log.WriteVerboseMessage(LOGTAG, "FileHashMismatch", null, "Block hash mismatch on {0}, expected {1} but got {2}", block.File, block.Hash, newhash); + } + } + } + else + { + Logging.Log.WriteVerboseMessage(LOGTAG, "FileLengthMismatch", null, "Block length mismatch on {0}, expected {1} but got {2}", block.File, size, f.Length); + } + } + } + else + { + Logging.Log.WriteVerboseMessage(LOGTAG, "FileNotFound", null, "File not found: {0}", block.File); + } + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", ex, "Failed to access file: {0}", block.File); + } + } + + // Then grab all blocks with metadata + foreach (var block in mbl.GetSourceItemsWithMetadataBlocks()) + { + if (block.Hash == lastRestoredHash && block.Size == lastRestoredSize) + continue; + + var size = (int)block.Size; + + try + { + var isFile = File.Exists(block.Path); + var isDir = Directory.Exists(block.Path); + if (isFile || isDir) + { + using var snapshot = Snapshots.SnapshotUtility.CreateNoSnapshot([block.Path], true, true); + var entry = snapshot.GetFilesystemEntry(block.Path, isDir); + if (entry == null) + { + Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", null, "Entry not found: {0}", block.Path); + continue; + } + + var metadata = MetadataGenerator.GenerateMetadata(entry, entry.Attributes, m_options); + var metahash = Utility.WrapMetadata(metadata, m_options); + if (metahash.FileHash == block.Hash && metahash.Blob.Length == size) + { + // Found it, no need to look again + lastRestoredHash = block.Hash; + lastRestoredSize = block.Size; + if (mbl.SetBlockRestored(block.Hash, block.Size)) + w.AddBlock(block.Hash, metahash.Blob, 0, metahash.Blob.Length, CompressionHint.Default); + } + else + { + Logging.Log.WriteVerboseMessage(LOGTAG, "MetadataHashMismatch", null, "Metadata block hash mismatch on {0}, expected {1} but got {2}", block.Path, block.Hash, metahash.FileHash); + } + } + else + { + Logging.Log.WriteVerboseMessage(LOGTAG, "EntryNotFound", null, "Entry not found: {0}", block.Path); + } + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", ex, "Failed to access file: {0}", block.Path); + } + } + + // Then restore any blocklists that are missing + using var blocklistHash = new MemoryStream(); + LocalRepairDatabase.BlocklistHashesEntry lastBlocklist = null; + var blockhashsize = m_options.BlockhashSize; + var hashesPerBlock = m_options.Blocksize / m_options.BlockhashSize; + + void EmitBlockListBlock() + { + blocklistHash.Position = 0; + using var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm); + var resulthash = Convert.ToBase64String(blockhasher.ComputeHash(blocklistHash)); + var totalHashes = (lastBlocklist.BlocklistHashLength + m_options.Blocksize - 1) / m_options.Blocksize; + var hashesInLastBlock = totalHashes % hashesPerBlock; + var targetsize = lastBlocklist.Index >= totalHashes - 1 + ? hashesInLastBlock * blockhashsize + : hashesPerBlock * blockhashsize; + + if (resulthash == lastBlocklist.BlocklistHash && targetsize == blocklistHash.Length) + { + if (mbl.SetBlockRestored(resulthash, blocklistHash.Length)) + w.AddBlock(resulthash, blocklistHash.ToArray(), 0, (int)blocklistHash.Length, CompressionHint.Default); + } + else + { + Logging.Log.WriteWarningMessage(LOGTAG, "BlocklistHashMismatch", null, "Blocklist hash mismatch on {0} / {1}, expected {2} but got {3}", lastBlocklist.BlocklistHash, lastBlocklist.Index, lastBlocklist.Hash, resulthash); + } + + blocklistHash.SetLength(0); + } + + foreach (var blocklist in mbl.GetBlocklistHashes(hashesPerBlock)) + { + lastBlocklist ??= blocklist; + + if (lastBlocklist.BlocksetId != blocklist.BlocksetId || lastBlocklist.BlocklistHashIndex != blocklist.BlocklistHashIndex) + EmitBlockListBlock(); + + var data = Convert.FromBase64String(blocklist.Hash); + blocklistHash.Write(data, 0, data.Length); + lastBlocklist = blocklist; + } + + // Handle any trailing blocklist hash + if (blocklistHash.Length > 0) + EmitBlockListBlock(); + + //Then we grab all remote volumes that have the missing blocks + await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(mbl.GetMissingBlockSources().ToList(), cancellationToken).ConfigureAwait(false)) + { + try + { + var buffer = new byte[m_options.Blocksize]; + using (tmpfile) + using (var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options)) + foreach (var b in f.Blocks) + if (mbl.SetBlockRestored(b.Key, b.Value)) + if (f.ReadBlock(b.Key, buffer) == b.Value) + w.AddBlock(b.Key, buffer, 0, (int)b.Value, CompressionHint.Default); + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileAccessError", ex, "Failed to access remote file: {0}", name); + } + } + + // If we managed to recover all blocks, NICE! + var missingBlocks = mbl.GetMissingBlockCount(); + if (missingBlocks > 0) + { + // TODO: If we managed to get ANY blocks recovered, we can register a new volume + // and move existing blocks to duplicate, so a later purge has less negative impact + + Logging.Log.WriteInformationMessage(LOGTAG, "RepairMissingBlocks", "Repair cannot acquire {0} required blocks for volume {1}, which are required by the following filesets: ", missingBlocks, n.Name); + foreach (var f in mbl.GetFilesetsUsingMissingBlocks()) + Logging.Log.WriteInformationMessage(LOGTAG, "AffectedFilesetName", f.Name); + + var recoverymsg = string.Format("If you want to continue working with the database, you can use the \"{0}\" and \"{1}\" commands to purge the missing data from the database and the remote storage.", "list-broken-files", "purge-broken-files"); + var logmsg = string.Format("Repair not possible, missing {0} blocks.\n" + recoverymsg, missingBlocks); + + Logging.Log.WriteInformationMessage(LOGTAG, "RecoverySuggestion", null, logmsg); + if (!m_options.Dryrun) + throw new UserInformationException(logmsg, "RepairIsNotPossible"); + } + else + { + w.VolumeID = db.RegisterRemoteVolume(w.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Uploading, -1, TimeSpan.Zero, rtr.Transaction); + + var fixedBlockCount = mbl.MoveBlocksToNewVolume(w.VolumeID, n.ID, rtr.Transaction); + if (fixedBlockCount != originalMissingBlockCount) + throw new UserInformationException(string.Format("Failed to move {0} blocks to new volume {1}, only moved {2}", originalMissingBlockCount, w.RemoteFilename, fixedBlockCount), "FailedToMoveBlocks"); + + // Create a new index file that points to the new volume + IndexVolumeWriter newvolindex = null; + Action indexVolumeFinished = null; + if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None) + { + newvolindex = new IndexVolumeWriter(m_options); + newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, rtr.Transaction); + newvolindex.StartVolume(w.RemoteFilename); + foreach (var b in db.GetBlocks(w.VolumeID)) + newvolindex.AddBlock(b.Hash, b.Size); + + db.AddIndexBlockLink(newvolindex.VolumeID, w.VolumeID, rtr.Transaction); + if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) + indexVolumeFinished = () => + { + foreach (var blocklist in db.GetBlocklists(w.VolumeID, m_options.Blocksize, m_options.BlockhashSize)) + newvolindex.WriteBlocklist(blocklist.Item1, blocklist.Item2, 0, blocklist.Item3); + }; + } + + var oldIndexFiles = db.GetIndexFilesReferencingBlockFile(n.ID, rtr.Transaction).ToList(); + var toDelete = new List(); + // Find all index files that point to the old volume + foreach (var oldIndexFile in oldIndexFiles) + { + var oldVolume = db.GetRemoteVolume(oldIndexFile, rtr.Transaction); + if (oldVolume.State == RemoteVolumeState.Uploading || oldVolume.State == RemoteVolumeState.Uploaded || oldVolume.State == RemoteVolumeState.Verified) + { + var blockVolumesReferenced = db.GetBlockVolumesFromIndexName(oldIndexFile).ToList(); + if (blockVolumesReferenced.Any(x => x.Name != n.Name)) + Logging.Log.WriteVerboseMessage(LOGTAG, "IndexFileNotDeleted", null, "Index file {0} references multiple remote volumes, skipping", oldVolume.Name); + else + toDelete.Add(oldVolume); + } + else + { + Logging.Log.WriteVerboseMessage(LOGTAG, "IndexFileNotDeleted", null, "Index file {0} is not in a state to be deleted, skipping", oldVolume.Name); + } + } + + // All information is in place, we can now upload the new volume + await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + if (m_options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReplaceBlockFile", "would replace block file {0} with {1}", n.Name, w.RemoteFilename); + else + { + rtr.Commit("PostRepairPreUploadBlockVolume"); + await backendManager.PutAsync(w, newvolindex, indexVolumeFinished, false, null, cancellationToken).ConfigureAwait(false); + } + + // Flush the database as we have a new volume uploaded + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + + // Prepare for deleting the old stuff + db.UpdateRemoteVolume(n.Name, RemoteVolumeState.Deleting, n.Size, n.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); + foreach (var vol in toDelete) + db.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Deleting, vol.Size, vol.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); + + // Persist desired state prior to deleting the old files + if (!m_options.Dryrun) + rtr.Commit("PostRepairBlockVolume"); + + // Delete the old files + foreach (var vol in toDelete) + if (m_options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteIndexFile", "would delete index file {0}", vol.Name); + else + await backendManager.DeleteAsync(vol.Name, vol.Size, false, cancellationToken).ConfigureAwait(false); + } + + // All done, the new dblock is now in place + await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); + if (!m_options.Dryrun) + rtr.Commit("PostRepairBlockVolume"); } } diff --git a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs index 4bf79d88e..79f384494 100644 --- a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs +++ b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs @@ -27,6 +27,7 @@ using Duplicati.Library.Main.Database; using Duplicati.Library.Main.Operation.Common; using System.Threading.Tasks; using Duplicati.Library.Utility; +using System.Data; namespace Duplicati.Library.Main.Volumes { @@ -244,12 +245,13 @@ namespace Duplicati.Library.Main.Volumes /// /// The database to check for clashes /// The options to use for the filename + /// The transaction to use /// The starting time to probe from /// The time to increment by each probe /// The maximum number of tries to probe /// The first unused filename - internal static DateTime ProbeUnusedFilenameName(LocalDatabase database, Options options, DateTime start, TimeSpan increment = default, int maxTries = 60) - => ProbeUnusedFilenameName((name) => Task.FromResult(database.GetRemoteVolumeID(name)), options, start, increment, maxTries).Await(); + internal static DateTime ProbeUnusedFilenameName(LocalDatabase database, IDbTransaction transaction, Options options, DateTime start, TimeSpan increment = default, int maxTries = 60) + => ProbeUnusedFilenameName((name) => Task.FromResult(database.GetRemoteVolumeID(name, transaction)), options, start, increment, maxTries).Await(); /// /// Probes for an unused filename, using the current time as a starting point diff --git a/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs b/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs new file mode 100644 index 000000000..b3d5e65a2 --- /dev/null +++ b/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs @@ -0,0 +1,382 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Duplicati.Library.Main.Database; +using Duplicati.Library.SQLiteHelper; +using NUnit.Framework; + +namespace Duplicati.UnitTest +{ + public class RepairWithMissingRemoteFiles : BasicSetupHelper + { + [Test] + [Category("Targeted")] + [TestCase(false, true, false)] + [TestCase(false, false, false)] + [TestCase(false, true, true)] + [TestCase(false, false, true)] + [TestCase(true, true, false)] + [TestCase(true, false, false)] + [TestCase(true, true, true)] + [TestCase(true, false, true)] + public void TestRepairWithMissingDblocks(bool deleteDblockFiles, bool deleteIndexFiles, bool deleteDlistFiles) + { + var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true, number_of_retries = 0 }); + + // 1. Make a backup of a single file (1 block + 3 bytes for a block with less data) + var data = new byte[1024 + 3]; + data.AsSpan().Fill((byte)'a'); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete files + if (deleteDblockFiles) + { + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dblockFiles) + File.Delete(file); + } + + if (deleteIndexFiles) + { + var dindexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dindexFiles) + File.Delete(file); + } + + if (deleteDlistFiles) + { + var dlistFiles = Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dlistFiles) + File.Delete(file); + } + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Repair()); + + // 4. Test that the database is now valid + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Test()); + } + + [Test] + [Category("Targeted")] + [TestCase(false, true, false)] + [TestCase(false, false, false)] + [TestCase(false, true, true)] + [TestCase(false, false, true)] + [TestCase(true, true, false)] + [TestCase(true, false, false)] + [TestCase(true, true, true)] + [TestCase(true, false, true)] + public void TestRepairWithMissingDblocksAndMultipleSources(bool deleteDblockFiles, bool deleteIndexFiles, bool deleteDlistFiles) + { + var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); + + // 1. Make some shared data + var data = new byte[1024 + 3]; + data.AsSpan().Fill((byte)'a'); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "b"), data.AsSpan().Slice(0, 1024).ToArray()); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "c"), data.AsSpan().Slice(0, 1024).ToArray()); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "d"), data.AsSpan().Slice(1024, 3).ToArray()); + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete files + if (deleteDblockFiles) + { + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dblockFiles) + File.Delete(file); + } + + if (deleteIndexFiles) + { + var dindexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dindexFiles) + File.Delete(file); + } + + if (deleteDlistFiles) + { + var dlistFiles = Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dlistFiles) + File.Delete(file); + } + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + TestUtils.AssertResults(c.Repair()); + TestUtils.AssertResults(c.Test()); + } + } + + [Test] + [Category("Targeted")] + [TestCase(false, true, false)] + [TestCase(false, false, false)] + [TestCase(false, true, true)] + [TestCase(false, false, true)] + [TestCase(true, true, false)] + [TestCase(true, false, false)] + [TestCase(true, true, true)] + [TestCase(true, false, true)] + public void TestRepairWithDataFillingMultipleBlocklists(bool deleteDblockFiles, bool deleteIndexFiles, bool deleteDlistFiles) + { + var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); + + // 1. Make some shared data 1kb blocks, 32 bytes hash, 3 blocklists, and 5 bytes extra + var data = new byte[1024 * 32 * 3 + 5]; + data.AsSpan().Fill((byte)'a'); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "b"), data.AsSpan().Slice(1024, 3).ToArray()); + Random.Shared.NextBytes(data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "c"), data); + Random.Shared.NextBytes(data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "d"), data.AsSpan().Slice(0, 1028).ToArray()); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "e"), data.AsSpan().Slice(4098, 67).ToArray()); + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete files + if (deleteDblockFiles) + { + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dblockFiles) + File.Delete(file); + } + + if (deleteIndexFiles) + { + var dindexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dindexFiles) + File.Delete(file); + } + + if (deleteDlistFiles) + { + var dlistFiles = Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dlistFiles) + File.Delete(file); + } + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + TestUtils.AssertResults(c.Repair()); + TestUtils.AssertResults(c.Test()); + } + } + + [Test] + [Category("Targeted")] + [TestCase(false, true, false)] + [TestCase(false, false, false)] + [TestCase(false, true, true)] + [TestCase(false, false, true)] + [TestCase(true, true, false)] + [TestCase(true, false, false)] + [TestCase(true, true, true)] + [TestCase(true, false, true)] + public void TestRepairWithSmallFileOnly(bool deleteDblockFiles, bool deleteIndexFiles, bool deleteDlistFiles) + { + var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); + + // 1. Make some shared data + var data = new byte[3]; + data.AsSpan().Fill((byte)'a'); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete files + if (deleteDblockFiles) + { + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dblockFiles) + File.Delete(file); + } + + if (deleteIndexFiles) + { + var dindexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dindexFiles) + File.Delete(file); + } + + if (deleteDlistFiles) + { + var dlistFiles = Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dlistFiles) + File.Delete(file); + } + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + TestUtils.AssertResults(c.Repair()); + TestUtils.AssertResults(c.Test()); + } + } + + [Test] + [Category("Targeted")] + [TestCase(false, true, false)] + [TestCase(false, false, false)] + [TestCase(false, true, true)] + [TestCase(false, false, true)] + [TestCase(true, true, false)] + [TestCase(true, false, false)] + [TestCase(true, true, true)] + [TestCase(true, false, true)] + public void TestRepairWithDataFillingMultipleBlocklistsRandom(bool deleteDblockFiles, bool deleteIndexFiles, bool deleteDlistFiles) + { + var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); + + // 1. Make some shared data 1kb blocks, 32 bytes hash, 3 blocklists, and 1028 bytes extra + var data = new byte[1024 * 32 * 3 + 1028]; + Random.Shared.NextBytes(data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete files + if (deleteDblockFiles) + { + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dblockFiles) + File.Delete(file); + } + + if (deleteIndexFiles) + { + var dindexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dindexFiles) + File.Delete(file); + } + + if (deleteDlistFiles) + { + var dlistFiles = Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dlistFiles) + File.Delete(file); + } + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + TestUtils.AssertResults(c.Repair()); + TestUtils.AssertResults(c.Test()); + } + } + + [Test] + [Category("Targeted")] + [TestCase(false, true, false)] + [TestCase(false, false, false)] + [TestCase(false, true, true)] + [TestCase(false, false, true)] + [TestCase(true, true, false)] + [TestCase(true, false, false)] + [TestCase(true, true, true)] + [TestCase(true, false, true)] + public void TestRepairWithDryRun(bool deleteDblockFiles, bool deleteIndexFiles, bool deleteDlistFiles) + { + var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); + + // 1. Make some random data + var data = new byte[1028]; + Random.Shared.NextBytes(data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete files + if (deleteDblockFiles) + { + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dblockFiles) + File.Delete(file); + } + + if (deleteIndexFiles) + { + var dindexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dindexFiles) + File.Delete(file); + } + + if (deleteDlistFiles) + { + var dlistFiles = Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dlistFiles) + File.Delete(file); + } + + (long remoteVolumes, long duplicatedBlocks, List blockVolumeIds, List<(long, long)> indexVolumeLinks) GetDbSignature() + { + using var db = SQLiteLoader.LoadConnection(DBFILE, 0); + using var cmd = db.CreateCommand(); + + var remoteVolumes = cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM RemoteVolume"); + var duplicatedBlocks = cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM DuplicateBlock"); + var blockVolumeIds = cmd.ExecuteReaderEnumerable("SELECT VolumeID FROM Block").Select(x => x.ConvertValueToInt64(0)).ToList(); + var indexVolumeLinks = cmd.ExecuteReaderEnumerable("SELECT IndexVolumeID, BlockVolumeID FROM IndexBlockLink").Select(x => (x.ConvertValueToInt64(0), x.ConvertValueToInt64(1))).ToList(); + return (remoteVolumes, duplicatedBlocks, blockVolumeIds, indexVolumeLinks); + } + + var preSignature = GetDbSignature(); + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { dry_run = true }), null)) + TestUtils.AssertResults(c.Repair()); + + var postSignature = GetDbSignature(); + + Assert.AreEqual(preSignature.remoteVolumes, postSignature.remoteVolumes, "Remote volumes count should be the same after dry run repair."); + Assert.AreEqual(preSignature.duplicatedBlocks, postSignature.duplicatedBlocks, "Duplicated blocks count should be the same after dry run repair."); + Assert.AreEqual(preSignature.blockVolumeIds.Count, postSignature.blockVolumeIds.Count, "Block volume IDs count should be the same after dry run repair."); + Assert.AreEqual(preSignature.indexVolumeLinks.Count, postSignature.indexVolumeLinks.Count, "Index volume links count should be the same after dry run repair."); + + Assert.IsTrue(preSignature.blockVolumeIds.All(x => postSignature.blockVolumeIds.Contains(x)), "All block volume IDs should be present after dry run repair."); + Assert.IsTrue(preSignature.indexVolumeLinks.All(x => postSignature.indexVolumeLinks.Contains(x)), "All index volume links should be present after dry run repair."); + } + + // TODO: Add more tests to cover the following scenarios: + + // Test with deleted files as sources + run purge-broken-files + // Test with partial data available + run purge-broken-files + // Test where a single block in a large file is missing + run purge-broken-files + // Test that fetching remote sources for blocks is working + + } +} + diff --git a/Duplicati/UnitTest/TestUtils.cs b/Duplicati/UnitTest/TestUtils.cs index 6243a31d6..1f58d7a80 100644 --- a/Duplicati/UnitTest/TestUtils.cs +++ b/Duplicati/UnitTest/TestUtils.cs @@ -296,8 +296,6 @@ namespace Duplicati.UnitTest { operation = ((Library.Main.OperationMode)operationProperty.GetValue(results)).ToString(); } - Assert.AreEqual(0, results.Errors.Count(), "{0} errors:\n{1}", operation, string.Join("\n", results.Errors)); - Assert.AreEqual(0, results.Warnings.Count(), "{0} warnings:\n{1}", operation, string.Join("\n", results.Warnings)); if (results is ITestResults testResults) { @@ -322,6 +320,24 @@ namespace Duplicati.UnitTest throw new TestVerificationException(sb.ToString()); } } + + if (results.Errors.Count() != 0) + { + var sb = new StringBuilder(); + sb.AppendLine($"Errors - {operation}:"); + foreach (var e in results.Errors) + sb.AppendLine(e.ToString()); + throw new TestVerificationException(sb.ToString()); + } + + if (results.Warnings.Count() != 0) + { + var sb = new StringBuilder(); + sb.AppendLine($"Warnings - {operation}:"); + foreach (var w in results.Warnings) + sb.AppendLine(w.ToString()); + throw new TestVerificationException(sb.ToString()); + } } public static string DumpTable(IDbCommand cmd, string table, string where = null) From fe866ceb9fefad461ddb9102782a415fb49d83e5 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 9 May 2025 11:12:29 +0200 Subject: [PATCH 04/15] Implemented support for repairing the database when only parts of the source data is available. This repair will leave the database in a still-broken state, but if this is run prior to purge-broken-files, less data will be lost. Also updated the purge-broken-files operation to keep files and directories in the set, if they are only missing metadata. --- .../Library/Interface/ResultInterfaces.cs | 1 + .../Database/LocalListBrokenFilesDatabase.cs | 89 +++ .../Main/Database/LocalPurgeDatabase.cs | 26 +- .../Main/Database/LocalRepairDatabase.cs | 155 ++-- .../Main/Operation/ListBrokenFilesHandler.cs | 5 + .../Main/Operation/PurgeBrokenFilesHandler.cs | 16 + .../Main/Operation/PurgeFilesHandler.cs | 42 +- .../Library/Main/Operation/RepairHandler.cs | 737 ++++++++++++------ Duplicati/Library/Main/Options.cs | 18 + Duplicati/Library/Main/ResultClasses.cs | 1 + Duplicati/Library/Main/Strings.cs | 4 + .../UnitTest/RepairWithMissingRemoteFiles.cs | 322 +++++++- 12 files changed, 1023 insertions(+), 393 deletions(-) diff --git a/Duplicati/Library/Interface/ResultInterfaces.cs b/Duplicati/Library/Interface/ResultInterfaces.cs index 3c8c8c1a5..a0d304312 100644 --- a/Duplicati/Library/Interface/ResultInterfaces.cs +++ b/Duplicati/Library/Interface/ResultInterfaces.cs @@ -494,6 +494,7 @@ namespace Duplicati.Library.Interface { long RemovedFileCount { get; } long RemovedFileSize { get; } + long UpdatedFileCount { get; } long RewrittenFileLists { get; } ICompactResults CompactResults { get; } } diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index 678c8e988..b30a5fc17 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -102,6 +102,30 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN yield return new Tuple(rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), rd.ConvertValueToInt64(1)); } + /// + /// Returns all index files that are orphaned, i.e., not referenced by any block files. + /// + /// Transaction to use for the query. + /// >All index files that are orphaned. + public IEnumerable GetOrphanedIndexFiles(IDbTransaction transaction) + { + using var cmd = Connection.CreateCommand(transaction, FormatInvariant($@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Index.ToString()}' AND ""ID"" NOT IN (SELECT ""IndexVolumeID"" FROM ""IndexBlockLink"")")); + + foreach (var rd in cmd.ExecuteReaderEnumerable()) + yield return new RemoteVolume( + rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), + rd.ConvertValueToString(1) ?? throw new Exception("Hash was null"), + rd.ConvertValueToInt64(2, -1) + ); + } + + /// + /// Inserts the broken file IDs into the given table. The table must have a single column with the same name as the ID field name. + /// + /// The filset id for the current operation + /// The name of the table to insert into + /// The name of the ID field in the table + /// The transaction to use for the query public void InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname, IDbTransaction transaction) { using var cmd = Connection.CreateCommand(transaction, INSERT_BROKEN_IDS(tablename, IDfieldname)) @@ -109,6 +133,71 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN cmd.ExecuteNonQuery(); } + /// + /// Returns the ID of an empty metadata blockset. If no empty blockset is found, it returns the ID of the smallest blockset that is not in the given block volume IDs. + /// If no such blockset is found, it returns -1. + /// + /// The volume ids to ignore when searching for a suitable metadata block + /// The hash of the empty blockset + /// The size of the empty blockset + /// The transaction to use for the query + /// The ID of the empty metadata blockset, or -1 if no suitable blockset is found + public long GetEmptyMetadataBlocksetId(IEnumerable blockVolumeIds, string emptyHash, long emptyHashSize, IDbTransaction transaction) + { + using var cmd = Connection.CreateCommand(transaction, @"SELECT ""ID"" FROM ""Blockset"" WHERE ""FullHash"" = @EmptyHash AND ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))") + .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) + .SetParameterValue("@EmptyHash", emptyHash) + .SetParameterValue("@EmptyHashSize", emptyHashSize); + + var res = cmd.ExecuteScalarInt64(-1); + + // No empty block found, try to find a zero-length block instead + if (res < 0 && emptyHashSize != 0) + res = cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))") + .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) + .SetParameterValue("@EmptyHashSize", 0) + .ExecuteScalarInt64(-1); + + // No empty block found, pick the smallest one + if (res < 0) + res = cmd.SetCommandAndParameters(@"SELECT ""Blockset"".""ID"" FROM ""BlocksetEntry"", ""Blockset"", ""Metadataset"", ""Block"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) ORDER BY ""Blockset"".""Length"" ASC LIMIT 1") + .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) + .ExecuteScalarInt64(-1); + + return res; + } + + /// + /// Replaces the metadata blockset ID in the Metadataset table with the empty blockset ID for all fileset entries that are not in any block volume. + /// This is used to clean up the metadata blocksets that are now missing. + /// + /// The filesetId to target + /// The empty blockset ID to replace with + /// The transaction to use for the query + /// The number of rows affected + public int ReplaceMetadata(long filesetId, long emptyBlocksetId, IDbTransaction transaction) + { + using var cmd = m_connection.CreateCommand(transaction, @" +UPDATE ""Metadataset"" +SET ""BlocksetID"" = @EmptyBlocksetID +WHERE + ""ID"" IN ( + SELECT ""FileLookup"".""MetadataID"" + FROM ""FileLookup"", ""FilesetEntry"" + WHERE + ""FilesetEntry"".""FilesetId"" = @FilesetId + AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" + ) + AND ""BlocksetID"" NOT IN ( + SELECT ""BlocksetID"" + FROM ""BlocksetEntry"", ""Block"" + WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + )") + .SetParameterValue("@EmptyBlocksetId", emptyBlocksetId) + .SetParameterValue("@FilesetId", filesetId); + return cmd.ExecuteNonQuery(); + } + public void RemoveMissingBlocks(IEnumerable names, IDbTransaction transaction) { if (names == null || !names.Any()) return; diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index d80e7d590..2bdb15209 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -72,9 +72,10 @@ namespace Duplicati.Library.Main.Database long ParentID { get; } long RemovedFileCount { get; } long RemovedFileSize { get; } + long UpdatedFileCount { get; } void ApplyFilter(Library.Utility.IFilter filter); - void ApplyFilter(Action filtercommand); + void ApplyFilter(Func filtercommand); Tuple ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup); IEnumerable> ListAllDeletedFiles(); } @@ -89,6 +90,7 @@ namespace Duplicati.Library.Main.Database public long ParentID { get; private set; } public long RemovedFileCount { get; private set; } public long RemovedFileSize { get; private set; } + public long UpdatedFileCount { get; private set; } public TemporaryFileset(long parentid, LocalPurgeDatabase parentdb, IDbConnection connection, IDbTransaction transaction) { @@ -102,12 +104,13 @@ namespace Duplicati.Library.Main.Database cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tablename}"" (""FileID"" INTEGER PRIMARY KEY) ")); } - public void ApplyFilter(Action filtercommand) + public void ApplyFilter(Func filtercommand) { + int updated; using (var cmd = m_connection.CreateCommand(m_transaction)) - filtercommand(cmd, ParentID, m_tablename); + updated = filtercommand(cmd, ParentID, m_tablename); - PostFilterChecks(); + PostFilterChecks(updated); } public void ApplyFilter(Library.Utility.IFilter filter) @@ -156,15 +159,24 @@ namespace Duplicati.Library.Main.Database } } - PostFilterChecks(); + PostFilterChecks(0); } - private void PostFilterChecks() + private void PostFilterChecks(int updated) { using (var cmd = m_connection.CreateCommand(m_transaction)) { + UpdatedFileCount = updated; RemovedFileCount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""{m_tablename}"""), 0); - RemovedFileSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""C"".""Length"") FROM ""{m_tablename}"" A, ""FileLookup"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" "), 0); + RemovedFileSize = cmd.ExecuteScalarInt64(FormatInvariant($@" + SELECT + SUM(""C"".""Length"") + FROM + ""{m_tablename}"" A, ""FileLookup"" B, ""Blockset"" C, ""Metadataset"" D + WHERE + ""A"".""FileID"" = ""B"".""ID"" + AND(""B"".""BlocksetID"" = ""C"".""ID"" OR(""B"".""MetadataID"" = ""D"".""ID"" AND ""D"".""BlocksetID"" = ""C"".""ID"")) + "), 0); var filesetcount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = {ParentID}"), 0); if (filesetcount == RemovedFileCount) throw new Interface.UserInformationException($"Refusing to purge {RemovedFileCount} files from fileset with ID {ParentID}, as that would remove the entire fileset.\nTo delete a fileset, use the \"delete\" command.", "PurgeWouldRemoveEntireFileset"); diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index 0d669ccb9..916cb2457 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -159,15 +159,6 @@ namespace Duplicati.Library.Main.Database yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); } - public IEnumerable GetBlockVolumesFromBlockName(string blockName) - { - using var cmd = m_connection.CreateCommand(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"" WHERE ""IndexVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = @Name)) ") - .SetParameterValue("@Name", blockName); - - foreach (var rd in cmd.ExecuteReaderEnumerable()) - yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); - } - /// /// A single block with source data for repair /// @@ -212,8 +203,9 @@ namespace Duplicati.Library.Main.Database /// /// The block hash /// The block size + /// The volume ID of the new target volume /// The restored blocks - bool SetBlockRestored(string hash, long size); + bool SetBlockRestored(string hash, long size, long volumeId); /// /// Gets the list of files that contains missing blocks /// @@ -246,14 +238,6 @@ namespace Duplicati.Library.Main.Database /// /// >A list of remote files IEnumerable GetMissingBlockSources(); - /// - /// Moves restored blocks to a new volume - /// - /// The volume that contains the blocks - /// The volume that used to contain the blocks - /// The transaction to use - /// >The number of moved blocks - long MoveBlocksToNewVolume(long targetVolumeId, long sourceVolumeId, IDbTransaction? transaction); } /// @@ -274,6 +258,14 @@ namespace Duplicati.Library.Main.Database /// private readonly IDbCommand m_insertCommand; /// + /// The command to copy blocks into the duplicate block table + /// + private readonly IDbCommand m_copyIntoDuplicatedBlocks; + /// + /// The command to assign blocks to a new volume + /// + private readonly IDbCommand m_assignBlocksToNewVolume; + /// /// The name of the temporary table /// private readonly string m_tablename; @@ -319,17 +311,38 @@ namespace Duplicati.Library.Main.Database } m_insertCommand = m_connection.CreateCommand(FormatInvariant($@"UPDATE ""{tablename}"" SET ""Restored"" = @NewRestoredValue WHERE ""Hash"" = @Hash AND ""Size"" = @Size AND ""Restored"" = @PreviousRestoredValue ")); + m_copyIntoDuplicatedBlocks = m_connection.CreateCommand(@"INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") SELECT ""Block"".""ID"", ""Block"".""VolumeID"" FROM ""Block"" WHERE ""Block"".""Hash"" = @Hash AND ""Block"".""Size"" = @Size"); + m_assignBlocksToNewVolume = m_connection.CreateCommand(@"UPDATE ""Block"" SET ""VolumeID"" = @TargetVolumeId WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); m_missingBlocksQuery = FormatInvariant($@"SELECT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Restored"" = @Restored "); } /// - public bool SetBlockRestored(string hash, long size) + public bool SetBlockRestored(string hash, long size, long targetVolumeId) { - return m_insertCommand.SetTransaction(m_transaction.Transaction).SetParameterValue("@NewRestoredValue", 1) + var restored = m_insertCommand.SetTransaction(m_transaction.Transaction).SetParameterValue("@NewRestoredValue", 1) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) .SetParameterValue("@PreviousRestoredValue", 0) .ExecuteNonQuery() == 1; + + if (restored) + { + m_copyIntoDuplicatedBlocks.SetTransaction(m_transaction.Transaction) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .ExecuteNonQuery(); + + var c = m_assignBlocksToNewVolume.SetTransaction(m_transaction.Transaction) + .SetParameterValue("@TargetVolumeId", targetVolumeId) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .ExecuteNonQuery(); + + if (c != 1) + throw new Exception($"Unexpected number of updated blocks: {c} != 1"); + } + + return restored; } /// @@ -368,48 +381,70 @@ namespace Duplicati.Library.Main.Database /// public IEnumerable GetBlocklistHashes(long hashesPerBlock) { - using var cmd = m_connection.CreateCommand(m_transaction.Transaction, FormatInvariant($@" - SELECT - b.""Hash"", - bs.""Id"", - bs.""Length"", - bse.""Index"" / @HashesPerBlock AS ""BlocklistHashIndex"", - ( - SELECT blh.""Hash"" - FROM ""BlocklistHash"" blh - WHERE blh.""BlocksetID"" = bs.""ID"" - AND blh.""Index"" == bse.""Index"" / @HashesPerBlock - LIMIT 1 - ) AS ""BlocklistHashHash"", - bse.""Index"" - FROM - ""BlocksetEntry"" bse - JOIN ""Block"" b ON b.""ID"" = bse.""BlockID"" - JOIN ""Blockset"" bs ON bs.""ID"" = bse.""BlocksetID"" - WHERE - EXISTS ( - SELECT 1 - FROM ""BlocklistHash"" blh - JOIN ""{m_tablename}"" mt ON mt.""Hash"" = blh.""Hash"" - WHERE blh.""BlocksetID"" = bs.""ID"" - AND mt.""Restored"" = @Restored - ) - ORDER BY - bse.""BlocksetID"", bse.""Index"" - ")) - .SetParameterValue("@HashesPerBlock", hashesPerBlock) - .SetParameterValue("@Restored", 0); - - foreach (var rd in cmd.ExecuteReaderEnumerable()) + using var cmd = m_connection.CreateCommand(m_transaction.Transaction); + var blocklistTableName = $"BlocklistHashList-{Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray())}"; + try { - var hash = rd.ConvertValueToString(0) ?? throw new Exception("Block.Hash is null"); - var blocksetId = rd.ConvertValueToInt64(1); - var length = rd.ConvertValueToInt64(2); - var blocklistHashIndex = rd.ConvertValueToInt64(3); - var blocklistHash = rd.ConvertValueToString(4) ?? throw new Exception("BlocklistHash is null"); - var index = rd.ConvertValueToInt64(5); + // We need to create a snapshot as we will be updating the m_tablename table during enumeration + cmd.SetCommandAndParameters(FormatInvariant($@" + CREATE TEMPORARY TABLE ""{blocklistTableName}"" AS + SELECT + b.""Hash"" AS ""BlockHash"", + bs.""Id"" AS ""BlocksetId"", + bs.""Length"" AS ""BlocksetLength"", + bse.""Index"" / @HashesPerBlock AS ""BlocklistHashIndex"", + ( + SELECT blh.""Hash"" + FROM ""BlocklistHash"" blh + WHERE blh.""BlocksetID"" = bs.""ID"" + AND blh.""Index"" == bse.""Index"" / @HashesPerBlock + LIMIT 1 + ) AS ""BlocklistHashHash"", + bse.""Index"" AS ""BlocksetEntryIndex"" + FROM + ""BlocksetEntry"" bse + JOIN ""Block"" b ON b.""ID"" = bse.""BlockID"" + JOIN ""Blockset"" bs ON bs.""ID"" = bse.""BlocksetID"" + WHERE + EXISTS ( + SELECT 1 + FROM ""BlocklistHash"" blh + JOIN ""{m_tablename}"" mt ON mt.""Hash"" = blh.""Hash"" + WHERE blh.""BlocksetID"" = bs.""ID"" + AND mt.""Restored"" = @Restored + ) + ")) + .SetParameterValue("@HashesPerBlock", hashesPerBlock) + .SetParameterValue("@Restored", 0) + .ExecuteNonQuery(); - yield return new BlocklistHashesEntry(blocksetId, blocklistHash, length, blocklistHashIndex, (int)index, hash); + cmd.SetCommandAndParameters(FormatInvariant($@" + SELECT + ""BlockHash"", + ""BlocksetId"", + ""BlocksetLength"", + ""BlocklistHashIndex"", + ""BlocklistHashHash"", + ""BlocksetEntryIndex"" + FROM ""{blocklistTableName}"" + ORDER BY ""BlocksetId"", ""BlocklistHashIndex"", ""BlocksetEntryIndex""")); + + foreach (var rd in cmd.ExecuteReaderEnumerable()) + { + var hash = rd.ConvertValueToString(0) ?? throw new Exception("Block.Hash is null"); + var blocksetId = rd.ConvertValueToInt64(1); + var length = rd.ConvertValueToInt64(2); + var blocklistHashIndex = rd.ConvertValueToInt64(3); + var blocklistHash = rd.ConvertValueToString(4) ?? throw new Exception("BlocklistHash is null"); + var index = rd.ConvertValueToInt64(5); + + yield return new BlocklistHashesEntry(blocksetId, blocklistHash, length, blocklistHashIndex, (int)index, hash); + } + } + finally + { + try { cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{blocklistTableName}"" ")); } + catch { } } } diff --git a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs index 43468540f..8eab6aa3d 100644 --- a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs @@ -86,6 +86,11 @@ namespace Duplicati.Library.Main.Operation // Drop all content from tables db.RemoveMissingBlocks(missing.Select(x => x.Name), transaction); + + // Mark all orphaned index files as disposable after removing the missing block files + foreach (var f in db.GetOrphanedIndexFiles(transaction).ToList()) + db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, transaction); + brokensets = db.GetBrokenFilesets(options.Time, options.Version, transaction).ToArray(); } diff --git a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs index 75b2323aa..350ccb157 100644 --- a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs @@ -123,6 +123,15 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(pgoffset); } + var emptyBlocksetId = -1L; + if (!m_options.DisableReplaceMissingMetadata) + { + var emptymetadata = Utility.WrapMetadata(new Dictionary(), m_options); + emptyBlocksetId = db.GetEmptyMetadataBlocksetId(missing.Select(x => x.ID), emptymetadata.FileHash, emptymetadata.Blob.Length, null); + if (emptyBlocksetId < 0) + throw new UserInformationException($"Failed to locate an empty metadata blockset to replace missing metadata. Set the option --disable-replace-missing-metadata=true to ignore this and drop files with missing metadata.", "FailedToLocateEmptyMetadataBlockset"); + } + if (to_purge.Length > 0) { m_result.PurgeResults = new PurgeFilesResults(m_result); @@ -148,7 +157,14 @@ namespace Duplicati.Library.Main.Operation { if (filesetid != bs.FilesetID) throw new Exception(string.Format("Unexpected filesetid: {0}, expected {1}", filesetid, bs.FilesetID)); + + // Update entries that would be removed because of missing metadata + var updatedEntries = 0; + if (!m_options.DisableReplaceMissingMetadata) + updatedEntries = db.ReplaceMetadata(filesetid, emptyBlocksetId, cmd.Transaction); + db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID", cmd.Transaction); + return updatedEntries; }).ConfigureAwait(false); } diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index edea73564..99cf71b8e 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -21,9 +21,9 @@ using System; using System.Linq; -using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Interface; +using Duplicati.Library.Main.Volumes; using Duplicati.Library.Utility; namespace Duplicati.Library.Main.Operation @@ -58,10 +58,10 @@ namespace Duplicati.Library.Main.Operation } } - public Task RunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, float pgoffset, float pgspan, Action filtercommand) + public Task RunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, float pgoffset, float pgspan, Func filtercommand) => DoRunAsync(backendManager, db, null, filtercommand, pgoffset, pgspan); - private async Task DoRunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, IFilter filter, Action filtercommand, float pgoffset, float pgspan) + private async Task DoRunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, IFilter filter, Func filtercommand, float pgoffset, float pgspan) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Begin); Logging.Log.WriteInformationMessage(LOGTAG, "StartingPurge", "Starting purge operation"); @@ -116,34 +116,13 @@ namespace Duplicati.Library.Main.Operation using (var tr = db.BeginTransaction()) { - var ix = -1; - for (var i = 0; i < filesets.Length; i++) - if (filesets[i].Key == versionid) - { - ix = i; - break; - } - - if (ix < 0) + (var _, var tsOriginal, var ix) = filesets.Select((x, i) => (x.Key, x.Value, i)).FirstOrDefault(x => x.Key == versionid); + if (ix < 0 || tsOriginal.Ticks == 0) throw new InvalidProgramException(string.Format("Fileset was reported with id {0}, but could not be found?", versionid)); - var secs = 0; - while (secs < 60) - { - secs++; - var tfn = Volumes.VolumeBase.GenerateFilename(RemoteVolumeType.Files, m_options, null, filesets[ix].Value.AddSeconds(secs)); - if (db.GetRemoteVolumeID(tfn, tr) < 0) - break; - } - - var tsOriginal = filesets[ix].Value; - var ts = tsOriginal.AddSeconds(secs); - + var ts = FilesetVolumeWriter.ProbeUnusedFilenameName(db, tr, m_options, tsOriginal); var prevfilename = db.GetRemoteVolumeNameForFileset(filesets[ix].Key, tr); - if (secs >= 60) - throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is more than 60 seconds away", prevfilename, ts)); - if (ix != 0 && filesets[ix - 1].Value <= ts) throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is larger than the next timestamp {2}", prevfilename, ts, filesets[ix - 1].Value)); @@ -154,7 +133,7 @@ namespace Duplicati.Library.Main.Operation else tempset.ApplyFilter(filtercommand); - if (tempset.RemovedFileCount == 0) + if (tempset.RemovedFileCount + tempset.UpdatedFileCount == 0) { Logging.Log.WriteInformationMessage(LOGTAG, "NotWritingNewFileset", "Not writing a new fileset for {0} as it was not changed", prevfilename); currentprogress += versionprogress; @@ -163,8 +142,8 @@ namespace Duplicati.Library.Main.Operation } else { - using (var tf = new Library.Utility.TempFile()) - using (var vol = new Volumes.FilesetVolumeWriter(m_options, ts)) + using (var tf = new TempFile()) + using (var vol = new FilesetVolumeWriter(m_options, ts)) { var isOriginalFilesetFullBackup = db.IsFilesetFullBackup(tsOriginal, tr); var newids = tempset.ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup); @@ -177,6 +156,7 @@ namespace Duplicati.Library.Main.Operation m_result.RemovedFileSize += tempset.RemovedFileSize; m_result.RemovedFileCount += tempset.RemovedFileCount; + m_result.UpdatedFileCount += tempset.UpdatedFileCount; m_result.RewrittenFileLists++; currentprogress += (versionprogress / 2); @@ -208,7 +188,7 @@ namespace Duplicati.Library.Main.Operation } else { - var lst = db.DropFilesetsFromTable(new[] { filesets[ix].Value }, tr).ToArray(); + var lst = db.DropFilesetsFromTable(new[] { tsOriginal }, tr).ToArray(); foreach (var f in lst) db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, tr); diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 61deca2ad..56f5b91fe 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -20,6 +20,7 @@ // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; +using System.Data; using System.IO; using System.Linq; using System.Threading; @@ -314,12 +315,9 @@ namespace Duplicati.Library.Main.Operation throw; } - if (!m_options.RebuildMissingDblockFiles) - { - var missingDblocks = tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Blocks).ToArray(); - if (missingDblocks.Length > 0) - throw new UserInformationException($"The backup storage destination is missing data files. You can either enable `--rebuild-missing-dblock-files` or run the purge command to remove these files. The following files are missing: {string.Join(", ", missingDblocks.Select(x => x.Name))}", "MissingDblockFiles"); - } + var missingDblocks = tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Blocks); + if (!m_options.RebuildMissingDblockFiles && missingDblocks.Count() > 0) + throw new UserInformationException($"The backup storage destination is missing data files. You can either enable `--rebuild-missing-dblock-files` or run the purge command to remove these files. The following files are missing: {string.Join(", ", missingDblocks.Select(x => x.Name))}", "MissingDblockFiles"); var anyDlistUploads = false; foreach (var (filesetId, timestamp, isfull) in missingRemoteFilesets) @@ -391,56 +389,63 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun && tp.MissingVolumes.Any()) db.TerminatedWithActiveUploads = true; - foreach (var n in tp.MissingVolumes.OrderBy(x => x.Type switch + if (tp.MissingVolumes.Any(x => x.Type != RemoteVolumeType.Index && x.Type != RemoteVolumeType.Files && x.Type != RemoteVolumeType.Blocks)) + throw new InvalidOperationException(string.Format("Unknown volume type {0} detected", tp.MissingVolumes.First(x => x.Type != RemoteVolumeType.Index && x.Type != RemoteVolumeType.Files && x.Type != RemoteVolumeType.Blocks).Type)); + + // Process each of the missing volumes in the order of blocks, files and index + // It is important that we process the blocks first, as the index files are derived from the blocks + + Action incrementProgress = () => { - // Make sure we process the content first, then the fileset - // process index last, as it may have been deleted during block processing - RemoteVolumeType.Blocks => 0, - RemoteVolumeType.Files => 1, - RemoteVolumeType.Index => 2, - _ => 3 - })) + progress++; + m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); + }; + + // Blocks are recreated with the entire list of missing files to handle cases where partial recreation is needed + await RunRepairDblocks(backendManager, db, rtr, missingDblocks, incrementProgress, cancellationToken).ConfigureAwait(false); + + // Filesets are recreated one at a time + foreach (var n in tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Files)) { - IDisposable newEntry = null; + FilesetVolumeWriter newEntry = null; + incrementProgress(); try { - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - { - await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); - return; - } + var timestamp = VolumeBase.ParseFilename(n.Name).Time; + var fileTime = FilesetVolumeWriter.ProbeUnusedFilenameName(db, rtr.Transaction, m_options, timestamp); + var volumeWriter = newEntry = new FilesetVolumeWriter(m_options, fileTime); - progress++; - m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); + await RunRepairDlist(backendManager, db, rtr, volumeWriter, n, fileTime, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + if (newEntry != null) + try { newEntry?.Dispose(); } + catch { } - if (n.Type == RemoteVolumeType.Files) - { - var timestamp = VolumeBase.ParseFilename(n.Name).Time; - var fileTime = FilesetVolumeWriter.ProbeUnusedFilenameName(db, rtr.Transaction, m_options, timestamp); - var volumeWriter = new FilesetVolumeWriter(m_options, fileTime); + Logging.Log.WriteErrorMessage(LOGTAG, "CleanupMissingFileError", ex, "Failed to perform cleanup for missing file: {0}, message: {1}", n.Name, ex.Message); - newEntry = volumeWriter; - await RunRepairDlist(backendManager, db, rtr, volumeWriter, n, fileTime, cancellationToken).ConfigureAwait(false); - } - else if (n.Type == RemoteVolumeType.Index) - { - var w = new IndexVolumeWriter(m_options); - newEntry = w; + if (ex.IsAbortException()) + throw; + } + } - // Check if the index file has already been deleted, beause the dblock was recreated - var currentState = db.GetRemoteVolume(n.Name, rtr.Transaction).State; - if (currentState == RemoteVolumeState.Deleted || currentState == RemoteVolumeState.Temporary || currentState == RemoteVolumeState.Deleting) - continue; + // Index files are recreated one at a time, as they are derived from the blocks + foreach (var n in tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Index)) + { + IndexVolumeWriter newEntry = null; + incrementProgress(); - await RunRepairDindex(backendManager, db, rtr, w, n, cancellationToken).ConfigureAwait(false); - } - else if (n.Type == RemoteVolumeType.Blocks) - { - var w = new BlockVolumeWriter(m_options); - newEntry = w; - await RunRepairDblock(backendManager, db, rtr, w, n, cancellationToken).ConfigureAwait(false); - } + try + { + // Check if the index file has already been deleted, beause the dblock was recreated + var currentState = db.GetRemoteVolume(n.Name, rtr.Transaction).State; + if (currentState == RemoteVolumeState.Deleted || currentState == RemoteVolumeState.Temporary || currentState == RemoteVolumeState.Deleting) + continue; + + var w = newEntry = new IndexVolumeWriter(m_options); + await RunRepairDindex(backendManager, db, rtr, w, n, cancellationToken).ConfigureAwait(false); } catch (Exception ex) { @@ -508,10 +513,21 @@ namespace Duplicati.Library.Main.Operation } } - private async Task RunRepairDlist(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, FilesetVolumeWriter volumeWriter, RemoteVolumeEntry n, DateTime filesetTime, CancellationToken cancellationToken) + /// + /// Repairs a single fileset by recreating it from the database content and uploading it + /// + /// The backend manager to use for uploading + /// The database to use for the repair + /// The transaction to use for the repair + /// The volume writer to use for the repair + /// The remote volume entry to repair + /// The time of the new fileset to create + /// Cancellation token to cancel the operation + /// A task representing the asynchronous operation + private async Task RunRepairDlist(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, FilesetVolumeWriter volumeWriter, RemoteVolumeEntry originalVolume, DateTime filesetTime, CancellationToken cancellationToken) { volumeWriter.VolumeID = db.RegisterRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, rtr.Transaction); - (var prevFilesetId, var _, var isPrevFull) = db.GetFilesetFromRemotename(n.Name, rtr.Transaction); + (var prevFilesetId, var _, var isPrevFull) = db.GetFilesetFromRemotename(originalVolume.Name, rtr.Transaction); if (!string.IsNullOrEmpty(m_options.ControlFiles)) foreach (var p in m_options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) @@ -526,11 +542,11 @@ namespace Duplicati.Library.Main.Operation volumeWriter.Close(); if (m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would re-upload fileset {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(volumeWriter.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)); + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would re-upload fileset {0}, with size {1}, previous size {2}", originalVolume.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(volumeWriter.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(originalVolume.Size)); else { db.UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null, rtr.Transaction); - db.UpdateRemoteVolume(n.Name, RemoteVolumeState.Deleting, n.Size, n.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); + db.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleting, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) rtr.Commit("CommitPriorToFilesetUpload"); @@ -542,21 +558,33 @@ namespace Duplicati.Library.Main.Operation rtr.Commit("CommitFilesetTransaction"); } - private async Task RunRepairDindex(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, IndexVolumeWriter w, RemoteVolumeEntry n, CancellationToken cancellationToken) + /// + /// Repairs a single index file by recreating it from the database content and uploading it + /// + /// The backend manager to use for uploading + /// The database to use for the repair + /// The transaction to use for the repair + /// The volume writer to use for the repair + /// The remote volume entry to repair + /// The time of the new fileset to create + /// Cancellation token to cancel the operation + /// A task representing the asynchronous operation + + private async Task RunRepairDindex(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, IndexVolumeWriter indexWriter, RemoteVolumeEntry originalVolume, CancellationToken cancellationToken) { - w.VolumeID = db.RegisterRemoteVolume(w.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Uploading, -1, TimeSpan.Zero, rtr.Transaction); + indexWriter.VolumeID = db.RegisterRemoteVolume(indexWriter.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Uploading, -1, TimeSpan.Zero, rtr.Transaction); var blockvolumeids = new List(); using var h = HashFactory.CreateHasher(m_options.BlockHashAlgorithm); - foreach (var blockvolume in db.GetBlockVolumesFromIndexName(n.Name)) + foreach (var blockvolume in db.GetBlockVolumesFromIndexName(originalVolume.Name)) { - w.StartVolume(blockvolume.Name); + indexWriter.StartVolume(blockvolume.Name); var volumeid = db.GetRemoteVolumeID(blockvolume.Name, rtr.Transaction); foreach (var b in db.GetBlocks(volumeid)) - w.AddBlock(b.Hash, b.Size); + indexWriter.AddBlock(b.Hash, b.Size); - w.FinishVolume(blockvolume.Hash, blockvolume.Size); + indexWriter.FinishVolume(blockvolume.Hash, blockvolume.Size); blockvolumeids.Add(volumeid); if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) @@ -566,22 +594,22 @@ namespace Duplicati.Library.Main.Operation if (bh != b.Item1) throw new Exception(string.Format("Internal consistency check failed, generated index block has wrong hash, {0} vs {1}", bh, b.Item1)); - w.WriteBlocklist(b.Item1, b.Item2, 0, b.Item3); + indexWriter.WriteBlocklist(b.Item1, b.Item2, 0, b.Item3); } } foreach (var blockvolumeid in blockvolumeids) - db.AddIndexBlockLink(w.VolumeID, blockvolumeid, rtr.Transaction); + db.AddIndexBlockLink(indexWriter.VolumeID, blockvolumeid, rtr.Transaction); - w.Close(); + indexWriter.Close(); if (m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadIndexFile", "would re-upload index file {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)); + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadIndexFile", "would re-upload index file {0}, with size {1}, previous size {2}", originalVolume.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(indexWriter.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(originalVolume.Size)); else { - db.UpdateRemoteVolume(n.Name, RemoteVolumeState.Deleting, n.Size, n.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); + db.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleting, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); - await backendManager.PutAsync(w, null, null, false, null, cancellationToken).ConfigureAwait(false); + await backendManager.PutAsync(indexWriter, null, null, false, null, cancellationToken).ConfigureAwait(false); } await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); @@ -589,17 +617,260 @@ namespace Duplicati.Library.Main.Operation rtr.Commit("CommitRepairTransaction"); } - private async Task RunRepairDblock(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, BlockVolumeWriter w, RemoteVolumeEntry n, CancellationToken cancellationToken) + /// + /// Class for holding the state of a dblock volume that is being constructed + /// + private sealed class InProgressDblockVolumes { - // TODO: Keep an open volume and append to that until we reach the size threshold, - // similar to how the creation works. For now we just attempt to recreate the original volume + /// + /// Database to use for the operation + /// + private readonly LocalDatabase m_database; + /// + /// Transaction to use for the operation + /// + private readonly ReusableTransaction m_transaction; + /// + /// List of volumes to delete after the upload is complete + /// + private readonly List m_toDelete = new List(); + /// + /// Options for the current operation + /// + private readonly Options m_options; + /// + /// Maximum size of the volume before it is considered full + /// + private readonly long m_maxVolumeSize; + /// + /// The writers that are completed and ready to be uploaded + /// + private readonly List m_completedWriters = new List(); + /// + /// Writer for the current in-progress volume + /// + private BlockVolumeWriter m_activeWriter; + /// + /// Flag indicating if any data has been added to the volume + /// + private bool m_anyData = false; + /// + /// Constructor for the InProgressDblockVolume class + /// + /// The options for the current operation + /// The database to use for registering a volume in progress + /// The transaction to use for the operation + public InProgressDblockVolumes(Options options, LocalDatabase db, ReusableTransaction transaction) + { + m_database = db; + m_transaction = transaction; + m_options = options; + m_maxVolumeSize = options.VolumeSize - m_options.Blocksize; + m_activeWriter = new BlockVolumeWriter(options); + m_activeWriter.VolumeID = m_database.RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, m_transaction.Transaction); + } + + /// + /// Checks if any data has been added to the current volume + /// + public bool AnyData => m_anyData; + /// + /// Checks if the current volume is full + /// + private bool IsFull => m_activeWriter.Filesize > m_maxVolumeSize; + + /// + /// Gets the volume ID of the current volume + /// + public long VolumeID => m_activeWriter.VolumeID; + /// + /// Gets the remote filename of the current volume + /// + public string RemoteFilename => m_activeWriter.RemoteFilename; + /// + /// Gets the list of volumes to delete after the upload is complete + /// + public List VolumesToDelete => m_toDelete; + /// + /// Gets the list of completed writers + /// + public List CompletedWriters => m_completedWriters; + + /// + /// Adds a block to the current volume, starting a new volume if the current one is full + /// + /// The hash of the block + /// The buffer containing the block data + /// The offset in the buffer where the block data starts + /// The size of the block data + /// The compression hint for the block + public void AddBlock(string hash, byte[] buffer, int offset, int size, CompressionHint hint) + { + m_activeWriter.AddBlock(hash, buffer, offset, size, hint); + m_anyData = true; + if (IsFull) + StartNewVolume(); + } + + /// + /// Starts a new volume for the current operation + /// + public void StartNewVolume() + { + if (!m_anyData) + return; + + m_completedWriters.Add(m_activeWriter); + + m_activeWriter = new BlockVolumeWriter(m_options); + m_activeWriter.VolumeID = m_database.RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, m_transaction.Transaction); + m_anyData = false; + } + } + + /// + /// Runs the repair process for missing dblock files + /// + /// The backend manager to use for uploading + /// The database to use for the repair + /// The transaction to use for the repair + /// The list of missing dblock files + /// + /// Cancellation token to cancel the operation + /// >A task representing the asynchronous operation + private async Task RunRepairDblocks(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, IEnumerable missingDblockFiles, Action incrementProgress, CancellationToken cancellationToken) + { + var currentVolume = new InProgressDblockVolumes(m_options, db, rtr); + + foreach (var n in missingDblockFiles) + { + try + { + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + { + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + if (!m_options.Dryrun) + rtr.Commit("CommitEarlyExit", false); + return; + } + + incrementProgress(); + + await RunRepairDblock(backendManager, db, rtr, n, currentVolume, cancellationToken).ConfigureAwait(false); + if (currentVolume.CompletedWriters.Count > 0) + await UploadCompletedBlockVolumes(backendManager, currentVolume, db, rtr, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "RepairDblockError", ex, "Failed to perform repair for dblock: {0}, message: {1}", n.Name, ex.Message); + if (ex.IsAbortException()) + throw; + } + } + + // If we have any data in the volume, complete it so it will be uploaded + if (currentVolume.AnyData) + currentVolume.StartNewVolume(); + // Complete any pending uploads + await UploadCompletedBlockVolumes(backendManager, currentVolume, db, rtr, cancellationToken).ConfigureAwait(false); + + // After reset, we need to remove the temporary volume from the database + db.RemoveRemoteVolume(currentVolume.RemoteFilename, rtr.Transaction); + } + + /// + /// Uploads all completed block volumes to the backend and deletes the old files + /// + /// The backend manager to use for uploading + /// The completed volume state to upload + /// The database to use for the upload + /// >The transaction to use for the upload + /// Cancellation token to cancel the operation + /// >A task representing the asynchronous operation + async Task UploadCompletedBlockVolumes(IBackendManager backendManager, InProgressDblockVolumes activeVolume, LocalRepairDatabase db, ReusableTransaction rtr, CancellationToken cancellationToken) + { + foreach (var completedVolume in activeVolume.CompletedWriters) + { + db.UpdateRemoteVolume(completedVolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, rtr.Transaction); + + // Create a new index file that points to the new volume + IndexVolumeWriter newvolindex = null; + Action indexVolumeFinished = null; + if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None) + { + newvolindex = new IndexVolumeWriter(m_options); + newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, rtr.Transaction); + newvolindex.StartVolume(completedVolume.RemoteFilename); + foreach (var b in db.GetBlocks(completedVolume.VolumeID)) + newvolindex.AddBlock(b.Hash, b.Size); + + db.AddIndexBlockLink(newvolindex.VolumeID, completedVolume.VolumeID, rtr.Transaction); + if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) + indexVolumeFinished = () => + { + foreach (var blocklist in db.GetBlocklists(completedVolume.VolumeID, m_options.Blocksize, m_options.BlockhashSize)) + newvolindex.WriteBlocklist(blocklist.Item1, blocklist.Item2, 0, blocklist.Item3); + }; + } + + // All information is in place, we can now upload the new volume + await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + if (m_options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReplaceBlockFile", "would upload new block file {0}", completedVolume.RemoteFilename); + else + { + rtr.Commit("PostRepairPreUploadBlockVolume"); + await backendManager.PutAsync(completedVolume, newvolindex, indexVolumeFinished, false, null, cancellationToken).ConfigureAwait(false); + } + } + + // Flush the database as we have new volumes uploaded + await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + + // Prepare for deleting the old stuff + foreach (var vol in activeVolume.VolumesToDelete) + db.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Deleting, vol.Size, vol.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); + + // Persist desired state prior to deleting the old files + if (!m_options.Dryrun) + rtr.Commit("PostRepairBlockVolume"); + + // Delete the old files + foreach (var vol in activeVolume.VolumesToDelete) + if (m_options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteIndexFile", "would delete index file {0}", vol.Name); + else + await backendManager.DeleteAsync(vol.Name, vol.Size, false, cancellationToken).ConfigureAwait(false); + + // All done, the new dblocks are now in place + await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); + if (!m_options.Dryrun) + rtr.Commit("PostRepairBlockVolume"); + + activeVolume.CompletedWriters.Clear(); + activeVolume.VolumesToDelete.Clear(); + } + + /// + /// Repairs a single dblock file by recreating it from available data + /// + /// The backend manager to use for uploading + /// The database to use for the repair + /// The transaction to use for the repair + /// The remote volume entry to recreate + /// The in-progress dblock volume to use for the recreate + /// Cancellation token to cancel the operation + /// >A task representing the asynchronous operation + private async Task RunRepairDblock(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, RemoteVolumeEntry originalVolume, InProgressDblockVolumes pendingVolume, CancellationToken cancellationToken) + { // The dblock files are the most complex to recreate // as data can be either file contents, metadata or blocklist hashes // We attempt to recover all three source parts in the steps below - using (var mbl = db.CreateBlockList(n.Name, rtr)) + using (var mbl = db.CreateBlockList(originalVolume.Name, rtr)) { var originalMissingBlockCount = mbl.GetMissingBlockCount(); + var recoveredSourceBlocks = 0L; // First we grab all known blocks from local files string lastRestoredHash = null; @@ -613,41 +884,37 @@ namespace Duplicati.Library.Main.Operation try { - if (File.Exists(block.File)) - { - using (var f = File.OpenRead(block.File)) - { - f.Position = block.Offset; - - var buffer = new byte[m_options.Blocksize]; - if (size == Library.Utility.Utility.ForceStreamRead(f, buffer, size)) - { - using (var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm)) - { - var newhash = Convert.ToBase64String(blockhasher.ComputeHash(buffer, 0, size)); - if (newhash == block.Hash) - { - // Found it, no need to look again - lastRestoredHash = block.Hash; - lastRestoredSize = block.Size; - if (mbl.SetBlockRestored(block.Hash, block.Size)) - w.AddBlock(block.Hash, buffer, 0, size, CompressionHint.Default); - } - else - { - Logging.Log.WriteVerboseMessage(LOGTAG, "FileHashMismatch", null, "Block hash mismatch on {0}, expected {1} but got {2}", block.File, block.Hash, newhash); - } - } - } - else - { - Logging.Log.WriteVerboseMessage(LOGTAG, "FileLengthMismatch", null, "Block length mismatch on {0}, expected {1} but got {2}", block.File, size, f.Length); - } - } - } - else + if (!File.Exists(block.File)) { Logging.Log.WriteVerboseMessage(LOGTAG, "FileNotFound", null, "File not found: {0}", block.File); + continue; + } + + using var f = File.OpenRead(block.File); + f.Position = block.Offset; + + var buffer = new byte[m_options.Blocksize]; + if (size != Library.Utility.Utility.ForceStreamRead(f, buffer, size)) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "FileLengthMismatch", null, "Block length mismatch on {0}, expected {1} but got {2}", block.File, size, f.Length); + continue; + } + + using var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm); + var newhash = Convert.ToBase64String(blockhasher.ComputeHash(buffer, 0, size)); + if (newhash != block.Hash) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "FileHashMismatch", null, "Block hash mismatch on {0}, expected {1} but got {2}", block.File, block.Hash, newhash); + continue; + } + + // Found it, no need to look again + lastRestoredHash = block.Hash; + lastRestoredSize = block.Size; + if (mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID)) + { + recoveredSourceBlocks++; + pendingVolume.AddBlock(block.Hash, buffer, 0, size, CompressionHint.Default); } } catch (Exception ex) @@ -668,34 +935,35 @@ namespace Duplicati.Library.Main.Operation { var isFile = File.Exists(block.Path); var isDir = Directory.Exists(block.Path); - if (isFile || isDir) - { - using var snapshot = Snapshots.SnapshotUtility.CreateNoSnapshot([block.Path], true, true); - var entry = snapshot.GetFilesystemEntry(block.Path, isDir); - if (entry == null) - { - Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", null, "Entry not found: {0}", block.Path); - continue; - } - - var metadata = MetadataGenerator.GenerateMetadata(entry, entry.Attributes, m_options); - var metahash = Utility.WrapMetadata(metadata, m_options); - if (metahash.FileHash == block.Hash && metahash.Blob.Length == size) - { - // Found it, no need to look again - lastRestoredHash = block.Hash; - lastRestoredSize = block.Size; - if (mbl.SetBlockRestored(block.Hash, block.Size)) - w.AddBlock(block.Hash, metahash.Blob, 0, metahash.Blob.Length, CompressionHint.Default); - } - else - { - Logging.Log.WriteVerboseMessage(LOGTAG, "MetadataHashMismatch", null, "Metadata block hash mismatch on {0}, expected {1} but got {2}", block.Path, block.Hash, metahash.FileHash); - } - } - else + if (!isFile || !isDir) { Logging.Log.WriteVerboseMessage(LOGTAG, "EntryNotFound", null, "Entry not found: {0}", block.Path); + continue; + } + + using var snapshot = Snapshots.SnapshotUtility.CreateNoSnapshot([block.Path], true, true); + var entry = snapshot.GetFilesystemEntry(block.Path, isDir); + if (entry == null) + { + Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", null, "Entry not found: {0}", block.Path); + continue; + } + + var metadata = MetadataGenerator.GenerateMetadata(entry, entry.Attributes, m_options); + var metahash = Utility.WrapMetadata(metadata, m_options); + if (metahash.FileHash != block.Hash || metahash.Blob.Length != size) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "MetadataHashMismatch", null, "Metadata block hash mismatch on {0}, expected {1} but got {2}", block.Path, block.Hash, metahash.FileHash); + continue; + } + + // Found it, no need to look again + lastRestoredHash = block.Hash; + lastRestoredSize = block.Size; + if (mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID)) + { + recoveredSourceBlocks++; + pendingVolume.AddBlock(block.Hash, metahash.Blob, 0, metahash.Blob.Length, CompressionHint.Default); } } catch (Exception ex) @@ -704,79 +972,83 @@ namespace Duplicati.Library.Main.Operation } } - // Then restore any blocklists that are missing - using var blocklistHash = new MemoryStream(); - LocalRepairDatabase.BlocklistHashesEntry lastBlocklist = null; - var blockhashsize = m_options.BlockhashSize; - var hashesPerBlock = m_options.Blocksize / m_options.BlockhashSize; - - void EmitBlockListBlock() + // If we did not find any source blocks, + // we only have the database contents and duplicated blocks to work with + // so no point in creating a new dblock volume for storing this data + if (recoveredSourceBlocks != 0) { - blocklistHash.Position = 0; - using var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm); - var resulthash = Convert.ToBase64String(blockhasher.ComputeHash(blocklistHash)); - var totalHashes = (lastBlocklist.BlocklistHashLength + m_options.Blocksize - 1) / m_options.Blocksize; - var hashesInLastBlock = totalHashes % hashesPerBlock; - var targetsize = lastBlocklist.Index >= totalHashes - 1 - ? hashesInLastBlock * blockhashsize - : hashesPerBlock * blockhashsize; + // Then restore any blocklists that are missing + using var blocklistHash = new MemoryStream(); + LocalRepairDatabase.BlocklistHashesEntry lastBlocklist = null; + var blockhashsize = m_options.BlockhashSize; + var hashesPerBlock = m_options.Blocksize / m_options.BlockhashSize; - if (resulthash == lastBlocklist.BlocklistHash && targetsize == blocklistHash.Length) + // Helper function to emit a blocklist hash + void EmitBlockListBlock() { - if (mbl.SetBlockRestored(resulthash, blocklistHash.Length)) - w.AddBlock(resulthash, blocklistHash.ToArray(), 0, (int)blocklistHash.Length, CompressionHint.Default); - } - else - { - Logging.Log.WriteWarningMessage(LOGTAG, "BlocklistHashMismatch", null, "Blocklist hash mismatch on {0} / {1}, expected {2} but got {3}", lastBlocklist.BlocklistHash, lastBlocklist.Index, lastBlocklist.Hash, resulthash); + blocklistHash.Position = 0; + using var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm); + var resulthash = Convert.ToBase64String(blockhasher.ComputeHash(blocklistHash)); + var totalHashes = (lastBlocklist.BlocklistHashLength + m_options.Blocksize - 1) / m_options.Blocksize; + var hashesInLastBlock = totalHashes % hashesPerBlock; + var targetsize = lastBlocklist.Index >= totalHashes - 1 + ? hashesInLastBlock * blockhashsize + : hashesPerBlock * blockhashsize; + + if (resulthash == lastBlocklist.BlocklistHash && targetsize == blocklistHash.Length) + { + if (mbl.SetBlockRestored(resulthash, blocklistHash.Length, pendingVolume.VolumeID)) + pendingVolume.AddBlock(resulthash, blocklistHash.ToArray(), 0, (int)blocklistHash.Length, CompressionHint.Default); + } + else + { + Logging.Log.WriteWarningMessage(LOGTAG, "BlocklistHashMismatch", null, "Internal consistency issue: blocklist hash mismatch on {0} / {1}, expected {2} but got {3}", lastBlocklist.BlocklistHash, lastBlocklist.Index, lastBlocklist.Hash, resulthash); + } + + blocklistHash.SetLength(0); } - blocklistHash.SetLength(0); - } + foreach (var blocklist in mbl.GetBlocklistHashes(hashesPerBlock)) + { + lastBlocklist ??= blocklist; - foreach (var blocklist in mbl.GetBlocklistHashes(hashesPerBlock)) - { - lastBlocklist ??= blocklist; + if (lastBlocklist.BlocksetId != blocklist.BlocksetId || lastBlocklist.BlocklistHashIndex != blocklist.BlocklistHashIndex) + EmitBlockListBlock(); - if (lastBlocklist.BlocksetId != blocklist.BlocksetId || lastBlocklist.BlocklistHashIndex != blocklist.BlocklistHashIndex) + var data = Convert.FromBase64String(blocklist.Hash); + blocklistHash.Write(data, 0, data.Length); + lastBlocklist = blocklist; + } + + // Handle any trailing blocklist hash + if (blocklistHash.Length > 0) EmitBlockListBlock(); - var data = Convert.FromBase64String(blocklist.Hash); - blocklistHash.Write(data, 0, data.Length); - lastBlocklist = blocklist; - } - - // Handle any trailing blocklist hash - if (blocklistHash.Length > 0) - EmitBlockListBlock(); - - //Then we grab all remote volumes that have the missing blocks - await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(mbl.GetMissingBlockSources().ToList(), cancellationToken).ConfigureAwait(false)) - { - try + //Then we grab all remote volumes that have the missing blocks + await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(mbl.GetMissingBlockSources().ToList(), cancellationToken).ConfigureAwait(false)) { - var buffer = new byte[m_options.Blocksize]; - using (tmpfile) - using (var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options)) - foreach (var b in f.Blocks) - if (mbl.SetBlockRestored(b.Key, b.Value)) - if (f.ReadBlock(b.Key, buffer) == b.Value) - w.AddBlock(b.Key, buffer, 0, (int)b.Value, CompressionHint.Default); - } - catch (Exception ex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileAccessError", ex, "Failed to access remote file: {0}", name); + try + { + var buffer = new byte[m_options.Blocksize]; + using (tmpfile) + using (var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options)) + foreach (var b in f.Blocks) + if (mbl.SetBlockRestored(b.Key, b.Value, pendingVolume.VolumeID)) + if (f.ReadBlock(b.Key, buffer) == b.Value) + pendingVolume.AddBlock(b.Key, buffer, 0, (int)b.Value, CompressionHint.Default); + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileAccessError", ex, "Failed to access remote file: {0}", name); + } } } - // If we managed to recover all blocks, NICE! var missingBlocks = mbl.GetMissingBlockCount(); - if (missingBlocks > 0) + var recoveredBlocks = originalMissingBlockCount - missingBlocks; + if (recoveredBlocks == 0 || (m_options.DisablePartialDblockRecovery && missingBlocks > 0)) { - // TODO: If we managed to get ANY blocks recovered, we can register a new volume - // and move existing blocks to duplicate, so a later purge has less negative impact - - Logging.Log.WriteInformationMessage(LOGTAG, "RepairMissingBlocks", "Repair cannot acquire {0} required blocks for volume {1}, which are required by the following filesets: ", missingBlocks, n.Name); + Logging.Log.WriteInformationMessage(LOGTAG, "RepairMissingBlocks", "Repair cannot acquire {0} required blocks for volume {1}, which are required by the following filesets: ", missingBlocks, originalVolume.Name); foreach (var f in mbl.GetFilesetsUsingMissingBlocks()) Logging.Log.WriteInformationMessage(LOGTAG, "AffectedFilesetName", f.Name); @@ -784,91 +1056,40 @@ namespace Duplicati.Library.Main.Operation var logmsg = string.Format("Repair not possible, missing {0} blocks.\n" + recoverymsg, missingBlocks); Logging.Log.WriteInformationMessage(LOGTAG, "RecoverySuggestion", null, logmsg); - if (!m_options.Dryrun) - throw new UserInformationException(logmsg, "RepairIsNotPossible"); + throw new UserInformationException(logmsg, "RepairIsNotPossible"); } - else + else if (recoveredBlocks > 0) { - w.VolumeID = db.RegisterRemoteVolume(w.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Uploading, -1, TimeSpan.Zero, rtr.Transaction); + if (missingBlocks > 0) + Logging.Log.WriteWarningMessage(LOGTAG, "RepairMissingBlocks", null, "Repair acquired {0} blocks for volume {1}, but {2} blocks are still missing. If you want to continue working with the database, you can use the \"{3}\" and \"{4}\" commands to purge the missing data from the database and the remote storage.", recoveredBlocks, originalVolume.Name, missingBlocks, "list-broken-files", "purge-broken-files"); - var fixedBlockCount = mbl.MoveBlocksToNewVolume(w.VolumeID, n.ID, rtr.Transaction); - if (fixedBlockCount != originalMissingBlockCount) - throw new UserInformationException(string.Format("Failed to move {0} blocks to new volume {1}, only moved {2}", originalMissingBlockCount, w.RemoteFilename, fixedBlockCount), "FailedToMoveBlocks"); - - // Create a new index file that points to the new volume - IndexVolumeWriter newvolindex = null; - Action indexVolumeFinished = null; - if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None) + // If we cannot fully recover the blocks, we will not be able to delete the old files. + // They will instead be registered with duplicated blocks, so the purge-broken-files command + // can remove them cleanly later + if (missingBlocks == 0) { - newvolindex = new IndexVolumeWriter(m_options); - newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, rtr.Transaction); - newvolindex.StartVolume(w.RemoteFilename); - foreach (var b in db.GetBlocks(w.VolumeID)) - newvolindex.AddBlock(b.Hash, b.Size); + pendingVolume.VolumesToDelete.Add(originalVolume); + var oldIndexFiles = db.GetIndexFilesReferencingBlockFile(originalVolume.ID, rtr.Transaction).ToList(); - db.AddIndexBlockLink(newvolindex.VolumeID, w.VolumeID, rtr.Transaction); - if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) - indexVolumeFinished = () => + // Find all index files that point to the old volume + foreach (var oldIndexFile in oldIndexFiles) + { + var oldVolume = db.GetRemoteVolume(oldIndexFile, rtr.Transaction); + if (oldVolume.State == RemoteVolumeState.Uploading || oldVolume.State == RemoteVolumeState.Uploaded || oldVolume.State == RemoteVolumeState.Verified) { - foreach (var blocklist in db.GetBlocklists(w.VolumeID, m_options.Blocksize, m_options.BlockhashSize)) - newvolindex.WriteBlocklist(blocklist.Item1, blocklist.Item2, 0, blocklist.Item3); - }; - } - - var oldIndexFiles = db.GetIndexFilesReferencingBlockFile(n.ID, rtr.Transaction).ToList(); - var toDelete = new List(); - // Find all index files that point to the old volume - foreach (var oldIndexFile in oldIndexFiles) - { - var oldVolume = db.GetRemoteVolume(oldIndexFile, rtr.Transaction); - if (oldVolume.State == RemoteVolumeState.Uploading || oldVolume.State == RemoteVolumeState.Uploaded || oldVolume.State == RemoteVolumeState.Verified) - { - var blockVolumesReferenced = db.GetBlockVolumesFromIndexName(oldIndexFile).ToList(); - if (blockVolumesReferenced.Any(x => x.Name != n.Name)) - Logging.Log.WriteVerboseMessage(LOGTAG, "IndexFileNotDeleted", null, "Index file {0} references multiple remote volumes, skipping", oldVolume.Name); + var blockVolumesReferenced = db.GetBlockVolumesFromIndexName(oldIndexFile).ToList(); + if (blockVolumesReferenced.Any(x => x.Name != originalVolume.Name)) + Logging.Log.WriteVerboseMessage(LOGTAG, "IndexFileNotDeleted", null, "Index file {0} references multiple remote volumes, skipping", oldVolume.Name); + else + pendingVolume.VolumesToDelete.Add(oldVolume); + } else - toDelete.Add(oldVolume); - } - else - { - Logging.Log.WriteVerboseMessage(LOGTAG, "IndexFileNotDeleted", null, "Index file {0} is not in a state to be deleted, skipping", oldVolume.Name); + { + Logging.Log.WriteVerboseMessage(LOGTAG, "IndexFileNotDeleted", null, "Index file {0} is not in a state to be deleted, skipping", oldVolume.Name); + } } } - - // All information is in place, we can now upload the new volume - await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); - if (m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReplaceBlockFile", "would replace block file {0} with {1}", n.Name, w.RemoteFilename); - else - { - rtr.Commit("PostRepairPreUploadBlockVolume"); - await backendManager.PutAsync(w, newvolindex, indexVolumeFinished, false, null, cancellationToken).ConfigureAwait(false); - } - - // Flush the database as we have a new volume uploaded - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); - - // Prepare for deleting the old stuff - db.UpdateRemoteVolume(n.Name, RemoteVolumeState.Deleting, n.Size, n.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); - foreach (var vol in toDelete) - db.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Deleting, vol.Size, vol.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); - - // Persist desired state prior to deleting the old files - if (!m_options.Dryrun) - rtr.Commit("PostRepairBlockVolume"); - - // Delete the old files - foreach (var vol in toDelete) - if (m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteIndexFile", "would delete index file {0}", vol.Name); - else - await backendManager.DeleteAsync(vol.Name, vol.Size, false, cancellationToken).ConfigureAwait(false); } - - // All done, the new dblock is now in place - await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); - if (!m_options.Dryrun) - rtr.Commit("PostRepairBlockVolume"); } } diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index f5f7bd8fb..18312efaa 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -471,6 +471,8 @@ namespace Duplicati.Library.Main new CommandLineArgument("ignore-filenames", CommandLineArgument.ArgumentType.Path, Strings.Options.IgnorefilenamesShort, Strings.Options.IgnorefilenamesLong, "CACHEDIR.TAG"), new CommandLineArgument("restore-symlink-metadata", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RestoresymlinkmetadataShort, Strings.Options.RestoresymlinkmetadataLong, "false"), new CommandLineArgument("rebuild-missing-dblock-files", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RebuildmissingdblockfilesShort, Strings.Options.RebuildmissingdblockfilesLong, "false"), + new CommandLineArgument("disable-partial-dblock-recovery", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablePartialDblockRecoveryShort, Strings.Options.DisablePartialDblockRecoveryLong, "false"), + new CommandLineArgument("disable-replace-missing-metadata", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisableReplaceMissingMetadataShort, Strings.Options.DisableReplaceMissingMetadataLong, "false"), new CommandLineArgument("auto-compact-interval", CommandLineArgument.ArgumentType.Timespan, Strings.Options.AutoCompactIntervalShort, Strings.Options.AutoCompactIntervalLong, "0m"), new CommandLineArgument("auto-vacuum-interval", CommandLineArgument.ArgumentType.Timespan, Strings.Options.AutoVacuumIntervalShort, Strings.Options.AutoVacuumIntervalLong, "0m"), @@ -1927,6 +1929,22 @@ namespace Duplicati.Library.Main get { return GetBool("rebuild-missing-dblock-files"); } } + /// + /// Gets a value indicating if partial dblock recovery is disabled + /// + public bool DisablePartialDblockRecovery + { + get { return GetBool("disable-partial-dblock-recovery"); } + } + + /// + /// Gets a value indicating if missing metadata is replaced with empty content on purge-broken-files + /// + public bool DisableReplaceMissingMetadata + { + get { return GetBool("disable-replace-missing-metadata"); } + } + /// /// Gets the threshold for when log data should be cleaned /// diff --git a/Duplicati/Library/Main/ResultClasses.cs b/Duplicati/Library/Main/ResultClasses.cs index 14d0909df..2ab7aa2ca 100644 --- a/Duplicati/Library/Main/ResultClasses.cs +++ b/Duplicati/Library/Main/ResultClasses.cs @@ -902,6 +902,7 @@ namespace Duplicati.Library.Main public long RemovedFileCount { get; set; } public long RemovedFileSize { get; set; } public long RewrittenFileLists { get; set; } + public long UpdatedFileCount { get; set; } public ICompactResults CompactResults { get; set; } } diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index e24966af4..6db36589e 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -310,6 +310,10 @@ namespace Duplicati.Library.Main.Strings public static string ProfilealldatabasequeriesShort { get { return LC.L("Activate logging of all database queries"); } } public static string RebuildmissingdblockfilesLong { get { return LC.L("If dblock files are missing from the destination, you can attempt to rebuild them using local source data. However, since the local data may have changed, it may not be possible to retrieve all the required data and the process may be slow. Use this option to attempt to rebuild missing dblock files."); } } public static string RebuildmissingdblockfilesShort { get { return LC.L("Rebuild dblock files when missing"); } } + public static string DisablePartialDblockRecoveryLong { get { return LC.L("If rebuilding dblock files, the process will attempt to use local data to fill in missing blocks. If it is not possible to fill in all missing blocks, Duplicati will attempt to create a dblock with as much data as possible. Use this option to disable this behavior and only create dblocks that are complete."); } } + public static string DisablePartialDblockRecoveryShort { get { return LC.L("Disable partial dblock recovery"); } } + public static string DisableReplaceMissingMetadataLong { get { return LC.L($"If metadata is missing during the {"list-broken-files"} or {"purge-broken-files"} command, Duplicati will attempt to replace the missing metadata with a new copy. Use this option to disable this behavior and drop entries that are missing metadata."); } } + public static string DisableReplaceMissingMetadataShort { get { return LC.L("Disable replacement of missing metadata"); } } public static string AutoCompactIntervalLong { get { return LC.L("The minimum amount of time that must elapse after the last compaction before another will be automatically triggered at the end of a backup job. Automatic compaction can be a long-running process and may not be desirable to run after every single backup."); } } public static string AutoCompactIntervalShort { get { return LC.L("Minimum time between auto compactions"); } } diff --git a/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs b/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs index b3d5e65a2..4507036f1 100644 --- a/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs +++ b/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs @@ -23,6 +23,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; using Duplicati.Library.SQLiteHelper; using NUnit.Framework; @@ -31,16 +32,15 @@ namespace Duplicati.UnitTest { public class RepairWithMissingRemoteFiles : BasicSetupHelper { + // TODO: Add more tests to cover the following scenarios: + // Test where a single block in a large file is missing + run purge-broken-files + // Test that fetching remote sources for blocks is working + [Test] [Category("Targeted")] [TestCase(false, true, false)] - [TestCase(false, false, false)] - [TestCase(false, true, true)] - [TestCase(false, false, true)] [TestCase(true, true, false)] [TestCase(true, false, false)] - [TestCase(true, true, true)] - [TestCase(true, false, true)] public void TestRepairWithMissingDblocks(bool deleteDblockFiles, bool deleteIndexFiles, bool deleteDlistFiles) { var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true, number_of_retries = 0 }); @@ -86,13 +86,8 @@ namespace Duplicati.UnitTest [Test] [Category("Targeted")] [TestCase(false, true, false)] - [TestCase(false, false, false)] - [TestCase(false, true, true)] - [TestCase(false, false, true)] [TestCase(true, true, false)] [TestCase(true, false, false)] - [TestCase(true, true, true)] - [TestCase(true, false, true)] public void TestRepairWithMissingDblocksAndMultipleSources(bool deleteDblockFiles, bool deleteIndexFiles, bool deleteDlistFiles) { var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); @@ -140,13 +135,8 @@ namespace Duplicati.UnitTest [Test] [Category("Targeted")] [TestCase(false, true, false)] - [TestCase(false, false, false)] - [TestCase(false, true, true)] - [TestCase(false, false, true)] [TestCase(true, true, false)] [TestCase(true, false, false)] - [TestCase(true, true, true)] - [TestCase(true, false, true)] public void TestRepairWithDataFillingMultipleBlocklists(bool deleteDblockFiles, bool deleteIndexFiles, bool deleteDlistFiles) { var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); @@ -196,14 +186,8 @@ namespace Duplicati.UnitTest [Test] [Category("Targeted")] - [TestCase(false, true, false)] - [TestCase(false, false, false)] - [TestCase(false, true, true)] - [TestCase(false, false, true)] - [TestCase(true, true, false)] - [TestCase(true, false, false)] [TestCase(true, true, true)] - [TestCase(true, false, true)] + [TestCase(true, false, false)] public void TestRepairWithSmallFileOnly(bool deleteDblockFiles, bool deleteIndexFiles, bool deleteDlistFiles) { var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); @@ -248,9 +232,6 @@ namespace Duplicati.UnitTest [Test] [Category("Targeted")] [TestCase(false, true, false)] - [TestCase(false, false, false)] - [TestCase(false, true, true)] - [TestCase(false, false, true)] [TestCase(true, true, false)] [TestCase(true, false, false)] [TestCase(true, true, true)] @@ -295,18 +276,18 @@ namespace Duplicati.UnitTest TestUtils.AssertResults(c.Repair()); TestUtils.AssertResults(c.Test()); } + + // 4. Verify that restore works + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { restore_path = RESTOREFOLDER }), null)) + TestUtils.AssertResults(c.Restore(null)); + + TestUtils.AssertDirectoryTreesAreEquivalent(DATAFOLDER, RESTOREFOLDER, !Library.Utility.Utility.ParseBoolOption(testopts, "skip-metadata"), "Restore"); } [Test] [Category("Targeted")] - [TestCase(false, true, false)] - [TestCase(false, false, false)] - [TestCase(false, true, true)] - [TestCase(false, false, true)] - [TestCase(true, true, false)] - [TestCase(true, false, false)] [TestCase(true, true, true)] - [TestCase(true, false, true)] + [TestCase(true, false, false)] public void TestRepairWithDryRun(bool deleteDblockFiles, bool deleteIndexFiles, bool deleteDlistFiles) { var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); @@ -370,13 +351,280 @@ namespace Duplicati.UnitTest Assert.IsTrue(preSignature.indexVolumeLinks.All(x => postSignature.indexVolumeLinks.Contains(x)), "All index volume links should be present after dry run repair."); } - // TODO: Add more tests to cover the following scenarios: + [Test] + [Category("Targeted")] + public void TestRepairWithNoBlockAvailableFails() + { + var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); - // Test with deleted files as sources + run purge-broken-files - // Test with partial data available + run purge-broken-files - // Test where a single block in a large file is missing + run purge-broken-files - // Test that fetching remote sources for blocks is working + // 1. Make some shared data 1kb blocks, 32 bytes hash, 3 blocklists, and 1028 bytes extra + var data = new byte[1024 * 32 * 3 + 1028]; + Random.Shared.NextBytes(data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete files + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dblockFiles) + File.Delete(file); + + // Destroy source data + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), new byte[0]); + Directory.SetLastWriteTime(DATAFOLDER, DateTime.UtcNow.AddSeconds(1)); + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + var res = c.Repair(); + Assert.IsTrue(res.Errors.Where(x => x.Contains("purge-broken-files")).Count() == 1, "Repair should fail with missing block data."); + } + } + + [Test] + [Category("Targeted")] + public void TestPartialRepairPossibleWithMissingMetadata() + { + var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); + + // 1. Make some shared data 1kb blocks, 32 bytes hash, 3 blocklists, and 1028 bytes extra + var data = new byte[1024 * 32 * 3 + 1028]; + Random.Shared.NextBytes(data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete files + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dblockFiles) + File.Delete(file); + + // Destroy some source data + Directory.SetLastWriteTime(DATAFOLDER, DateTime.UtcNow.AddSeconds(1)); + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + var res = c.Repair(); + Assert.IsTrue(res.Warnings.Where(x => x.Contains("purge-broken-files")).Count() == 1, "Repair should warn about missing block data."); + + // Ensure that the partial repair did not clean up fully + Assert.Throws(() => c.Test()); + + // Purge broken files should be possible + TestUtils.AssertResults(c.PurgeBrokenFiles(null)); + TestUtils.AssertResults(c.Test(int.MaxValue)); + } + + // 4. Verify that restore works + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { restore_path = RESTOREFOLDER }), null)) + TestUtils.AssertResults(c.Restore(null)); + + TestUtils.AssertDirectoryTreesAreEquivalent(DATAFOLDER, RESTOREFOLDER, !Library.Utility.Utility.ParseBoolOption(testopts, "skip-metadata"), "Restore"); + } + + [Test] + [Category("Targeted")] + [TestCase(false)] + [TestCase(true)] + public void TestPartialRepairPossibleWithDeletedFiles(bool deleteLargeFile) + { + var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); + + // 1. Make some shared data 1kb blocks, 32 bytes hash, 3 blocklists, and 1028 bytes extra + var data = new byte[1024 * 32 * 3 + 1028]; + Random.Shared.NextBytes(data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "b"), data.AsSpan().Slice(0, 1024).ToArray()); + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete files + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dblockFiles) + File.Delete(file); + + // Destroy some source data (also destroys the metadata) + if (deleteLargeFile) + File.Delete(Path.Combine(DATAFOLDER, "a")); + else + File.Delete(Path.Combine(DATAFOLDER, "b")); + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + var res = c.Repair(); + Assert.IsTrue(res.Warnings.Where(x => x.Contains("purge-broken-files")).Count() == 1, "Repair should warn about missing block data."); + + // Ensure that the partial repair did not clean up fully + Assert.Throws(() => c.Test()); + + // Purge broken files should be possible + TestUtils.AssertResults(c.PurgeBrokenFiles(null)); + TestUtils.AssertResults(c.Test(int.MaxValue)); + + var files = c.List("*").Files.ToList(); + Console.WriteLine("Files: " + string.Join(", ", files.Select(x => x.Path))); + } + + // 4. Verify that restore works + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { restore_path = RESTOREFOLDER }), null)) + TestUtils.AssertResults(c.Restore(null)); + + Assert.That(File.Exists(Path.Combine(RESTOREFOLDER, "a")), Is.EqualTo(!deleteLargeFile), "Restored file 'a' should be restored."); + Assert.That(File.Exists(Path.Combine(RESTOREFOLDER, "b")), Is.EqualTo(true), "File 'b' should always be restored."); + if (!deleteLargeFile) + Assert.That(File.ReadAllBytes(Path.Combine(RESTOREFOLDER, "a")), Is.EqualTo(data), "Restored file 'a' should be the same as the original."); + else + Assert.That(File.ReadAllBytes(Path.Combine(RESTOREFOLDER, "b")), Is.EqualTo(data.AsSpan().Slice(0, 1024).ToArray()), "Restored file 'b' should be the same as the original."); + } + + [Test] + [Category("Targeted")] + [TestCase(0)] // Large file only + [TestCase(1)] // Small file only + [TestCase(2)] // Both files + [TestCase(3)] // Metadata only + public void TestPartialRepairPossibleWithPartialData(int scenario) + { + var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); + + // 1. Make some shared data 1kb blocks, 32 bytes hash, 3 blocklists, and 1028 bytes extra + var data = new byte[1024 * 32 * 3 + 1028]; + Random.Shared.NextBytes(data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "b"), data.AsSpan().Slice(0, 1024).ToArray()); + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete files + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToList(); + foreach (var file in dblockFiles) + File.Delete(file); + + var corruptLargeFile = scenario == 0 || scenario == 2; + var corruptSmallFile = scenario == 1 || scenario == 2; + var corruptBothFiles = corruptLargeFile && corruptSmallFile; + var destroyMetadata = scenario == 3; + + // Destroy some source data (keep metadata) + var prev = data[1]; + data[1] = 1; + if (corruptLargeFile) + { + var timestamp = File.GetLastWriteTime(Path.Combine(DATAFOLDER, "a")); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + File.SetLastWriteTime(Path.Combine(DATAFOLDER, "a"), timestamp); + } + + if (corruptSmallFile) + { + var timestamp = File.GetLastWriteTime(Path.Combine(DATAFOLDER, "b")); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "b"), data.AsSpan().Slice(0, 1024).ToArray()); + File.SetLastWriteTime(Path.Combine(DATAFOLDER, "b"), timestamp); + } + data[1] = prev; + + // Make sure that destroying the metadata does not destroy the files + if (destroyMetadata) + { + File.SetLastWriteTime(Path.Combine(DATAFOLDER, "a"), DateTime.UtcNow.AddSeconds(1)); + File.SetLastWriteTime(Path.Combine(DATAFOLDER, "b"), DateTime.UtcNow.AddSeconds(1.1)); + } + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + var res = c.Repair(); + + if (corruptBothFiles || destroyMetadata) + { + Assert.IsTrue(res.Warnings.Where(x => x.Contains("purge-broken-files")).Count() == 1, "Repair should warn about missing block data."); + + // Ensure that the partial repair did not clean up fully + Assert.Throws(() => c.Test()); + } + else + { + TestUtils.AssertResults(res); + TestUtils.AssertResults(c.Test(int.MaxValue)); + } + + // Purge broken files should be possible + TestUtils.AssertResults(c.PurgeBrokenFiles(null)); + TestUtils.AssertResults(c.Test(int.MaxValue)); + } + + // 4. Verify that restore works + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { restore_path = RESTOREFOLDER }), null)) + { + var res = c.Restore(null); + if (corruptBothFiles) + { + // We get a warning because neither file can be restored + Assert.That(res.Errors.Count(), Is.EqualTo(0), "Restore should not fail."); + } + else + { + TestUtils.AssertResults(res); + } + } + + Assert.That(File.Exists(Path.Combine(RESTOREFOLDER, "a")), Is.EqualTo(!corruptBothFiles), "File 'a' should be restored."); + Assert.That(File.Exists(Path.Combine(RESTOREFOLDER, "b")), Is.EqualTo(!corruptBothFiles), "File 'b' should be restored."); + if (!corruptLargeFile) + Assert.That(Convert.ToBase64String(File.ReadAllBytes(Path.Combine(RESTOREFOLDER, "a"))), Is.EqualTo(Convert.ToBase64String(data)), "Restored file 'a' should be the same as the original."); + else if (!corruptBothFiles) + Assert.That(Convert.ToBase64String(File.ReadAllBytes(Path.Combine(RESTOREFOLDER, "b"))), Is.EqualTo(Convert.ToBase64String(data.AsSpan().Slice(0, 1024))), "Restored file 'b' should be the same as the original."); + } + + [Test] + [Category("Targeted")] + [TestCase(1)] + [TestCase(5)] + [TestCase(10)] + public void TestRepairPossibleWithMultipleDblockFiles(int dblocksToDelete) + { + var testopts = TestOptions.Expand(new + { + blocksize = "1kb", + no_encryption = true, + rebuild_missing_dblock_files = true, + zip_compression_level = 0, + dblock_size = "10kb" + }); + + // 1. Make some shared data 1kb blocks, 32 bytes hash, 3 blocklists, and 1028 bytes extra + var data = new byte[1024 * 32 * 3 + 1028]; + Random.Shared.NextBytes(data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // 2. Delete files + var dblockFiles = Directory.GetFiles(TARGETFOLDER, "*.dblock.*", SearchOption.TopDirectoryOnly).ToArray(); + Random.Shared.Shuffle(dblockFiles); + foreach (var file in dblockFiles.Take(dblocksToDelete)) + File.Delete(file); + + // 3. Try to repair the remote destination + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + TestUtils.AssertResults(c.Repair()); + TestUtils.AssertResults(c.Test()); + } + + // 4. Verify that restore works + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { restore_path = RESTOREFOLDER }), null)) + TestUtils.AssertResults(c.Restore(null)); + + TestUtils.AssertDirectoryTreesAreEquivalent(DATAFOLDER, RESTOREFOLDER, !Library.Utility.Utility.ParseBoolOption(testopts, "skip-metadata"), "Restore"); + } } } From b947f7a7a0b093e4ae7e4677ca0a88adc4a7b6d2 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 9 May 2025 11:12:45 +0200 Subject: [PATCH 05/15] Simplified a query --- Duplicati/Library/Main/Database/LocalDatabase.cs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index ac3795f4f..acd07f918 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -469,17 +469,13 @@ namespace Duplicati.Library.Main.Database // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{volidstable}"" (""ID"" INTEGER PRIMARY KEY)")); - deletecmd.SetCommandAndParameters(FormatInvariant($@"INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") VALUES (@Id)")); - foreach (var name in names) - { - var volumeid = GetRemoteVolumeID(name, tr.Parent); - deletecmd.SetParameterValue("@Id", volumeid) - .ExecuteNonQuery(); - } + deletecmd.SetCommandAndParameters(FormatInvariant($@"INSERT OR IGNORE INTO ""{volidstable}"" SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@VolumeNames)")) + .ExpandInClauseParameter("@VolumeNames", names.ToArray()) + .ExecuteNonQuery(); + var volIdsSubQuery = FormatInvariant($@"SELECT ""ID"" FROM ""{volidstable}"" "); deletecmd.Parameters.Clear(); - var bsIdsSubQuery = FormatInvariant(@$" SELECT DISTINCT ""BlocksetEntry"".""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" IN ({volIdsSubQuery}) From 183ee39d6ec8019b35de7175b861e6ce874fc1e4 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 9 May 2025 11:13:10 +0200 Subject: [PATCH 06/15] Fixed a bug in list-changes --- Duplicati/Library/Main/Database/LocalListChangesDatabase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs index 32c24ee33..e334ffb3d 100644 --- a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs @@ -204,7 +204,7 @@ namespace Duplicati.Library.Main.Database var cmd = asNew ? m_insertCurrentElementCommand : m_insertPreviousElementCommand; cmd.SetParameterValue("@Path", path); cmd.SetParameterValue("@FileHash", filehash); - cmd.SetParameterValue(@"MetaHash", metahash); + cmd.SetParameterValue("@MetaHash", metahash); cmd.SetParameterValue("@Size", size); cmd.SetParameterValue("@Type", (int)type); cmd.ExecuteNonQuery(); From 51bcec3efc07d8a96bf82c4f8e1bb477118dcae4 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 9 May 2025 17:34:09 +0200 Subject: [PATCH 07/15] Style cleanup, named tuples. Fix indentation. Move metadata check up. --- .../Database/LocalListBrokenFilesDatabase.cs | 318 +++++++++--------- .../Main/Operation/ListBrokenFilesHandler.cs | 2 +- .../Main/Operation/PurgeBrokenFilesHandler.cs | 24 +- .../UnitTest/RepairWithMissingRemoteFiles.cs | 14 +- 4 files changed, 183 insertions(+), 175 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index b30a5fc17..8a2b6ad18 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -28,17 +28,17 @@ using System.Linq; namespace Duplicati.Library.Main.Database { - internal class LocalListBrokenFilesDatabase : LocalDatabase - { - private static readonly string BLOCK_VOLUME_IDS = FormatInvariant($@"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks.ToString()}'"); + internal class LocalListBrokenFilesDatabase : LocalDatabase + { + private static readonly string BLOCK_VOLUME_IDS = FormatInvariant($@"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks.ToString()}'"); - // Invalid blocksets include those that: - // - Have BlocksetEntries with unknown/invalid blocks (meaning the data to rebuild the blockset isn't available) - // - Invalid blocks include those that appear to be in non-Blocks volumes (e.g., are listed as being in an Index or Files volume) or that appear in an unknown volume (-1) - // - Have BlocklistHash entries with unknown/invalid blocks (meaning the data which defines the list of hashes that makes up the blockset isn't available) - // - Are defined in the Blockset table but have no entries in the BlocksetEntries table (this can happen during recreate if Files volumes reference blocksets that are not found in any Index files) - // However, blocksets with a length of 0 are excluded from this check, as the corresponding blocks for these are not needed. - private static readonly string BROKEN_FILE_IDS = FormatInvariant($@" + // Invalid blocksets include those that: + // - Have BlocksetEntries with unknown/invalid blocks (meaning the data to rebuild the blockset isn't available) + // - Invalid blocks include those that appear to be in non-Blocks volumes (e.g., are listed as being in an Index or Files volume) or that appear in an unknown volume (-1) + // - Have BlocklistHash entries with unknown/invalid blocks (meaning the data which defines the list of hashes that makes up the blockset isn't available) + // - Are defined in the Blockset table but have no entries in the BlocksetEntries table (this can happen during recreate if Files volumes reference blocksets that are not found in any Index files) + // However, blocksets with a length of 0 are excluded from this check, as the corresponding blocks for these are not needed. + private static readonly string BROKEN_FILE_IDS = FormatInvariant($@" SELECT DISTINCT ""ID"" FROM ( SELECT ""ID"" AS ""ID"", ""BlocksetID"" AS ""BlocksetID"" FROM ""FileLookup"" WHERE ""BlocksetID"" != {FOLDER_BLOCKSET_ID} AND ""BlocksetID"" != {SYMLINK_BLOCKSET_ID} UNION @@ -61,123 +61,123 @@ WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN WHERE ""BlocksetID"" NOT IN (SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" == 0) ) "); - private static readonly string BROKEN_FILE_SETS = FormatInvariant($@"SELECT DISTINCT ""B"".""Timestamp"", ""A"".""FilesetID"", COUNT(""A"".""FileID"") AS ""FileCount"" FROM ""FilesetEntry"" A, ""Fileset"" B WHERE ""A"".""FilesetID"" = ""B"".""ID"" AND ""A"".""FileID"" IN ({BROKEN_FILE_IDS})"); + private static readonly string BROKEN_FILE_SETS = FormatInvariant($@"SELECT DISTINCT ""B"".""Timestamp"", ""A"".""FilesetID"", COUNT(""A"".""FileID"") AS ""FileCount"" FROM ""FilesetEntry"" A, ""Fileset"" B WHERE ""A"".""FilesetID"" = ""B"".""ID"" AND ""A"".""FileID"" IN ({BROKEN_FILE_IDS})"); - private static readonly string BROKEN_FILE_NAMES = FormatInvariant($@"SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A LEFT JOIN ""Blockset"" B ON (""A"".""BlocksetID"" = ""B"".""ID"") WHERE ""A"".""ID"" IN ({BROKEN_FILE_IDS}) AND ""A"".""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId)"); + private static readonly string BROKEN_FILE_NAMES = FormatInvariant($@"SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A LEFT JOIN ""Blockset"" B ON (""A"".""BlocksetID"" = ""B"".""ID"") WHERE ""A"".""ID"" IN ({BROKEN_FILE_IDS}) AND ""A"".""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId)"); - private static string INSERT_BROKEN_IDS(string tablename, string IDfieldname) => FormatInvariant($@"INSERT INTO ""{tablename}"" (""{IDfieldname}"") {BROKEN_FILE_IDS} AND ""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId)"); + private static string INSERT_BROKEN_IDS(string tablename, string IDfieldname) => FormatInvariant($@"INSERT INTO ""{tablename}"" (""{IDfieldname}"") {BROKEN_FILE_IDS} AND ""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId)"); - public LocalListBrokenFilesDatabase(string path, long pagecachesize) - : base(path, "ListBrokenFiles", false, pagecachesize) - { - ShouldCloseConnection = true; - } + public LocalListBrokenFilesDatabase(string path, long pagecachesize) + : base(path, "ListBrokenFiles", false, pagecachesize) + { + ShouldCloseConnection = true; + } - public LocalListBrokenFilesDatabase(LocalDatabase parent) - : base(parent) - { - ShouldCloseConnection = false; - } + public LocalListBrokenFilesDatabase(LocalDatabase parent) + : base(parent) + { + ShouldCloseConnection = false; + } - public IEnumerable> GetBrokenFilesets(DateTime time, long[] versions, IDbTransaction transaction) - { - var query = BROKEN_FILE_SETS; - var clause = GetFilelistWhereClause(time, versions); - if (!string.IsNullOrWhiteSpace(clause.Item1)) - query += FormatInvariant($@" AND ""A"".""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" {clause.Item1})"); + public IEnumerable<(DateTime FilesetTime, long FilesetID, long RemoveFileCount)> GetBrokenFilesets(DateTime time, long[] versions, IDbTransaction transaction) + { + var query = BROKEN_FILE_SETS; + var clause = GetFilelistWhereClause(time, versions); + if (!string.IsNullOrWhiteSpace(clause.Item1)) + query += FormatInvariant($@" AND ""A"".""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" {clause.Item1})"); - query += @" GROUP BY ""A"".""FilesetID"""; + query += @" GROUP BY ""A"".""FilesetID"""; - using (var cmd = Connection.CreateCommand(transaction)) - foreach (var rd in cmd.SetCommandAndParameters(query).SetParameterValues(clause.Values).ExecuteReaderEnumerable()) - if (!rd.IsDBNull(0)) - yield return new Tuple(ParseFromEpochSeconds(rd.ConvertValueToInt64(0, 0)), rd.ConvertValueToInt64(1, -1), rd.ConvertValueToInt64(2, 0)); - } + using (var cmd = Connection.CreateCommand(transaction)) + foreach (var rd in cmd.SetCommandAndParameters(query).SetParameterValues(clause.Values).ExecuteReaderEnumerable()) + if (!rd.IsDBNull(0)) + yield return (ParseFromEpochSeconds(rd.ConvertValueToInt64(0, 0)), rd.ConvertValueToInt64(1, -1), rd.ConvertValueToInt64(2, 0)); + } - public IEnumerable> GetBrokenFilenames(long filesetid, IDbTransaction transaction) - { - using (var cmd = Connection.CreateCommand(transaction, BROKEN_FILE_NAMES).SetParameterValue("@FilesetId", filesetid)) - foreach (var rd in cmd.ExecuteReaderEnumerable()) - if (!rd.IsDBNull(0)) - yield return new Tuple(rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), rd.ConvertValueToInt64(1)); - } + public IEnumerable> GetBrokenFilenames(long filesetid, IDbTransaction transaction) + { + using (var cmd = Connection.CreateCommand(transaction, BROKEN_FILE_NAMES).SetParameterValue("@FilesetId", filesetid)) + foreach (var rd in cmd.ExecuteReaderEnumerable()) + if (!rd.IsDBNull(0)) + yield return new Tuple(rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), rd.ConvertValueToInt64(1)); + } - /// - /// Returns all index files that are orphaned, i.e., not referenced by any block files. - /// - /// Transaction to use for the query. - /// >All index files that are orphaned. - public IEnumerable GetOrphanedIndexFiles(IDbTransaction transaction) - { - using var cmd = Connection.CreateCommand(transaction, FormatInvariant($@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Index.ToString()}' AND ""ID"" NOT IN (SELECT ""IndexVolumeID"" FROM ""IndexBlockLink"")")); + /// + /// Returns all index files that are orphaned, i.e., not referenced by any block files. + /// + /// Transaction to use for the query. + /// >All index files that are orphaned. + public IEnumerable GetOrphanedIndexFiles(IDbTransaction transaction) + { + using var cmd = Connection.CreateCommand(transaction, FormatInvariant($@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Index.ToString()}' AND ""ID"" NOT IN (SELECT ""IndexVolumeID"" FROM ""IndexBlockLink"")")); - foreach (var rd in cmd.ExecuteReaderEnumerable()) - yield return new RemoteVolume( - rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), - rd.ConvertValueToString(1) ?? throw new Exception("Hash was null"), - rd.ConvertValueToInt64(2, -1) - ); - } + foreach (var rd in cmd.ExecuteReaderEnumerable()) + yield return new RemoteVolume( + rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), + rd.ConvertValueToString(1) ?? throw new Exception("Hash was null"), + rd.ConvertValueToInt64(2, -1) + ); + } - /// - /// Inserts the broken file IDs into the given table. The table must have a single column with the same name as the ID field name. - /// - /// The filset id for the current operation - /// The name of the table to insert into - /// The name of the ID field in the table - /// The transaction to use for the query - public void InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname, IDbTransaction transaction) - { - using var cmd = Connection.CreateCommand(transaction, INSERT_BROKEN_IDS(tablename, IDfieldname)) - .SetParameterValue("@FilesetId", filesetid); - cmd.ExecuteNonQuery(); - } + /// + /// Inserts the broken file IDs into the given table. The table must have a single column with the same name as the ID field name. + /// + /// The filset id for the current operation + /// The name of the table to insert into + /// The name of the ID field in the table + /// The transaction to use for the query + public void InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname, IDbTransaction transaction) + { + using var cmd = Connection.CreateCommand(transaction, INSERT_BROKEN_IDS(tablename, IDfieldname)) + .SetParameterValue("@FilesetId", filesetid); + cmd.ExecuteNonQuery(); + } - /// - /// Returns the ID of an empty metadata blockset. If no empty blockset is found, it returns the ID of the smallest blockset that is not in the given block volume IDs. - /// If no such blockset is found, it returns -1. - /// - /// The volume ids to ignore when searching for a suitable metadata block - /// The hash of the empty blockset - /// The size of the empty blockset - /// The transaction to use for the query - /// The ID of the empty metadata blockset, or -1 if no suitable blockset is found - public long GetEmptyMetadataBlocksetId(IEnumerable blockVolumeIds, string emptyHash, long emptyHashSize, IDbTransaction transaction) - { - using var cmd = Connection.CreateCommand(transaction, @"SELECT ""ID"" FROM ""Blockset"" WHERE ""FullHash"" = @EmptyHash AND ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))") - .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) - .SetParameterValue("@EmptyHash", emptyHash) - .SetParameterValue("@EmptyHashSize", emptyHashSize); + /// + /// Returns the ID of an empty metadata blockset. If no empty blockset is found, it returns the ID of the smallest blockset that is not in the given block volume IDs. + /// If no such blockset is found, it returns -1. + /// + /// The volume ids to ignore when searching for a suitable metadata block + /// The hash of the empty blockset + /// The size of the empty blockset + /// The transaction to use for the query + /// The ID of the empty metadata blockset, or -1 if no suitable blockset is found + public long GetEmptyMetadataBlocksetId(IEnumerable blockVolumeIds, string emptyHash, long emptyHashSize, IDbTransaction transaction) + { + using var cmd = Connection.CreateCommand(transaction, @"SELECT ""ID"" FROM ""Blockset"" WHERE ""FullHash"" = @EmptyHash AND ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))") + .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) + .SetParameterValue("@EmptyHash", emptyHash) + .SetParameterValue("@EmptyHashSize", emptyHashSize); - var res = cmd.ExecuteScalarInt64(-1); + var res = cmd.ExecuteScalarInt64(-1); - // No empty block found, try to find a zero-length block instead - if (res < 0 && emptyHashSize != 0) - res = cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))") - .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) - .SetParameterValue("@EmptyHashSize", 0) - .ExecuteScalarInt64(-1); + // No empty block found, try to find a zero-length block instead + if (res < 0 && emptyHashSize != 0) + res = cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))") + .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) + .SetParameterValue("@EmptyHashSize", 0) + .ExecuteScalarInt64(-1); - // No empty block found, pick the smallest one - if (res < 0) - res = cmd.SetCommandAndParameters(@"SELECT ""Blockset"".""ID"" FROM ""BlocksetEntry"", ""Blockset"", ""Metadataset"", ""Block"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) ORDER BY ""Blockset"".""Length"" ASC LIMIT 1") - .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) - .ExecuteScalarInt64(-1); + // No empty block found, pick the smallest one + if (res < 0) + res = cmd.SetCommandAndParameters(@"SELECT ""Blockset"".""ID"" FROM ""BlocksetEntry"", ""Blockset"", ""Metadataset"", ""Block"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) ORDER BY ""Blockset"".""Length"" ASC LIMIT 1") + .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) + .ExecuteScalarInt64(-1); - return res; - } + return res; + } - /// - /// Replaces the metadata blockset ID in the Metadataset table with the empty blockset ID for all fileset entries that are not in any block volume. - /// This is used to clean up the metadata blocksets that are now missing. - /// - /// The filesetId to target - /// The empty blockset ID to replace with - /// The transaction to use for the query - /// The number of rows affected - public int ReplaceMetadata(long filesetId, long emptyBlocksetId, IDbTransaction transaction) - { - using var cmd = m_connection.CreateCommand(transaction, @" + /// + /// Replaces the metadata blockset ID in the Metadataset table with the empty blockset ID for all fileset entries that are not in any block volume. + /// This is used to clean up the metadata blocksets that are now missing. + /// + /// The filesetId to target + /// The empty blockset ID to replace with + /// The transaction to use for the query + /// The number of rows affected + public int ReplaceMetadata(long filesetId, long emptyBlocksetId, IDbTransaction transaction) + { + using var cmd = m_connection.CreateCommand(transaction, @" UPDATE ""Metadataset"" SET ""BlocksetID"" = @EmptyBlocksetID WHERE @@ -193,56 +193,56 @@ WHERE FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" )") - .SetParameterValue("@EmptyBlocksetId", emptyBlocksetId) - .SetParameterValue("@FilesetId", filesetId); - return cmd.ExecuteNonQuery(); - } - - public void RemoveMissingBlocks(IEnumerable names, IDbTransaction transaction) - { - if (names == null || !names.Any()) return; - if (transaction == null) - throw new Exception("This function cannot be called when not in a transaction, as it leaves the database in an inconsistent state"); - - using (var deletecmd = m_connection.CreateCommand(transaction)) - { - var temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var volidstable = "DelVolSetIds-" + temptransguid; - - // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. - deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{volidstable}"" (""ID"" INTEGER PRIMARY KEY)")); - - using (var tmptable = new TemporaryDbValueList(m_connection, transaction, names)) - deletecmd.SetCommandAndParameters(FormatInvariant($@"INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)")) - .ExpandInClauseParameter("@Names", tmptable) - .ExecuteNonQuery(); - - var volIdsSubQuery = FormatInvariant($@"SELECT ""ID"" FROM ""{volidstable}"" "); - deletecmd.Parameters.Clear(); - - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); - - // Clean up temp tables for subqueries. We truncate content and then try to delete. - // Drop in try-block, as it fails in nested transactions (SQLite problem) - // SQLite.SQLiteException (0x80004005): database table is locked - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{volidstable}"" ")); - try - { - deletecmd.CommandTimeout = 2; - deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{volidstable}"" ")); + .SetParameterValue("@EmptyBlocksetId", emptyBlocksetId) + .SetParameterValue("@FilesetId", filesetId); + return cmd.ExecuteNonQuery(); } - catch { /* Ignore, will be deleted on close anyway. */ } - } - } - public long GetFilesetFileCount(long filesetid, IDbTransaction transaction) - { - using var cmd = m_connection.CreateCommand(transaction, @"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId") - .SetParameterValue("@FilesetId", filesetid); - return cmd.ExecuteScalarInt64(0); + public void RemoveMissingBlocks(IEnumerable names, IDbTransaction transaction) + { + if (names == null || !names.Any()) return; + if (transaction == null) + throw new Exception("This function cannot be called when not in a transaction, as it leaves the database in an inconsistent state"); + + using (var deletecmd = m_connection.CreateCommand(transaction)) + { + var temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + var volidstable = "DelVolSetIds-" + temptransguid; + + // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. + deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{volidstable}"" (""ID"" INTEGER PRIMARY KEY)")); + + using (var tmptable = new TemporaryDbValueList(m_connection, transaction, names)) + deletecmd.SetCommandAndParameters(FormatInvariant($@"INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)")) + .ExpandInClauseParameter("@Names", tmptable) + .ExecuteNonQuery(); + + var volIdsSubQuery = FormatInvariant($@"SELECT ""ID"" FROM ""{volidstable}"" "); + deletecmd.Parameters.Clear(); + + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery})")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); + + // Clean up temp tables for subqueries. We truncate content and then try to delete. + // Drop in try-block, as it fails in nested transactions (SQLite problem) + // SQLite.SQLiteException (0x80004005): database table is locked + deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{volidstable}"" ")); + try + { + deletecmd.CommandTimeout = 2; + deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{volidstable}"" ")); + } + catch { /* Ignore, will be deleted on close anyway. */ } + } + } + + public long GetFilesetFileCount(long filesetid, IDbTransaction transaction) + { + using var cmd = m_connection.CreateCommand(transaction, @"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId") + .SetParameterValue("@FilesetId", filesetid); + return cmd.ExecuteScalarInt64(0); + } } - } } diff --git a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs index 8eab6aa3d..dcca28a80 100644 --- a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs @@ -55,7 +55,7 @@ namespace Duplicati.Library.Main.Operation await DoRunAsync(backendManager, db, tr, filter, callbackhandler).ConfigureAwait(false); } - public static async Task<(Tuple[], List Missing)> GetBrokenFilesetsFromRemote(IBackendManager backendManager, BasicResults result, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Options options) + public static async Task<((DateTime FilesetTime, long FilesetID, long RemoveCount)[], List Missing)> GetBrokenFilesetsFromRemote(IBackendManager backendManager, BasicResults result, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Options options) { List missing = null; var brokensets = db.GetBrokenFilesets(options.Time, options.Version, transaction).ToArray(); diff --git a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs index 350ccb157..d17cb5782 100644 --- a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs @@ -88,9 +88,16 @@ namespace Duplicati.Library.Main.Operation SetCount = db.GetFilesetFileCount(x.Item2, tr) }).ToArray(); - if (m_options.Dryrun) - tr.Rollback(); - else + var emptyBlocksetId = -1L; + if (!m_options.DisableReplaceMissingMetadata) + { + var emptymetadata = Utility.WrapMetadata(new Dictionary(), m_options); + emptyBlocksetId = db.GetEmptyMetadataBlocksetId(missing.Select(x => x.ID), emptymetadata.FileHash, emptymetadata.Blob.Length, null); + if (emptyBlocksetId < 0) + throw new UserInformationException($"Failed to locate an empty metadata blockset to replace missing metadata. Set the option --disable-replace-missing-metadata=true to ignore this and drop files with missing metadata.", "FailedToLocateEmptyMetadataBlockset"); + } + + if (!m_options.Dryrun) tr.Commit(); var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray(); @@ -104,7 +111,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing {0} filesets where all file(s) are broken: {1}", fully_emptied.Length, string.Join(", ", fully_emptied.Select(x => x.Timestamp.ToLocalTime().ToString()))); m_result.DeleteResults = new DeleteResults(m_result); - using (var rmdb = new Database.LocalDeleteDatabase(db)) + using (var rmdb = new LocalDeleteDatabase(db)) using (var deltr = new ReusableTransaction(rmdb)) { var opts = new Options(new Dictionary(m_options.RawOptions)); @@ -123,15 +130,6 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(pgoffset); } - var emptyBlocksetId = -1L; - if (!m_options.DisableReplaceMissingMetadata) - { - var emptymetadata = Utility.WrapMetadata(new Dictionary(), m_options); - emptyBlocksetId = db.GetEmptyMetadataBlocksetId(missing.Select(x => x.ID), emptymetadata.FileHash, emptymetadata.Blob.Length, null); - if (emptyBlocksetId < 0) - throw new UserInformationException($"Failed to locate an empty metadata blockset to replace missing metadata. Set the option --disable-replace-missing-metadata=true to ignore this and drop files with missing metadata.", "FailedToLocateEmptyMetadataBlockset"); - } - if (to_purge.Length > 0) { m_result.PurgeResults = new PurgeFilesResults(m_result); diff --git a/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs b/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs index 4507036f1..827f23fc3 100644 --- a/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs +++ b/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs @@ -386,7 +386,12 @@ namespace Duplicati.UnitTest [Category("Targeted")] public void TestPartialRepairPossibleWithMissingMetadata() { - var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); + var testopts = TestOptions.Expand(new + { + blocksize = "1kb", + no_encryption = true, + rebuild_missing_dblock_files = true + }); // 1. Make some shared data 1kb blocks, 32 bytes hash, 3 blocklists, and 1028 bytes extra var data = new byte[1024 * 32 * 3 + 1028]; @@ -431,7 +436,12 @@ namespace Duplicati.UnitTest [TestCase(true)] public void TestPartialRepairPossibleWithDeletedFiles(bool deleteLargeFile) { - var testopts = TestOptions.Expand(new { blocksize = "1kb", no_encryption = true, rebuild_missing_dblock_files = true }); + var testopts = TestOptions.Expand(new + { + blocksize = "1kb", + no_encryption = true, + rebuild_missing_dblock_files = true + }); // 1. Make some shared data 1kb blocks, 32 bytes hash, 3 blocklists, and 1028 bytes extra var data = new byte[1024 * 32 * 3 + 1028]; From ed9e6c2bb0318eb6fc89fadeb5ece4af882e17fc Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 9 May 2025 17:34:20 +0200 Subject: [PATCH 08/15] Fixed logic error --- Duplicati/Library/Main/Operation/RepairHandler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 56f5b91fe..654840372 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -935,7 +935,7 @@ namespace Duplicati.Library.Main.Operation { var isFile = File.Exists(block.Path); var isDir = Directory.Exists(block.Path); - if (!isFile || !isDir) + if (!isFile && !isDir) { Logging.Log.WriteVerboseMessage(LOGTAG, "EntryNotFound", null, "Entry not found: {0}", block.Path); continue; From 91cfdebd95f0b8ae3c793f95aa9895e36dbdd716 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 10 May 2025 09:12:11 +0200 Subject: [PATCH 09/15] Fixed issue with repair leaving old filesets in the database. Fixed issue with the full/partial state not being replicated for re-created filesets --- .../Library/Main/Database/LocalRepairDatabase.cs | 14 +++++++++++++- Duplicati/Library/Main/Operation/RepairHandler.cs | 12 ++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index 916cb2457..710f55eda 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -63,7 +63,7 @@ namespace Duplicati.Library.Main.Database if (!rd.Read()) throw new Exception($"No such remote file: {filelist}"); - return (rd.ConvertValueToInt64(0, -1), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime(), rd.GetInt32(0) == BackupType.FULL_BACKUP); + return (rd.ConvertValueToInt64(0, -1), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime(), rd.GetInt32(2) == BackupType.FULL_BACKUP); } } @@ -86,7 +86,19 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@CurrentFilesetId", filesetid) .SetParameterValue("@PreviousFilesetId", prevFilesetId) .ExecuteNonQuery(); + } + /// + /// Deletes a fileset from the database + /// + /// The fileset ID + /// The transaction + public void DeleteFileset(long filesetid, IDbTransaction transaction) + { + using var cmd = m_connection.CreateCommand(transaction); + cmd.SetCommandAndParameters(@"DELETE FROM ""Fileset"" WHERE ""ID"" = @FilesetId") + .SetParameterValue("@FilesetId", filesetid) + .ExecuteNonQuery(); } /// diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 654840372..b7f33312a 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -535,19 +535,23 @@ namespace Duplicati.Library.Main.Operation volumeWriter.CreateFilesetFile(isPrevFull); var newFilesetID = db.CreateFileset(volumeWriter.VolumeID, filesetTime, rtr.Transaction); + db.UpdateFullBackupStateInFileset(newFilesetID, isPrevFull); + db.LinkFilesetToVolume(newFilesetID, volumeWriter.VolumeID, rtr.Transaction); db.MoveFilesFromFileset(newFilesetID, prevFilesetId, rtr.Transaction); db.WriteFileset(volumeWriter, newFilesetID, rtr.Transaction); volumeWriter.Close(); + db.UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null, rtr.Transaction); + db.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleting, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); + db.DeleteFileset(prevFilesetId, rtr.Transaction); + await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + if (m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would re-upload fileset {0}, with size {1}, previous size {2}", originalVolume.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(volumeWriter.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(originalVolume.Size)); + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would upload fileset {0}, with size {1}, previous size {2}", originalVolume.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(volumeWriter.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(originalVolume.Size)); else { - db.UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null, rtr.Transaction); - db.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleting, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); - await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) rtr.Commit("CommitPriorToFilesetUpload"); await backendManager.PutAsync(volumeWriter, null, null, false, null, cancellationToken).ConfigureAwait(false); From 48ad8b8688deb9e3a096775c52eeca94d4356efb Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 10 May 2025 09:12:34 +0200 Subject: [PATCH 10/15] Added a guard against full-removal which could leave the database inconsistent --- .../Library/Main/Operation/PurgeBrokenFilesHandler.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs index d17cb5782..491958fee 100644 --- a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs @@ -97,11 +97,15 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException($"Failed to locate an empty metadata blockset to replace missing metadata. Set the option --disable-replace-missing-metadata=true to ignore this and drop files with missing metadata.", "FailedToLocateEmptyMetadataBlockset"); } + var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray(); + var to_purge = compare_list.Where(x => x.RemoveCount != x.SetCount).ToArray(); + + if (fully_emptied.Length == db.FilesetTimes.Count()) + throw new UserInformationException("All filesets are fully broken and needs to be removed. To avoid unexpected deletions, you must manually remove the remote files and delete the database.", "AllFilesetsBroken"); + if (!m_options.Dryrun) tr.Commit(); - var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray(); - var to_purge = compare_list.Where(x => x.RemoveCount != x.SetCount).ToArray(); if (fully_emptied.Length != 0) { From 2b364d8ca1bca4e36871b47e8971a09622dd24d3 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 10 May 2025 09:47:50 +0200 Subject: [PATCH 11/15] Fixed wrong assumption in tests --- Duplicati/UnitTest/RepairHandlerTests.cs | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/Duplicati/UnitTest/RepairHandlerTests.cs b/Duplicati/UnitTest/RepairHandlerTests.cs index 533b67d06..6b1f560b0 100644 --- a/Duplicati/UnitTest/RepairHandlerTests.cs +++ b/Duplicati/UnitTest/RepairHandlerTests.cs @@ -164,7 +164,7 @@ namespace Duplicati.UnitTest Assert.AreEqual(0, backupResults.Warnings.Count()); } - var dindexFiles = Directory.EnumerateFiles(this.TARGETFOLDER, "*dindex*").ToArray(); + var dindexFiles = Directory.EnumerateFiles(this.TARGETFOLDER, "*.dindex.*").ToArray(); Assert.Greater(dindexFiles.Length, 0); foreach (var f in dindexFiles) { @@ -178,10 +178,8 @@ namespace Duplicati.UnitTest Assert.AreEqual(0, repairResults.Warnings.Count()); } - foreach (var file in dindexFiles) - { - Assert.IsTrue(File.Exists(Path.Combine(this.TARGETFOLDER, file))); - } + var recreatedIndexFiles = Directory.EnumerateFiles(this.TARGETFOLDER, "*dindex*").ToArray(); + Assert.AreEqual(dindexFiles.Length, recreatedIndexFiles.Length); } [Test] @@ -216,7 +214,7 @@ namespace Duplicati.UnitTest Assert.AreEqual(0, backupResults.Warnings.Count()); } - var dindexFiles = Directory.EnumerateFiles(this.TARGETFOLDER, "*dindex*").ToArray(); + var dindexFiles = Directory.EnumerateFiles(this.TARGETFOLDER, "*.dindex.*").ToArray(); Assert.Greater(dindexFiles.Length, 0); foreach (var f in dindexFiles) { @@ -230,10 +228,8 @@ namespace Duplicati.UnitTest Assert.AreEqual(0, repairResults.Warnings.Count()); } - foreach (var file in dindexFiles) - { - Assert.IsTrue(File.Exists(Path.Combine(this.TARGETFOLDER, file))); - } + var recreatedIndexFiles = Directory.EnumerateFiles(this.TARGETFOLDER, "*dindex*").ToArray(); + Assert.AreEqual(dindexFiles.Length, recreatedIndexFiles.Length); } [Test] From 33a21c782da03e718c33f13424d7549152c2ed27 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 10 May 2025 09:55:13 +0200 Subject: [PATCH 12/15] Fixed incorrect transaction use --- .../Library/Main/Database/LocalRepairDatabase.cs | 15 ++++++++------- Duplicati/Library/Main/Operation/RepairHandler.cs | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index 710f55eda..758c084b1 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -876,13 +876,12 @@ namespace Duplicati.Library.Main.Database } } - public void CheckAllBlocksAreInVolume(string filename, IEnumerable> blocks) + public void CheckAllBlocksAreInVolume(string filename, IEnumerable> blocks, IDbTransaction transaction) { - using (var tr = m_connection.BeginTransactionSafe()) - using (var cmd = m_connection.CreateCommand(tr)) + using var cmd = m_connection.CreateCommand(transaction); + var tablename = "ProbeBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + try { - var tablename = "ProbeBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)")); cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{tablename}"" (""Hash"", ""Size"") VALUES (@Hash, @Size)")); @@ -901,11 +900,13 @@ namespace Duplicati.Library.Main.Database .SetParameterValue("@VolumeId", id) .ExecuteScalarInt64(0); - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tablename}"" ")); - if (aliens != 0) throw new Exception($"Not all blocks were found in {filename}"); } + finally + { + cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tablename}"" ")); + } } public void CheckBlocklistCorrect(string hash, long length, IEnumerable blocklist, long blocksize, long blockhashlength, IDbTransaction? transaction) diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index b7f33312a..1e1b75590 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -267,7 +267,7 @@ namespace Duplicati.Library.Main.Operation if (entry.Hash != rv.Hash || entry.Size != rv.Length || !new[] { RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified }.Contains(entry.State)) throw new Exception(string.Format("Volume {0} hash/size mismatch ({1} - {2}) vs ({3} - {4})", rv.Filename, entry.Hash, entry.Size, rv.Hash, rv.Length)); - db.CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks); + db.CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks, rtr.Transaction); } var blocksize = m_options.Blocksize; From 513e67418d2f7924da7e6b789f7dd496c04f32a8 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 10 May 2025 22:12:45 +0200 Subject: [PATCH 13/15] Ignore filesets that are in the deleting state --- Duplicati/Library/Main/Database/LocalDatabase.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index acd07f918..89b4cf15c 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -897,17 +897,17 @@ ON if (!laxVerifyForRepair) { - var filesetsMissingVolumes = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State)") + var filesetsMissingVolumes = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" NOT IN (@States))") .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()) + .ExpandInClauseParameter("@States", [RemoteVolumeState.Deleted.ToString(), RemoteVolumeState.Deleting.ToString()]) .ExecuteScalarInt64(0); if (filesetsMissingVolumes != 0) { if (filesetsMissingVolumes == 1) - using (var reader = cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Timestamp"", ""VolumeID"" FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State)") + using (var reader = cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Timestamp"", ""VolumeID"" FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" NOT IN (@States))") .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()) + .ExpandInClauseParameter("@States", [RemoteVolumeState.Deleted.ToString(), RemoteVolumeState.Deleting.ToString()]) .ExecuteReader()) if (reader.Read()) throw new DatabaseInconsistencyException($"Detected 1 fileset with missing volume: FilesetId = {reader.ConvertValueToInt64(0)}, Time = ({ParseFromEpochSeconds(reader.ConvertValueToInt64(1))}), unmatched VolumeID {reader.ConvertValueToInt64(2)}"); @@ -915,16 +915,16 @@ ON throw new DatabaseInconsistencyException($"Detected {filesetsMissingVolumes} filesets with missing volumes"); } - var volumesMissingFilests = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")") + var volumesMissingFilests = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" NOT IN (@States) AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")") .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()) + .ExpandInClauseParameter("@States", [RemoteVolumeState.Deleted.ToString(), RemoteVolumeState.Deleting.ToString()]) .ExecuteScalarInt64(0); if (volumesMissingFilests != 0) { if (volumesMissingFilests == 1) - using (var reader = cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Name"", ""State"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")") + using (var reader = cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Name"", ""State"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" NOT IN (@States) AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")") .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()) + .ExpandInClauseParameter("@States", [RemoteVolumeState.Deleted.ToString(), RemoteVolumeState.Deleting.ToString()]) .ExecuteReader()) if (reader.Read()) throw new DatabaseInconsistencyException($"Detected 1 volume with missing filesets: VolumeId = {reader.ConvertValueToInt64(0)}, Name = {reader.ConvertValueToString(1)}, State = {reader.ConvertValueToString(2)}"); From 547ad8437cbf3f016c676ddda1bd726ed9c9aecb Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 11 May 2025 15:39:27 +0200 Subject: [PATCH 14/15] Revert "Ignore filesets that are in the deleting state" This reverts commit 513e67418d2f7924da7e6b789f7dd496c04f32a8. --- Duplicati/Library/Main/Database/LocalDatabase.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 89b4cf15c..acd07f918 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -897,17 +897,17 @@ ON if (!laxVerifyForRepair) { - var filesetsMissingVolumes = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" NOT IN (@States))") + var filesetsMissingVolumes = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State)") .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .ExpandInClauseParameter("@States", [RemoteVolumeState.Deleted.ToString(), RemoteVolumeState.Deleting.ToString()]) + .SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()) .ExecuteScalarInt64(0); if (filesetsMissingVolumes != 0) { if (filesetsMissingVolumes == 1) - using (var reader = cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Timestamp"", ""VolumeID"" FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" NOT IN (@States))") + using (var reader = cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Timestamp"", ""VolumeID"" FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State)") .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .ExpandInClauseParameter("@States", [RemoteVolumeState.Deleted.ToString(), RemoteVolumeState.Deleting.ToString()]) + .SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()) .ExecuteReader()) if (reader.Read()) throw new DatabaseInconsistencyException($"Detected 1 fileset with missing volume: FilesetId = {reader.ConvertValueToInt64(0)}, Time = ({ParseFromEpochSeconds(reader.ConvertValueToInt64(1))}), unmatched VolumeID {reader.ConvertValueToInt64(2)}"); @@ -915,16 +915,16 @@ ON throw new DatabaseInconsistencyException($"Detected {filesetsMissingVolumes} filesets with missing volumes"); } - var volumesMissingFilests = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" NOT IN (@States) AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")") + var volumesMissingFilests = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")") .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .ExpandInClauseParameter("@States", [RemoteVolumeState.Deleted.ToString(), RemoteVolumeState.Deleting.ToString()]) + .SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()) .ExecuteScalarInt64(0); if (volumesMissingFilests != 0) { if (volumesMissingFilests == 1) - using (var reader = cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Name"", ""State"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" NOT IN (@States) AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")") + using (var reader = cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Name"", ""State"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")") .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .ExpandInClauseParameter("@States", [RemoteVolumeState.Deleted.ToString(), RemoteVolumeState.Deleting.ToString()]) + .SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()) .ExecuteReader()) if (reader.Read()) throw new DatabaseInconsistencyException($"Detected 1 volume with missing filesets: VolumeId = {reader.ConvertValueToInt64(0)}, Name = {reader.ConvertValueToString(1)}, State = {reader.ConvertValueToString(2)}"); From b6978afb3a022ac0f9fd23f2b16ea460f662337c Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 11 May 2025 15:50:31 +0200 Subject: [PATCH 15/15] Fixed registering missing files as "deleted" and not "deleting", because they are somehow deleted. --- Duplicati/Library/Main/Operation/RepairHandler.cs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 1e1b75590..61b5d6ab5 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -544,16 +544,14 @@ namespace Duplicati.Library.Main.Operation volumeWriter.Close(); db.UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null, rtr.Transaction); - db.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleting, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); - db.DeleteFileset(prevFilesetId, rtr.Transaction); + db.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); if (m_options.Dryrun) Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would upload fileset {0}, with size {1}, previous size {2}", originalVolume.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(volumeWriter.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(originalVolume.Size)); else { - if (!m_options.Dryrun) - rtr.Commit("CommitPriorToFilesetUpload"); + rtr.Commit("CommitPriorToFilesetUpload"); await backendManager.PutAsync(volumeWriter, null, null, false, null, cancellationToken).ConfigureAwait(false); } @@ -611,7 +609,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadIndexFile", "would re-upload index file {0}, with size {1}, previous size {2}", originalVolume.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(indexWriter.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(originalVolume.Size)); else { - db.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleting, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); + db.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); await backendManager.PutAsync(indexWriter, null, null, false, null, cancellationToken).ConfigureAwait(false); }