diff --git a/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs b/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs index 6d1b0c37c..369812aeb 100644 --- a/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs +++ b/Duplicati/CommandLine/DatabaseTool/Commands/Downgrade.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.CommandLine; using System.CommandLine.NamingConventionBinder; @@ -24,6 +24,7 @@ using Duplicati.Library.AutoUpdater; using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; using Duplicati.Library.SQLiteHelper; +using Duplicati.Library.Utility; namespace Duplicati.CommandLine.DatabaseTool.Commands; @@ -50,7 +51,7 @@ public static class Downgrade } .WithHandler(CommandHandler.Create((databases, serverdatafolder, serverversion, localversion, nobackups, includeuntrackeddatabases) => { - databases = Helper.FindAllDatabases(databases, serverdatafolder.FullName, includeuntrackeddatabases); + databases = Helper.FindAllDatabases(databases, serverdatafolder.FullName, includeuntrackeddatabases).Await(); if (databases.Length == 0) { Console.WriteLine("No databases found to downgrade"); @@ -100,7 +101,7 @@ public static class Downgrade bool isserverdb; try { - (version, isserverdb) = Helper.ExamineDatabase(db); + (version, isserverdb) = Helper.ExamineDatabase(db).Await(); } catch (Exception ex) { @@ -112,7 +113,8 @@ public static class Downgrade ApplyDowngrade(db, version, isserverdb ? serverversion : localversion, isserverdb ? serverVersions : localVersions, - nobackups); + nobackups) + .Await(); } })); @@ -132,7 +134,8 @@ public static class Downgrade /// The target database version /// The downgrade scripts /// Flag to disable backups - private static void ApplyDowngrade(string db, int dbversion, int targetversion, IEnumerable scripts, bool nobackups) + /// A task that completes when the downgrade is done. + private static async Task ApplyDowngrade(string db, int dbversion, int targetversion, IEnumerable scripts, bool nobackups) { if (dbversion > targetversion) { @@ -158,21 +161,27 @@ public static class Downgrade if (!nobackups) Helper.CreateFileBackup(db); - using var con = SQLiteLoader.LoadConnection(db, 0); - using var tr = con.BeginTransaction(); - using var cmd = con.CreateCommand(tr); + await using var con = await SQLiteLoader.LoadConnectionAsync(db) + .ConfigureAwait(false); + await using var tr = con.BeginTransaction(); + await using var cmd = con.CreateCommand(tr); foreach (var script in downgradeScripts) { Console.WriteLine($"Applying downgrade script {script.Filename} ..."); try { cmd.SetCommandAndParameters(script.Content); - var r = cmd.ExecuteScalar(); + var r = await cmd.ExecuteScalarAsync().ConfigureAwait(false); if (r != null) throw new UserInformationException($"{r}", "DowngradeScriptFailure"); - cmd.SetCommandAndParameters("UPDATE Version SET Version = @Version") - .SetParameterValue("@Version", script.Version - 1); - cmd.ExecuteNonQuery(); + + await cmd.SetCommandAndParameters(@" + UPDATE ""Version"" + SET ""Version"" = @Version + ") + .SetParameterValue("@Version", script.Version - 1) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } catch (Exception ex) { @@ -180,7 +189,7 @@ public static class Downgrade } } - tr.Commit(); + await tr.CommitAsync().ConfigureAwait(false); Console.WriteLine($"Downgraded {db} to version {targetversion}"); } else if (dbversion == targetversion) diff --git a/Duplicati/CommandLine/DatabaseTool/Commands/Execute.cs b/Duplicati/CommandLine/DatabaseTool/Commands/Execute.cs index d2f57df6a..30372ea18 100644 --- a/Duplicati/CommandLine/DatabaseTool/Commands/Execute.cs +++ b/Duplicati/CommandLine/DatabaseTool/Commands/Execute.cs @@ -1,23 +1,23 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.CommandLine; using System.CommandLine.NamingConventionBinder; @@ -48,7 +48,7 @@ public static class Execute if (createbackup) Helper.CreateFileBackup(database); - using var connection = SQLiteLoader.LoadConnection(database, 0); + using var connection = SQLiteLoader.LoadConnection(database); try { if (command.IndexOfAny(Path.GetInvalidPathChars()) < 0 && File.Exists(command)) diff --git a/Duplicati/CommandLine/DatabaseTool/Commands/List.cs b/Duplicati/CommandLine/DatabaseTool/Commands/List.cs index 5b9429251..c8b8f10a7 100644 --- a/Duplicati/CommandLine/DatabaseTool/Commands/List.cs +++ b/Duplicati/CommandLine/DatabaseTool/Commands/List.cs @@ -1,23 +1,23 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.CommandLine; using System.CommandLine.NamingConventionBinder; @@ -54,7 +54,7 @@ public static class List return; } - using var con = SQLiteLoader.LoadConnection(database, 0); + using var con = SQLiteLoader.LoadConnection(database); using var cmd = con.CreateCommand(); if (tables == null || tables.Length == 0) diff --git a/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs b/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs index 5454ffcf0..4bf77c1fb 100644 --- a/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs +++ b/Duplicati/CommandLine/DatabaseTool/Commands/Upgrade.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.CommandLine; using System.CommandLine.NamingConventionBinder; @@ -24,6 +24,7 @@ using Duplicati.Library.AutoUpdater; using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; using Duplicati.Library.SQLiteHelper; +using Duplicati.Library.Utility; namespace Duplicati.CommandLine.DatabaseTool.Commands; @@ -50,7 +51,7 @@ public static class Upgrade } .WithHandler(CommandHandler.Create((databases, serverdatafolder, serverversion, localversion, nobackups, includeuntrackeddatabases) => { - databases = Helper.FindAllDatabases(databases, serverdatafolder.FullName, includeuntrackeddatabases); + databases = Helper.FindAllDatabases(databases, serverdatafolder.FullName, includeuntrackeddatabases).Await(); if (databases.Length == 0) { Console.WriteLine("No databases found to upgrade"); @@ -83,7 +84,7 @@ public static class Upgrade bool isserverdb; try { - (version, isserverdb) = Helper.ExamineDatabase(db); + (version, isserverdb) = Helper.ExamineDatabase(db).Await(); } catch (Exception ex) { @@ -95,7 +96,8 @@ public static class Upgrade ApplyUpgrade(db, version, isserverdb ? serverversion : localversion, isserverdb ? serverVersions : localVersions, - nobackups); + nobackups) + .Await(); } })); @@ -145,7 +147,8 @@ public static class Upgrade /// The target database version /// The upgrade scripts /// Flag to disable backups - private static void ApplyUpgrade(string db, int dbversion, int targetversion, IEnumerable scripts, bool nobackups) + /// A task that completes when the upgrade is done. + private static async Task ApplyUpgrade(string db, int dbversion, int targetversion, IEnumerable scripts, bool nobackups) { if (targetversion > dbversion) { @@ -170,21 +173,27 @@ public static class Upgrade if (!nobackups) Helper.CreateFileBackup(db); - using var con = SQLiteLoader.LoadConnection(db, 0); - using var tr = con.BeginTransaction(); - using var cmd = con.CreateCommand(tr); + await using var con = await SQLiteLoader.LoadConnectionAsync(db) + .ConfigureAwait(false); + await using var tr = con.BeginTransaction(); + await using var cmd = con.CreateCommand(tr); foreach (var script in upgradeScripts) { Console.WriteLine($"Applying upgrade script {script.Filename} ..."); try { cmd.SetCommandAndParameters(script.Content); - var r = cmd.ExecuteScalar(); + var r = await cmd.ExecuteScalarAsync() + .ConfigureAwait(false); if (r != null) throw new UserInformationException($"{r}", "UpgradeScriptFailure"); - cmd.SetCommandAndParameters("UPDATE Version SET Version = @Version") - .SetParameterValue("@Version", script.Version); - cmd.ExecuteNonQuery(); + await cmd.SetCommandAndParameters(@" + UPDATE ""Version"" + SET ""Version"" = @Version + ") + .SetParameterValue("@Version", script.Version) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); } catch (Exception ex) { diff --git a/Duplicati/CommandLine/DatabaseTool/Helper.cs b/Duplicati/CommandLine/DatabaseTool/Helper.cs index bba1578c8..022f1cdd6 100644 --- a/Duplicati/CommandLine/DatabaseTool/Helper.cs +++ b/Duplicati/CommandLine/DatabaseTool/Helper.cs @@ -41,8 +41,8 @@ public static class Helper /// The input databases /// The folder to scan /// Whether to scan for extra databases - /// The list of databases - public static string[] FindAllDatabases(string[]? databases, string datafolder, bool scanExtra) + /// A task that when awaited contains the list of databases. + public static async Task FindAllDatabases(string[]? databases, string datafolder, bool scanExtra) { databases ??= []; if (databases.Length != 0) @@ -60,9 +60,13 @@ public static class Helper { try { - using var con = SQLiteLoader.LoadConnection(serverdb, 0); - using var cmd = con.CreateCommand(); - foreach (var rd in cmd.ExecuteReaderEnumerable(@"SELECT ""DBPath"" FROM ""Backup""")) + await using var con = await SQLiteLoader.LoadConnectionAsync(serverdb) + .ConfigureAwait(false); + await using var cmd = con.CreateCommand(@" + SELECT ""DBPath"" + FROM ""Backup"" + "); + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(CancellationToken.None).ConfigureAwait(false)) dbpaths.Add(rd.ConvertValueToString(0) ?? ""); } catch @@ -86,16 +90,30 @@ public static class Helper /// Examines the database and returns the version and whether it is a server database /// /// The path to the database - /// A tuple containing the version and whether it is a server database - public static (int Version, bool isserver) ExamineDatabase(string db) + /// A task that when awaited contains a tuple with the version and whether it is a server database + public static async Task<(int Version, bool isserver)> ExamineDatabase(string db) { - using (var con = SQLiteLoader.LoadConnection(db, 0)) - { - using var cmd = con.CreateCommand(); - var isserverdb = cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='Backup' OR name='Schedule'") == 2; - var version = (int)cmd.ExecuteScalarInt64("SELECT MAX(Version) FROM Version"); - return (version, isserverdb); - } + await using var con = await SQLiteLoader.LoadConnectionAsync(db) + .ConfigureAwait(false); + await using var cmd = con.CreateCommand(); + + var isserverdb = await cmd.ExecuteScalarInt64Async(@" + SELECT COUNT(*) + FROM sqlite_master + WHERE + type='table' + AND name='Backup' + OR name='Schedule' + ", CancellationToken.None) + .ConfigureAwait(false) == 2; + + var version = (int)await cmd.ExecuteScalarInt64Async(@" + SELECT MAX(Version) + FROM Version + ", CancellationToken.None) + .ConfigureAwait(false); + + return (version, isserverdb); } /// diff --git a/Duplicati/Library/Backend/Storj/Duplicati.Library.Backend.Storj.csproj b/Duplicati/Library/Backend/Storj/Duplicati.Library.Backend.Storj.csproj index 38fd940be..2564e783d 100644 --- a/Duplicati/Library/Backend/Storj/Duplicati.Library.Backend.Storj.csproj +++ b/Duplicati/Library/Backend/Storj/Duplicati.Library.Backend.Storj.csproj @@ -10,6 +10,8 @@ + + diff --git a/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs b/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs index 74bcc3327..6f54e71fc 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.DatabaseCollector.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Duplicati.Library.Logging; using Duplicati.Library.Main.Database; @@ -17,7 +19,7 @@ partial class BackendManager /// Since the backend communication is done in parallel with the actual operations, /// we store the performed operations in a queue and flush them to the database when the database is available. /// The main operations are responsible for flushing the messages when comitting a transaction. - /// If the operation fails, the logged messages here should still be flushed to the database, + /// If the operation fails, the logged messages here should still be flushed to the database, /// as they have already been performed on the remote destination. /// private class DatabaseCollector @@ -111,12 +113,12 @@ partial class BackendManager } /// - /// Flushes all messages to the database + /// Flushes all messages to the database. /// - /// The database to write pending messages to - /// The transaction to use, if any - /// Whether any messages were flushed - public bool FlushPendingMessages(LocalDatabase db, System.Data.IDbTransaction? transaction) + /// The database to write pending messages to. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains whether any messages were flushed. + public async Task FlushPendingMessages(LocalDatabase db, CancellationToken cancellationToken) { List entries; lock (m_dbqueuelock) @@ -134,22 +136,32 @@ partial class BackendManager // As we replace the list, we can now freely access the elements without locking foreach (var e in entries) if (e is RemoteOperationLogEntry operation) - db.LogRemoteOperation(operation.Action, operation.File, operation.Result, transaction); + await db.LogRemoteOperation(operation.Action, operation.File, operation.Result, cancellationToken) + .ConfigureAwait(false); else if (e is RemoteVolumeUpdate update && update.State == RemoteVolumeState.Deleted) { - db.UpdateRemoteVolume(update.Remotename, RemoteVolumeState.Deleted, update.Size, update.Hash, true, TimeSpan.FromHours(2), null, transaction); + await db.UpdateRemoteVolume(update.Remotename, RemoteVolumeState.Deleted, update.Size, update.Hash, true, TimeSpan.FromHours(2), null, cancellationToken) + .ConfigureAwait(false); volsRemoved.Add(update.Remotename); } else if (e is RemoteVolumeUpdate dbUpdate) - db.UpdateRemoteVolume(dbUpdate.Remotename, dbUpdate.State, dbUpdate.Size, dbUpdate.Hash, transaction); + await db.UpdateRemoteVolume(dbUpdate.Remotename, dbUpdate.State, dbUpdate.Size, dbUpdate.Hash, cancellationToken) + .ConfigureAwait(false); else if (e is RenameRemoteVolume rename) - db.RenameRemoteFile(rename.Oldname, rename.Newname, transaction); + await db.RenameRemoteFile(rename.Oldname, rename.Newname, cancellationToken) + .ConfigureAwait(false); else if (e != null) Log.WriteErrorMessage(LOGTAG, "InvalidQueueElement", null, "Queue had element of type: {0}, {1}", e.GetType(), e); // Finally remove volumes from DB. if (volsRemoved.Count > 0) - db.RemoveRemoteVolumes(volsRemoved); + { + await db.RemoveRemoteVolumes(volsRemoved, cancellationToken).ConfigureAwait(false); + // This commit is to mimic the pre Microsoft.Data.Sqlite backend + // behavior, in which RemoveRemoteVolumes started a new + // transaction, if none was passed down. + await db.Transaction.CommitAsync(token: cancellationToken).ConfigureAwait(false); + } return true; } diff --git a/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs b/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs index 4cffff12f..160622b3c 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.PutOperation.cs @@ -98,14 +98,14 @@ partial class BackendManager /// /// A callback that is invoked when the index volume is finished /// - public required Action? IndexVolumeFinishedCallback { get; init; } + public required Func? IndexVolumeFinishedCallback { get; init; } /// /// The default callback for database updates (no-op) /// public static Func OnDbUpdateDefault = () => Task.CompletedTask; /// - /// Creates a new put operation + /// Creates a new put operation /// /// The execution context /// True if the operation should wait for completion @@ -234,7 +234,8 @@ partial class BackendManager OriginalIndexFile.FinishVolume(hash, size); - IndexVolumeFinishedCallback?.Invoke(); + if (IndexVolumeFinishedCallback != null) + await IndexVolumeFinishedCallback(); OriginalIndexFile.Close(); indexOperation = (OriginalIndexFile, req2); @@ -344,7 +345,7 @@ partial class BackendManager Context.Database.LogRemoteVolumeRenamed(oldname, newname); remoteFilename = newname; - // If there is an index file attached to the block file, + // If there is an index file attached to the block file, // it references the block filename, so we create a new index file // which is a copy of the current, but with the new name if (indexOperation != null) diff --git a/Duplicati/Library/Main/Backend/BackendManager.cs b/Duplicati/Library/Main/Backend/BackendManager.cs index 072307bfe..f235d7669 100644 --- a/Duplicati/Library/Main/Backend/BackendManager.cs +++ b/Duplicati/Library/Main/Backend/BackendManager.cs @@ -246,7 +246,7 @@ internal partial class BackendManager : IBackendManager /// The callback to call when the database is updated /// The cancellation token /// An awaitable task - public async Task PutAsync(VolumeWriterBase volume, IndexVolumeWriter? indexVolume, Action? indexVolumeFinished, bool waitForComplete, Func? onDbUpdate, CancellationToken cancelToken) + public async Task PutAsync(VolumeWriterBase volume, IndexVolumeWriter? indexVolume, Func? indexVolumeFinished, bool waitForComplete, Func? onDbUpdate, CancellationToken cancelToken) { volume.Close(); @@ -292,23 +292,21 @@ internal partial class BackendManager : IBackendManager } /// - /// Flushes the database messages to the database + /// Flushes the database messages to the database. /// - /// The database to write to - /// The transaction to use - /// The cancellation token - /// - public Task FlushPendingMessagesAsync(LocalDatabase database, IDbTransaction? transaction, CancellationToken cancellationToken) + /// The database to write to. + /// The cancellation token. + /// A task that completes when the messages are flushed. + public async Task FlushPendingMessagesAsync(LocalDatabase database, CancellationToken cancellationToken) { - context.Database.FlushPendingMessages(database, transaction); - return Task.CompletedTask; + await context.Database.FlushPendingMessages(database, cancellationToken).ConfigureAwait(false); } /// - /// Waits for the backend queue to be empty + /// Waits for the backend queue to be empty. /// - /// The cancellation token - /// An awaitable task + /// The cancellation token. + /// An awaitable task. public async Task WaitForEmptyAsync(CancellationToken cancellationToken) { var op = new WaitForEmptyOperation(context, cancellationToken); @@ -323,22 +321,22 @@ internal partial class BackendManager : IBackendManager /// The transaction to use /// The cancellation token /// An awaitable task - public async Task WaitForEmptyAsync(LocalDatabase database, IDbTransaction? transaction, CancellationToken cancellationToken) + public async Task WaitForEmptyAsync(LocalDatabase database, CancellationToken cancellationToken) { - await FlushPendingMessagesAsync(database, transaction, cancellationToken).ConfigureAwait(false); + await FlushPendingMessagesAsync(database, cancellationToken).ConfigureAwait(false); await WaitForEmptyAsync(cancellationToken).ConfigureAwait(false); - await FlushPendingMessagesAsync(database, transaction, cancellationToken).ConfigureAwait(false); + await FlushPendingMessagesAsync(database, cancellationToken).ConfigureAwait(false); } /// - /// Stops the backend manager and flushes any pending messages to the database + /// Stops the backend manager and flushes any pending messages to the database. /// - /// The database to write pending messages to - /// The transaction to use, if any - public async Task StopRunnerAndFlushMessages(LocalDatabase database, IDbTransaction? transaction) + /// The database to write pending messages to. + /// A task that completes when the runner is stopped and messages are flushed. + public async Task StopRunnerAndFlushMessages(LocalDatabase database) { await requestChannel.RetireAsync().ConfigureAwait(false); - await FlushPendingMessagesAsync(database, transaction, CancellationToken.None).ConfigureAwait(false); + await FlushPendingMessagesAsync(database, CancellationToken.None).ConfigureAwait(false); if (queueRunner.IsFaulted) Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerShutdown", queueRunner.Exception, "Backend manager queue runner crashed"); @@ -368,7 +366,8 @@ internal partial class BackendManager : IBackendManager yield break; // Get the first volume, so we do not have pending parallel transfers - var prevResult = await GetWithInfoAsync(prevVolume.Name, prevVolume.Hash, prevVolume.Size, cancelToken); + var prevResult = await GetWithInfoAsync(prevVolume.Name, prevVolume.Hash, prevVolume.Size, cancelToken) + .ConfigureAwait(false); foreach (var volume in volumes.Skip(1)) { @@ -381,7 +380,7 @@ internal partial class BackendManager : IBackendManager // Set up for next iteration prevVolume = volume; - prevResult = await nextTask; + prevResult = await nextTask.ConfigureAwait(false); } // Return the last result diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index 75df9b5af..b5270d10d 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -252,9 +252,9 @@ namespace Duplicati.Library.Main { return RunAction(new ListRemoteResults(), async (result, backendManager) => { - using (var tf = File.Exists(m_options.Dbpath) ? null : new TempFile()) - using (var db = new LocalDatabase(((string)tf) ?? m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache)) - result.SetResult(await backendManager.ListAsync(CancellationToken.None).ConfigureAwait(false)); + using var tf = File.Exists(m_options.Dbpath) ? null : new TempFile(); + await using var db = await LocalDatabase.CreateLocalDatabaseAsync(((string)tf) ?? m_options.Dbpath, "list-remote", true, null, result.TaskControl.ProgressToken).ConfigureAwait(false); + result.SetResult(await backendManager.ListAsync(CancellationToken.None).ConfigureAwait(false)); }); } @@ -277,12 +277,15 @@ namespace Duplicati.Library.Main // and uses the same prefix (see issues #2678, #3845, and #4244). if (File.Exists(m_options.Dbpath)) { - using (LocalDatabase db = new LocalDatabase(m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache)) - { - var dbRemoteVolumes = db.GetRemoteVolumes(); - var dbRemoteFiles = dbRemoteVolumes.Select(x => x.Name).ToHashSet(); - list = list.Where(x => dbRemoteFiles.Contains(x.File.Name)).ToList(); - } + await using LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "list-remote", true, null, result.TaskControl.ProgressToken).ConfigureAwait(false); + var dbRemoteVolumes = db.GetRemoteVolumes(result.TaskControl.ProgressToken); + var dbRemoteFiles = await dbRemoteVolumes + .Select(x => x.Name) + .ToHashSetAsync(cancellationToken: result.TaskControl.ProgressToken) + .ConfigureAwait(false); + list = list.Where(x => + dbRemoteFiles.Contains(x.File.Name) + ).ToList(); } result.OperationProgressUpdater.UpdatePhase(OperationPhase.Delete_Deleting); @@ -517,10 +520,10 @@ namespace Duplicati.Library.Main // This would also allow us to control the unclean shutdown flag, // by toggling this on start and completion of transfers in the manager, // instead of relying on the operations to correctly toggle the flag - if (LocalDatabase.Exists(m_options.Dbpath) && !m_options.NoLocalDb) + if (File.Exists(m_options.Dbpath) && !m_options.NoLocalDb) { - using (var db = new LocalDatabase(m_options.Dbpath, result.MainOperation.ToString(), true, m_options.SqlitePageCache)) - backend.StopRunnerAndFlushMessages(db, null).Await(); + using var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, result.MainOperation.ToString(), true, null, CancellationToken.None).Await(); + backend.StopRunnerAndFlushMessages(db).Await(); } else { @@ -533,29 +536,27 @@ namespace Duplicati.Library.Main resultSetter.EndTime = DateTime.UtcNow; resultSetter.Interrupted = false; - if (LocalDatabase.Exists(m_options.Dbpath) && !m_options.Dryrun) + if (File.Exists(m_options.Dbpath) && !m_options.Dryrun) { - using (var db = new LocalDatabase(m_options.Dbpath, null, true, m_options.SqlitePageCache)) - { - db.WriteResults(result); - db.PurgeLogData(m_options.LogRetention); - db.PurgeDeletedVolumes(DateTime.UtcNow); + using var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, null, true, null, CancellationToken.None).Await(); + db.WriteResults(result, CancellationToken.None).Await(); + db.PurgeLogData(m_options.LogRetention, CancellationToken.None).Await(); + db.PurgeDeletedVolumes(DateTime.UtcNow, CancellationToken.None).Await(); - // Vacuum is done AFTER the results are written to the database - // This means that the information about the vacuum is not stored in the database, - // but will be reported in the log output and messages sent with any of the reporting modules - if (m_options.AutoVacuum && result is IResultsWithVacuum vacuumResults && result is BasicResults basicResults) + // Vacuum is done AFTER the results are written to the database + // This means that the information about the vacuum is not stored in the database, + // but will be reported in the log output and messages sent with any of the reporting modules + if (m_options.AutoVacuum && result is IResultsWithVacuum vacuumResults && result is BasicResults basicResults) + { + try { - try - { - vacuumResults.VacuumResults = new VacuumResults(basicResults); - new Operation.VacuumHandler(m_options, (VacuumResults)vacuumResults.VacuumResults) - .RunAsync().Await(); - } - catch (Exception ex) - { - Logging.Log.WriteVerboseMessage(LOGTAG, "FailedToVacuum", ex, "Failed to vacuum database"); - } + vacuumResults.VacuumResults = new VacuumResults(basicResults); + new Operation.VacuumHandler(m_options, (VacuumResults)vacuumResults.VacuumResults) + .RunAsync().Await(); + } + catch (Exception ex) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "FailedToVacuum", ex, "Failed to vacuum database"); } } } @@ -578,9 +579,9 @@ namespace Duplicati.Library.Main try { // No operation was started in database, so write logs to new operation - if (LocalDatabase.Exists(m_options.Dbpath) && !m_options.Dryrun) - using (var db = new LocalDatabase(m_options.Dbpath, result.MainOperation.ToString(), true, m_options.SqlitePageCache)) - db.WriteResults(result); + if (File.Exists(m_options.Dbpath) && !m_options.Dryrun) + using (var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, result.MainOperation.ToString(), true, null, CancellationToken.None).Await()) + db.WriteResults(result, CancellationToken.None).Await(); } catch (Exception we) { @@ -603,9 +604,9 @@ namespace Duplicati.Library.Main resultSetter.Fatal = true; // Write logs to previous operation if database exists - if (LocalDatabase.Exists(m_options.Dbpath) && !m_options.Dryrun) - using (var db = new LocalDatabase(m_options.Dbpath, null, true, m_options.SqlitePageCache)) - db.WriteResults(result); + if (File.Exists(m_options.Dbpath) && !m_options.Dryrun) + using (var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, null, true, null, CancellationToken.None).Await()) + db.WriteResults(result, CancellationToken.None).Await(); } catch (Exception we) { @@ -756,7 +757,7 @@ namespace Duplicati.Library.Main private async Task ApplySecretProvider(CancellationToken cancellationToken) { var args = new[] { new Library.Utility.Uri(m_backendUrl) }; - await SecretProviderHelper.ApplySecretProviderAsync([], args, m_options.RawOptions, TempFolder.SystemTempPath, SecretProvider, cancellationToken); + await SecretProviderHelper.ApplySecretProviderAsync([], args, m_options.RawOptions, TempFolder.SystemTempPath, SecretProvider, cancellationToken).ConfigureAwait(false); // Write back the backend argument, if it was modified by the secret provider m_backendUrl = args[0].ToString(); } diff --git a/Duplicati/Library/Main/Database/ChangeStatistics.cs b/Duplicati/Library/Main/Database/ChangeStatistics.cs index 49f69ec11..b4a79a5e4 100644 --- a/Duplicati/Library/Main/Database/ChangeStatistics.cs +++ b/Duplicati/Library/Main/Database/ChangeStatistics.cs @@ -1,27 +1,29 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable using System; -using System.Data; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database; @@ -36,132 +38,197 @@ public static class ChangeStatistics private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(ChangeStatistics)); /// - /// Calculates the change statistics for the current and previous fileset + /// Calculates the change statistics for the current and previous fileset. /// - /// The database command to use - /// The results object to update with the statistics - /// The ID of the current fileset - /// The ID of the previous fileset - internal static void UpdateChangeStatistics(IDbCommand cmd, BackupResults results, long currentFilesetId, long previousFilesetId) + /// The database command to use. + /// The results object to update with the statistics. + /// The ID of the current fileset. + /// The ID of the previous fileset. + /// The cancellation token to monitor for cancellation requests. + /// A task that completes when the statistics are updated. + internal static async Task UpdateChangeStatistics(SqliteCommand cmd, BackupResults results, long currentFilesetId, long previousFilesetId, CancellationToken token) { var tmpName = $"TmpFileState_{Guid.NewGuid():N}"; try { // Create temp table - cmd.SetCommandAndParameters(LocalDatabase.FormatInvariant($@" - CREATE TEMP TABLE ""{tmpName}"" AS - SELECT - FL.""PrefixID"", - FL.""Path"", - FL.""BlocksetID"", - BS_Meta.""Fullhash"" AS ""Metahash"", - CASE FE.""FilesetID"" - WHEN @LastFilesetId THEN 0 - WHEN @CurrentFilesetId THEN 1 - END AS Source - FROM ""FileLookup"" FL - JOIN ""FilesetEntry"" FE ON FL.""ID"" = FE.""FileID"" - LEFT JOIN ""Blockset"" BS_Data ON FL.""BlocksetID"" = BS_Data.""ID"" - LEFT JOIN ""Metadataset"" M ON FL.""MetadataID"" = M.""ID"" - LEFT JOIN ""Blockset"" BS_Meta ON M.""BlocksetID"" = BS_Meta.""ID"" - WHERE FE.""FilesetID"" IN (@LastFilesetId, @CurrentFilesetId); -")) -.SetParameterValue("@LastFilesetId", previousFilesetId) -.SetParameterValue("@CurrentFilesetId", currentFilesetId) -.ExecuteNonQuery(); + await cmd.SetCommandAndParameters($@" + CREATE TEMP TABLE ""{tmpName}"" AS + SELECT + ""FL"".""PrefixID"", + ""FL"".""Path"", + ""FL"".""BlocksetID"", + ""BS_Meta"".""Fullhash"" AS ""Metahash"", + CASE ""FE"".""FilesetID"" + WHEN @LastFilesetId THEN 0 + WHEN @CurrentFilesetId THEN 1 + END AS ""Source"" + FROM ""FileLookup"" ""FL"" + JOIN ""FilesetEntry"" ""FE"" + ON ""FL"".""ID"" = ""FE"".""FileID"" + LEFT JOIN ""Blockset"" ""BS_Data"" + ON ""FL"".""BlocksetID"" = ""BS_Data"".""ID"" + LEFT JOIN ""Metadataset"" M + ON ""FL"".""MetadataID"" = M.""ID"" + LEFT JOIN ""Blockset"" ""BS_Meta"" + ON M.""BlocksetID"" = ""BS_Meta"".""ID"" + WHERE ""FE"".""FilesetID"" IN ( + @LastFilesetId, + @CurrentFilesetId + ); + ") + .SetParameterValue("@LastFilesetId", previousFilesetId) + .SetParameterValue("@CurrentFilesetId", currentFilesetId) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); // Index for fast comparison - cmd.ExecuteNonQuery(LocalDatabase.FormatInvariant($@"CREATE INDEX ""idx_{tmpName}"" ON ""{tmpName}"" (""PrefixID"", ""Path"")")); + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""idx_{tmpName}"" + ON ""{tmpName}"" ( + ""PrefixID"", + ""Path"" + ) + ", token) + .ConfigureAwait(false); // Added - results.AddedFolders = CountAdded(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID); - results.AddedSymlinks = CountAdded(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID); - results.AddedFiles = CountAdded(cmd, tmpName, null, new[] { LocalDatabase.FOLDER_BLOCKSET_ID, LocalDatabase.SYMLINK_BLOCKSET_ID }); + results.AddedFolders = await CountAdded(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID, null, token) + .ConfigureAwait(false); + results.AddedSymlinks = await CountAdded(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID, null, token) + .ConfigureAwait(false); + results.AddedFiles = await CountAdded(cmd, tmpName, null, [ + LocalDatabase.FOLDER_BLOCKSET_ID, + LocalDatabase.SYMLINK_BLOCKSET_ID + ], token) + .ConfigureAwait(false); // Deleted - results.DeletedFolders = CountDeleted(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID); - results.DeletedSymlinks = CountDeleted(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID); - results.DeletedFiles = CountDeleted(cmd, tmpName, null, new[] { LocalDatabase.FOLDER_BLOCKSET_ID, LocalDatabase.SYMLINK_BLOCKSET_ID }); + results.DeletedFolders = await CountDeleted(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID, null, token) + .ConfigureAwait(false); + results.DeletedSymlinks = await CountDeleted(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID, null, token) + .ConfigureAwait(false); + results.DeletedFiles = await CountDeleted(cmd, tmpName, null, [ + LocalDatabase.FOLDER_BLOCKSET_ID, + LocalDatabase.SYMLINK_BLOCKSET_ID + ], token) + .ConfigureAwait(false); // Modified - results.ModifiedFolders = CountModified(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID); - results.ModifiedSymlinks = CountModified(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID); - results.ModifiedFiles = CountModified(cmd, tmpName, null, new[] { LocalDatabase.FOLDER_BLOCKSET_ID, LocalDatabase.SYMLINK_BLOCKSET_ID }); + results.ModifiedFolders = await CountModified(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID, null, token) + .ConfigureAwait(false); + results.ModifiedSymlinks = await CountModified(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID, null, token) + .ConfigureAwait(false); + results.ModifiedFiles = await CountModified(cmd, tmpName, null, [ + LocalDatabase.FOLDER_BLOCKSET_ID, + LocalDatabase.SYMLINK_BLOCKSET_ID + ], token) + .ConfigureAwait(false); } finally { - try { cmd.ExecuteNonQuery(LocalDatabase.FormatInvariant($@"DROP TABLE IF EXISTS ""{tmpName}"";")); } - catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "DropTemp", ex, $"Failed to drop {tmpName}"); } + try + { + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmpName}"";", token) + .ConfigureAwait(false); + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "DropTemp", ex, $"Failed to drop {tmpName}"); + } } } /// - /// Counts the number of added files or folders + /// Counts the number of added files or folders. /// - /// The database command to use - /// The name of the temporary table - /// The ID of the blockset to count, or null for all - /// The blocksets to exclude from the count - /// The number of added files or folders - private static long CountAdded(IDbCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null) + /// The database command to use. + /// The name of the temporary table. + /// The ID of the blockset to count, or null for all. + /// The blocksets to exclude from the count. + /// The cancellation token to monitor for cancellation requests. + /// A task that when awaited contains the number of added files or folders. + private static async Task CountAdded(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets, CancellationToken token) { - var conditions = LocalDatabase.FormatInvariant($@"Source = 1 AND NOT EXISTS ( - SELECT 1 FROM ""{tmpName}"" B - WHERE B.Source = 0 - AND B.PrefixID = A.PrefixID - AND B.Path = A.Path - )"); + var conditions = $@" + ""Source"" = 1 + AND NOT EXISTS ( + SELECT 1 + FROM ""{tmpName}"" ""B"" + WHERE + ""B"".""Source"" = 0 + AND ""B"".""PrefixID"" = ""A"".""PrefixID"" + AND ""B"".""Path"" = ""A"".""Path"" + ) + "; - return CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets); + return await CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets, token) + .ConfigureAwait(false); } /// - /// Counts the number of deleted files or folders + /// Counts the number of deleted files or folders. /// - /// The database command to use - /// The name of the temporary table - /// The ID of the blockset to count, or null for all - /// The blocksets to exclude from the count - /// The number of deleted files or folders - private static long CountDeleted(IDbCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null) + /// The database command to use. + /// The name of the temporary table. + /// The ID of the blockset to count, or null for all. + /// The blocksets to exclude from the count. + /// The cancellation token to monitor for cancellation requests. + /// A task that when awaited contains the number of deleted files or folders. + private static async Task CountDeleted(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets, CancellationToken token) { - var conditions = @"Source = 0 AND NOT EXISTS ( - SELECT 1 FROM """ + tmpName + @""" B - WHERE B.Source = 1 - AND B.PrefixID = A.PrefixID - AND B.Path = A.Path - )"; + var conditions = @$" + ""Source"" = 0 + AND NOT EXISTS ( + SELECT 1 + FROM ""{tmpName}"" ""B"" + WHERE + ""B"".""Source"" = 1 + AND ""B"".""PrefixID"" = ""A"".""PrefixID"" + AND ""B"".""Path"" = ""A"".""Path"" + ) + "; - return CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets); + return await CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets, token) + .ConfigureAwait(false); } /// - /// Counts the number of modified files or folders + /// Counts the number of modified files or folders. /// - /// The database command to use - /// The name of the temporary table - /// The ID of the blockset to count, or null for all - /// The hash column to compare - /// The blocksets to exclude from the count - /// The number of modified files or folders - private static long CountModified(IDbCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null) + /// The database command to use. + /// The name of the temporary table. + /// The ID of the blockset to count, or null for all. + /// The blocksets to exclude from the count. + /// The cancellation token to monitor for cancellation requests. + /// A task that when awaited contains the number of modified files or folders. + private static async Task CountModified(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets, CancellationToken token) { string conditions; if (blocksetId == LocalDatabase.FOLDER_BLOCKSET_ID || blocksetId == LocalDatabase.SYMLINK_BLOCKSET_ID) { conditions = @" - A.Source = 0 AND B.Source = 1 - AND A.PrefixID = B.PrefixID AND A.Path = B.Path - AND A.Metahash IS NOT B.Metahash"; + ""A"".""Source"" = 0 + AND ""B"".""Source"" = 1 + AND ""A"".""PrefixID"" = ""B"".""PrefixID"" + AND ""A"".""Path"" = ""B"".""Path"" + AND ""A"".""Metahash"" IS NOT ""B"".""Metahash"" + "; } else { conditions = @" - A.Source = 0 AND B.Source = 1 - AND A.PrefixID = B.PrefixID AND A.Path = B.Path - AND (A.BlocksetID IS NOT B.BlocksetID OR A.Metahash IS NOT B.Metahash)"; + ""A"".""Source"" = 0 + AND ""B"".""Source"" = 1 + AND ""A"".""PrefixID"" = ""B"".""PrefixID"" + AND ""A"".""Path"" = ""B"".""Path"" + AND ( + ""A"".""BlocksetID"" IS NOT ""B"".""BlocksetID"" + OR ""A"".""Metahash"" IS NOT ""B"".""Metahash"" + ) + "; } string blocksetCondition = GetBlocksetCondition("A", blocksetId, excludeBlocksets); @@ -169,33 +236,48 @@ public static class ChangeStatistics conditions += $" AND {blocksetCondition}"; string sql = $@" - SELECT COUNT(*) - FROM ""{tmpName}"" A - JOIN ""{tmpName}"" B ON A.PrefixID = B.PrefixID AND A.Path = B.Path - WHERE {conditions}"; + SELECT COUNT(*) + FROM ""{tmpName}"" ""A"" + JOIN ""{tmpName}"" ""B"" + ON ""A"".""PrefixID"" = ""B"".""PrefixID"" + AND ""A"".""Path"" = ""B"".""Path"" + WHERE {conditions} + "; - return cmd.SetCommandAndParameters(sql).ExecuteScalarInt64(0); + return await cmd + .SetCommandAndParameters(sql) + .ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false); } /// - /// Counts the number of files or folders with a specific condition + /// Counts the number of files or folders with a specific condition. /// - /// The database command to use - /// The name of the temporary table - /// The alias for the temporary table - /// The base condition for the count - /// The ID of the blockset to count, or null for all - /// The blocksets to exclude from the count - /// The number of files or folders that match the condition - private static long CountWithCondition(IDbCommand cmd, string tmpName, string alias, string baseCondition, long? blocksetId, long[]? excludeBlocksets) + /// The database command to use. + /// The name of the temporary table. + /// The alias for the temporary table. + /// The base condition for the count. + /// The ID of the blockset to count, or null for all. + /// The blocksets to exclude from the count. + /// The cancellation token to monitor for cancellation requests. + /// A task that when awaited contains the number of files or folders that match the condition. + private static async Task CountWithCondition(SqliteCommand cmd, string tmpName, string alias, string baseCondition, long? blocksetId, long[]? excludeBlocksets, CancellationToken token) { var fullCondition = baseCondition; var blocksetCondition = GetBlocksetCondition(alias, blocksetId, excludeBlocksets); if (!string.IsNullOrEmpty(blocksetCondition)) fullCondition += " AND " + blocksetCondition; - var sql = LocalDatabase.FormatInvariant($@"SELECT COUNT(*) FROM ""{tmpName}"" {alias} WHERE {fullCondition}"); - return cmd.SetCommandAndParameters(sql).ExecuteScalarInt64(0); + var sql = $@" + SELECT COUNT(*) + FROM ""{tmpName}"" ""{alias}"" + WHERE {fullCondition} + "; + + return await cmd + .SetCommandAndParameters(sql) + .ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false); } /// @@ -208,9 +290,9 @@ public static class ChangeStatistics private static string GetBlocksetCondition(string alias, long? blocksetId, long[]? exclude) { if (blocksetId.HasValue) - return LocalDatabase.FormatInvariant($"{alias}.\"BlocksetID\" = {blocksetId.Value}"); + return @$"{alias}.""BlocksetID"" = {blocksetId.Value}"; if (exclude?.Length > 0) - return LocalDatabase.FormatInvariant($"{alias}.\"BlocksetID\" NOT IN ({string.Join(",", exclude)})"); + return @$"{alias}.""BlocksetID"" NOT IN ({string.Join(",", exclude)})"; return string.Empty; } } \ No newline at end of file diff --git a/Duplicati/Library/Main/Database/ExtensionMethods.cs b/Duplicati/Library/Main/Database/ExtensionMethods.cs index a294c4c04..a13f8e413 100644 --- a/Duplicati/Library/Main/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/Database/ExtensionMethods.cs @@ -1,28 +1,32 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.Data; using System.Linq; using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.Utility; +using Microsoft.Data.Sqlite; #nullable enable @@ -31,13 +35,674 @@ namespace Duplicati.Library.Main.Database; /// /// Extension method for /// -public static class ExtensionMethods +public static partial class ExtensionMethods { + + /// + /// Converts the value at the given index in the reader to an Int64. + /// + /// The to read from. + /// The index of the value to convert. + /// The default value to return if the value is null or cannot be converted. + /// The converted Int64 value, or the default value if the conversion fails. + public static long ConvertValueToInt64(this SqliteDataReader reader, int index, long defaultvalue = -1) + { + try + { + if (!reader.IsDBNull(index)) + return reader.GetInt64(index); + } + catch { } + + return defaultvalue; + } + + /// + /// Converts the value at the given index in the reader to a string. + /// + /// The to read from. + /// The index of the value to convert. + /// The converted string value, or null if the value is null or cannot be converted. + public static string? ConvertValueToString(this SqliteDataReader reader, int index) + { + var v = reader.GetValue(index); + if (v == null || v == DBNull.Value) + return null; + + return v.ToString(); + } + + /// + /// Creates a new with the given command text. + /// + /// The to create the command for. + /// The command text to set for the command. + /// A new with the command text set. + public static SqliteCommand CreateCommand(this SqliteConnection self, string cmdtext) + { + return CreateCommandAsync(self, cmdtext, default).Await(); + } + + /// + /// Creates a new with the given command text and prepares it asynchronously. + /// + /// The to create the command for. + /// The command text to set for the command. + /// Cancellation token to cancel the operation. + /// A new with the command text set and prepared asynchronously. + public static async Task CreateCommandAsync(this SqliteConnection self, string cmdtext, CancellationToken cancellationToken) + { + var cmd = self.CreateCommand() + .SetCommandAndParameters(cmdtext); + + await cmd.PrepareAsync(cancellationToken).ConfigureAwait(false); + + return cmd; + } + + /// + /// Creates a new with the given transaction. + /// + /// The to create the command for. + /// The to set for the command. + /// A new with the transaction set. + public static SqliteCommand CreateCommand(this SqliteConnection self, SqliteTransaction transaction) + { + var cmd = self.CreateCommand(); + cmd.SetTransaction(transaction); + + return cmd; + } + + /// + /// Creates a new with the given reusable transaction. + /// + /// The to create the command for. + /// The to set for the command. + /// A new with the transaction set. + internal static SqliteCommand CreateCommand(this SqliteConnection self, ReusableTransaction rtr) + { + return self.CreateCommand(rtr.Transaction); + } + + /// + /// Executes the command asynchronously and returns the number of rows affected. + /// + /// The instance to execute on. + /// The command text to execute. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the number of rows affected. + public static async Task ExecuteNonQueryAsync(this SqliteCommand self, string? cmdtext, CancellationToken cancellationToken) + { + return await ExecuteNonQueryAsync(self, true, cmdtext, null, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns the number of rows affected. + /// + /// The instance to execute on. + /// The command text to execute. + /// The values to use as parameters. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the number of rows affected. + public static async Task ExecuteNonQueryAsync(this SqliteCommand self, string cmd, Dictionary values, CancellationToken cancellationToken) + { + return await ExecuteNonQueryAsync(self, true, cmd, values, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns the number of rows affected. + /// + /// The instance to execute on. + /// Whether to write a log entry. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the number of rows affected. + public static async Task ExecuteNonQueryAsync(this SqliteCommand self, bool writeLog, CancellationToken cancellationToken) + { + return await ExecuteNonQueryAsync(self, writeLog, null, null, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns the number of rows affected. + /// + /// The instance to execute on. + /// Whether to write a log entry. + /// The command text to execute. + /// The values to set as parameters. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the number of rows affected. + public static async Task ExecuteNonQueryAsync(this SqliteCommand self, bool writeLog, string? cmd, Dictionary? values, CancellationToken cancellationToken) + { + if (cmd != null) + self.SetCommandAndParameters(cmd); + + if (values != null && values.Count > 0) + self.SetParameterValues(values); + + using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteNonQueryAsync", string.Format("ExecuteNonQueryAsync: {0}", self.GetPrintableCommandText())) : null) + return await self.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns a . + /// + /// The instance to execute on. + /// The command text to execute. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the . + public static async Task ExecuteReaderAsync(this SqliteCommand self, string cmdtext, CancellationToken cancellationToken) + { + return await ExecuteReaderAsync(self, true, cmdtext, null, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns a . + /// + /// The instance to execute on. + /// Whether to write a log entry. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the . + public static async Task ExecuteReaderAsync(this SqliteCommand self, bool writeLog, CancellationToken cancellationToken) + { + return await ExecuteReaderAsync(self, writeLog, null, null, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns a . + /// + /// The instance to execute on. + /// The command text to execute. + /// The values to use as parameters. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the . + public static async Task ExecuteReaderAsync(this SqliteCommand self, string cmdtext, Dictionary? values, CancellationToken cancellationToken) + { + return await ExecuteReaderAsync(self, true, cmdtext, values, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns a . + /// + /// The instance to execute on. + /// Whether to write a log entry. + /// The command text to execute. + /// The values to set as parameters. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the . + public static async Task ExecuteReaderAsync(this SqliteCommand self, bool writeLog, string? cmdtext, Dictionary? values, CancellationToken cancellationToken) + { + if (cmdtext != null) + self.SetCommandAndParameters(cmdtext); + + if (values != null && values.Count > 0) + self.SetParameterValues(values); + + using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteReader", string.Format("ExecuteReader: {0}", self.GetPrintableCommandText())) : null) + return await self.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns an enumerable of . + /// + /// The instance to execute on. + /// Cancellation token to cancel the operation. + /// An asynchronous enumerable of . + public static IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self, CancellationToken cancellationToken) + { + return ExecuteReaderEnumerableAsync(self, true, null, null, cancellationToken); + } + + /// + /// Executes the command asynchronously and returns an enumerable of . + /// + /// The instance to execute on. + /// The command text to execute. + /// Cancellation token to cancel the operation. + /// An asynchronous enumerable of . + public static IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self, string cmdtext, CancellationToken cancellationToken) + { + return ExecuteReaderEnumerableAsync(self, true, cmdtext, null, cancellationToken); + } + + /// + /// Executes the command asynchronously and returns an enumerable of . + /// + /// The instance to execute on. + /// The command text to execute. + /// The values to use as parameters. + /// Cancellation token to cancel the operation. + /// An asynchronous enumerable of . + public static IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self, string cmdtext, Dictionary? values, CancellationToken cancellationToken) + { + return ExecuteReaderEnumerableAsync(self, true, cmdtext, values, cancellationToken); + } + + /// + /// Executes the command asynchronously and returns an enumerable of . + /// + /// The instance to execute on. + /// Whether to write a log entry. + /// The command text to execute. + /// The values to set as parameters. + /// An asynchronous enumerable of . + public static async IAsyncEnumerable ExecuteReaderEnumerableAsync(this SqliteCommand self, bool writeLog, string? cmdtext, Dictionary? values, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + if (cmdtext != null) + self.SetCommandAndParameters(cmdtext); + + if (values != null && values.Count > 0) + self.SetParameterValues(values); + + using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteReaderEnumerableAsync", $"ExecuteReaderEnumerableAsync: {self.GetPrintableCommandText()}") : null) + await using (var rd = await self.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false)) + while (await rd.ReadAsync(cancellationToken).ConfigureAwait(false)) + yield return rd; + } + + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set. + /// + /// The instance to execute on. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the first column of the first row in the result set, or null if no rows are returned. + public static async Task ExecuteScalarAsync(this SqliteCommand self, CancellationToken cancellationToken) + { + return await self.ExecuteScalarAsync(true, null, null, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set. + /// + /// The instance to execute on. + /// The command text to execute. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the first column of the first row in the result set, or null if no rows are returned. + public static async Task ExecuteScalarAsync(this SqliteCommand self, string cmdtext, CancellationToken cancellationToken) + { + return await self.ExecuteScalarAsync(true, cmdtext, null, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set. + /// + /// The instance to execute on. + /// Whether to write a log entry. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the first column of the first row in the result set, or null if no rows are returned. + public static async Task ExecuteScalarAsync(this SqliteCommand self, bool writeLog, CancellationToken cancellationToken) + { + return await self.ExecuteScalarAsync(writeLog, null, null, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set. + /// + /// The instance to execute on. + /// The command text to execute. + /// The values to use as parameters. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the first column of the first row in the result set, or null if no rows are returned. + public static async Task ExecuteScalarAsync(this SqliteCommand self, string cmdtext, Dictionary? values, CancellationToken cancellationToken) + { + return await self.ExecuteScalarAsync(true, cmdtext, values, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set. + /// + /// The instance to execute on. + /// Whether to write a log entry. + /// The command text to execute. + /// The values to set as parameters. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the first column of the first row in the result set, or null if no rows are returned. + public static async Task ExecuteScalarAsync(this SqliteCommand self, bool writeLog, string? cmd, Dictionary? values, CancellationToken cancellationToken) + { + if (cmd != null) + self.SetCommandAndParameters(cmd); + + if (values != null && values.Count > 0) + self.SetParameterValues(values); + + using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalarAsync", string.Format("ExecuteScalarAsync: {0}", self.GetPrintableCommandText())) : null) + return await self.ExecuteScalarAsync(cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64. + /// + /// The instance to execute on. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the first column of the first row in the result set as an Int64, or -1 if no rows are returned or the value cannot be converted. + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, CancellationToken cancellationToken) + { + return await ExecuteScalarInt64Async(self, true, null, null, -1, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64. + /// + /// The instance to execute on. + /// The default value to return if no rows are returned or the value cannot be converted. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the first column of the first row in the result set as an Int64, or -1 if no rows are returned or the value cannot be converted. + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, long defaultvalue, CancellationToken cancellationToken) + { + return await ExecuteScalarInt64Async(self, true, null, null, defaultvalue, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64. + /// + /// The instance to execute on. + /// Whether to write a log entry. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the first column of the first row in the result set as an Int64, or -1 if no rows are returned or the value cannot be converted. + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, CancellationToken cancellationToken) + { + return await ExecuteScalarInt64Async(self, writeLog, null, null, -1, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64. + /// + /// The instance to execute on. + /// Whether to write a log entry. + /// The default value to return if no rows are returned or the value cannot be converted. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted. + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, long defaultvalue, CancellationToken cancellationToken) + { + return await ExecuteScalarInt64Async(self, writeLog, null, null, defaultvalue, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64. + /// + /// The instance to execute on. + /// The command text to execute. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted. + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, string? cmd, CancellationToken cancellationToken) + { + return await ExecuteScalarInt64Async(self, true, cmd, null, -1, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64. + /// + /// The instance to execute on. + /// The command text to execute. + /// The default value to return if no rows are returned or the value cannot be converted. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted. + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, string? cmd, long defaultvalue, CancellationToken cancellationToken) + { + return await ExecuteScalarInt64Async(self, true, cmd, null, defaultvalue, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64. + /// + /// The instance to execute on. + /// The command text to execute. + /// The values to use as parameters. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted. + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, string? cmd, Dictionary? values, CancellationToken cancellationToken) + { + return await ExecuteScalarInt64Async(self, true, cmd, values, -1, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64. + /// + /// The instance to execute on. + /// The command text to execute. + /// The values to use as parameters. + /// The default value to return if no rows are returned or the value cannot be converted. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted. + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, string? cmd, Dictionary? values, long defaultvalue, CancellationToken cancellationToken) + { + return await ExecuteScalarInt64Async(self, true, cmd, values, defaultvalue, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64. + /// + /// The instance to execute on. + /// Whether to write a log entry. + /// The command text to execute. + /// The values to set as parameters. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted. + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, string? cmd, Dictionary? values, CancellationToken cancellationToken) + { + return await ExecuteScalarInt64Async(self, writeLog, cmd, values, -1, cancellationToken) + .ConfigureAwait(false); + } + + /// + /// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64. + /// + /// The instance to execute on. + /// Whether to write a log entry. + /// The command text to execute. + /// The values to set as parameters. + /// The default value to return if no rows are returned or the value cannot be converted. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted. + public static async Task ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, string? cmd, Dictionary? values, long defaultvalue, CancellationToken cancellationToken) + { + if (cmd != null) + self.SetCommandAndParameters(cmd); + + if (values != null && values.Count > 0) + self.SetParameterValues(values); + + using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalarInt64Async", string.Format("ExecuteScalarInt64Async: {0}", self.GetPrintableCommandText())) : null) + await using (var rd = await self.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false)) + if (await rd.ReadAsync(cancellationToken).ConfigureAwait(false)) + return ConvertValueToInt64(rd, 0, defaultvalue); + + return defaultvalue; + } + + /// + /// Expands an IN clause parameter in a SqliteCommand by replacing the parameter with multiple parameters for each value in the collection. + /// + /// The to expand the IN clause parameter for. + /// The name of the original parameter to replace, must start with '@'. + /// The collection of values to expand the IN clause with. + /// The type of the values in the collection. + /// The modified with the IN clause expanded. + public static SqliteCommand ExpandInClauseParameterMssqlite(this SqliteCommand cmd, string originalParamName, IEnumerable values) + { + if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@")) + throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName)); + + foreach (var p in cmd.Parameters) + if (p is IDataParameter parameter && parameter.ParameterName.Equals(originalParamName, StringComparison.OrdinalIgnoreCase)) + { + cmd.Parameters.Remove(parameter); + break; + } + + foreach ((var value, var index) in values.Select((value, index) => (value, index))) + cmd.AddNamedParameter($"{originalParamName}{index}", value); + + var inClause = string.Join(", ", values.Select((_, index) => $"{originalParamName}{index}")); + if (string.IsNullOrWhiteSpace(inClause) && values.Any()) + throw new ArgumentException("IN clause cannot be empty", nameof(values)); + +#if DEBUG + if (!cmd.CommandText.Contains(originalParamName, StringComparison.OrdinalIgnoreCase)) + throw new ArgumentException($"Command text does not contain parameter '{originalParamName}'", nameof(originalParamName)); +#endif + cmd.CommandText = cmd.CommandText.Replace(originalParamName, inClause, StringComparison.OrdinalIgnoreCase); + + return cmd; + } + + /// + /// Expands an IN clause parameter in a SqliteCommand by replacing the parameter with multiple parameters for each value in the collection. + /// If a temporary table is used, it replaces the parameter with the table name. + /// + /// The to expand the IN clause parameter for. + /// The name of the original parameter to replace, must start with '@'. + /// The collection of values to expand the IN clause with. + /// Cancellation token to cancel the operation. + /// A task that when awaited contains the modified with the IN clause expanded. + internal static async Task ExpandInClauseParameterMssqliteAsync(this SqliteCommand cmd, string originalParamName, TemporaryDbValueList values, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@")) + throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName)); + + if (!values.IsTableCreated) + return ExpandInClauseParameterMssqlite(cmd, originalParamName, values.Values); + + // We have a temporary table, so we need to replace the parameter with the table name + cmd.CommandText = cmd.CommandText.Replace( + originalParamName, + await values.GetInClause(cancellationToken).ConfigureAwait(false), + StringComparison.OrdinalIgnoreCase + ); + + return cmd; + } + + /// + /// Sets the command text and parameters for the given . + /// + /// The to set the command text and parameters for. + /// The command text to set for the command, must not contain '?' as a parameter placeholder. + /// The modified with the command text and parameters set. + /// If the command text contains '?' as a parameter placeholder. + public static SqliteCommand SetCommandAndParameters(this SqliteCommand cmd, string cmdtext) + { + cmd.CommandText = Library.Utility.Utility.FormatInvariant(cmdtext); + cmd.Parameters.Clear(); + +#if DEBUG + if (cmd.CommandText.Contains('?', StringComparison.OrdinalIgnoreCase)) + throw new ArgumentException("Command text cannot contain '?' as a parameter placeholder, use '@' instead.", nameof(cmdtext)); +#endif + + var parameters = MyRegex().Matches(cmdtext); + var found = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (Match match in parameters) + { + if (found.Contains(match.Value)) + continue; + found.Add(match.Value); + cmd.AddNamedParameter(match.Value); + } + + return cmd; + } + + /// + /// Sets the value of a parameter in the command. + /// + /// The to set the parameter value for. + /// The name of the parameter to set the value for. + /// The value to set for the parameter, or null to set it to DBNull.Value. + /// The with the parameter value set. + public static SqliteCommand SetParameterValue(this SqliteCommand self, string name, object? value) + { +#if DEBUG + if (value is not null && value is System.Collections.IEnumerable && value is not string) + throw new ArgumentException($"Cannot set parameter '{name}' to an array or enumerable type, as the SQLite bindings does not support it.", nameof(value)); +#endif + self.Parameters[name].Value = value ?? DBNull.Value; + + return self; + } + + // Special case for DateTime, as we need to convert it to a long + /// + /// Sets the value of a DateTime parameter in the command. + /// + /// The to set the parameter value for. + /// The name of the parameter to set the value for. + /// The DateTime value to set for the parameter. + /// The with the parameter value set. + public static SqliteCommand SetParameterValue(this SqliteCommand self, string name, DateTime value) + { + self.Parameters[name].Value = Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(value); + + return self; + } + + /// + /// Sets the parameter values for the command, the parameters must already be added. + /// + /// The to set the parameter values for. + /// The values to set the parameters to, where the key is the parameter name. + /// The with the parameter values set. + public static SqliteCommand SetParameterValues(this SqliteCommand self, Dictionary values) + { + foreach (var kvp in values) + self.Parameters[kvp.Key].Value = kvp.Value; + + return self; + } + + /// + /// Sets the transaction for the command. + /// + /// The to set the transaction for. + /// The to set for the command. + /// The with the transaction set. + public static SqliteCommand SetTransaction(this SqliteCommand self, SqliteTransaction transaction) + { + self.Transaction = transaction; + + return self; + } + + /// + /// Sets the transaction for the command using a . + /// + /// The to set the transaction for. + /// The to set for the command. + /// The with the transaction set. + internal static SqliteCommand SetTransaction(this SqliteCommand self, ReusableTransaction rtr) + { + self.Transaction = rtr.Transaction; + + return self; + } + /// /// The tag used for logging /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(ExtensionMethods)); + // This regex matches named parameters in the form of @paramName + [GeneratedRegex(@"@\w+")] + private static partial Regex MyRegex(); + /// /// Adds a parameter to the command with the given value. /// @@ -46,8 +711,28 @@ public static class ExtensionMethods /// The name of the parameter /// The optional value of the parameter /// The command with the parameter added - public static T AddNamedParameter(this T self, string name, object? value = null) - where T : IDbCommand + public static SqliteCommand AddNamedParameter(this SqliteCommand self, string name, object? value = null) + { + var p = self.CreateParameter(); + p.ParameterName = name; + if (value != null) + p.Value = value; + self.Parameters.Add(p); + return self; + } + + // TODO At some point, when the entire codebase is migrated to + // Microsoft.Data.Sqlite, the extension methods using IDb* should be + // removed. Currently, they're kept for compatibility. + + /// + /// Adds a named parameter to the command with the given value. + /// + /// The command to add the parameter to + /// The name of the parameter + /// The optional value of the parameter + /// The command with the parameter added + public static IDbCommand AddNamedParameter(this IDbCommand self, string name, object? value = null) { var p = self.CreateParameter(); p.ParameterName = name; @@ -97,7 +782,7 @@ public static class ExtensionMethods /// The type of the command /// The command to set the transaction for /// The transaction to set for the command - /// >The command with the transaction set + /// The command with the transaction set public static T SetTransaction(this T self, IDbTransaction? transaction) where T : IDbCommand { @@ -114,21 +799,27 @@ public static class ExtensionMethods { var txt = self.CommandText; - foreach (var p in self.Parameters.Cast()) + // Replace each named parameter with its value + foreach (IDataParameter p in self.Parameters) { - var ix = txt.IndexOf('?'); - if (ix >= 0) - { - string v; - if (p.Value is string) - v = string.Format("\"{0}\"", p.Value); - else if (p.Value == null) - v = "NULL"; - else - v = string.Format("{0}", p.Value); + var paramName = p.ParameterName; + if (string.IsNullOrEmpty(paramName)) + continue; - txt = txt.Substring(0, ix) + v + txt.Substring(ix + 1); - } + string v; + if (p.Value == null || p.Value == DBNull.Value) + v = "NULL"; + else if (p.Value is string) + v = $"\"{p.Value}\""; + else if (p.Value is DateTime dt) + v = $"\"{dt:O}\""; + else if (p.Value is byte[] bytes) + v = $"X'{BitConverter.ToString(bytes).Replace("-", "")}'"; + else + v = p.Value.ToString() ?? "NULL"; + + // Replace all occurrences of the parameter (with word boundary) + txt = Regex.Replace(txt, $@"\B{Regex.Escape(paramName)}\b", v, RegexOptions.IgnoreCase); } return txt; @@ -500,15 +1191,17 @@ public static class ExtensionMethods /// /// The connection to create the transaction on /// The transaction - public static IDbTransaction BeginTransactionSafe(this IDbConnection self) - { - var transaction = self.BeginTransaction(); - // using (var cmd = self.CreateCommand(transaction, "BEGIN IMMEDIATE;")) - // try { cmd.ExecuteNonQuery(); } - // catch { /* ignore */ } - - return transaction; - } + //public static IDbTransaction BeginTransactionSafe(this IDbConnection self) + //{ + // var transaction = self.BeginTransaction(); + // // using (var cmd = self.CreateCommand(transaction, "BEGIN IMMEDIATE;")) + // // try { cmd.ExecuteNonQuery(); } + // // catch { + // // ignore + // } + // + // return transaction; + //} /// @@ -528,8 +1221,7 @@ public static class ExtensionMethods /// The command to expand the parameter for /// The original parameter name to expand /// The values to expand the parameter for - public static TCommand ExpandInClauseParameter(this TCommand cmd, string originalParamName, IEnumerable values) - where TCommand : IDbCommand + public static IDbCommand ExpandInClauseParameter(this IDbCommand cmd, string originalParamName, IEnumerable values) { if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@")) throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName)); @@ -564,8 +1256,7 @@ public static class ExtensionMethods /// The command to expand the parameter for /// The original parameter name to expand /// The values to expand the parameter for - public static T ExpandInClauseParameter(this T cmd, string originalParamName, TemporaryDbValueList values) - where T : IDbCommand + internal static IDbCommand ExpandInClauseParameter(this IDbCommand cmd, string originalParamName, TemporaryDbValueList values) { if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@")) throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName)); @@ -574,7 +1265,8 @@ public static class ExtensionMethods return ExpandInClauseParameter(cmd, originalParamName, values.Values); // We have a temporary table, so we need to replace the parameter with the table name - cmd.CommandText = cmd.CommandText.Replace(originalParamName, values.GetInClause(), StringComparison.OrdinalIgnoreCase); + cmd.CommandText = cmd.CommandText.Replace(originalParamName, values.GetInClause(default).Await(), StringComparison.OrdinalIgnoreCase); return cmd; } + } \ No newline at end of file diff --git a/Duplicati/Library/Main/Database/IRemoteVolume.cs b/Duplicati/Library/Main/Database/IRemoteVolume.cs index b5047e2dc..9cb24f49c 100644 --- a/Duplicati/Library/Main/Database/IRemoteVolume.cs +++ b/Duplicati/Library/Main/Database/IRemoteVolume.cs @@ -1,29 +1,41 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. namespace Duplicati.Library.Main.Database { + /// + /// Interface for a remote volume, containing its name, hash, and size. + /// public interface IRemoteVolume { + /// + /// Gets the name of the remote volume. + /// string Name { get; } + /// + /// Gets the hash of the remote volume. + /// string Hash { get; } + /// + /// Gets the size of the remote volume in bytes. + /// long Size { get; } } } diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 8e1918bcc..d28425ac5 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -1,66 +1,130 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. + #nullable enable using System; using System.Collections.Generic; -using System.Data; -using System.Globalization; using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.Utility; +using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database { + /// + /// A local backup database that stores blocks, files, and metadata for backup operations. + /// This database is used to track the state of backups and to allow for efficient retrieval of blocks and files. + /// It supports operations such as finding blocks by hash and size, inserting new blocks, files, and blocksets, + /// and managing metadata datasets. + /// internal class LocalBackupDatabase : LocalDatabase { /// - /// The tag used for logging + /// The tag used for logging. /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(); - private readonly IDbCommand m_findblockCommand; - private readonly IDbCommand m_findblocksetCommand; - private readonly IDbCommand m_findfilesetCommand; - private readonly IDbCommand m_findmetadatasetCommand; + /// + /// The command used to find a block by its hash and size. + /// + private SqliteCommand m_findblockCommand = null!; + /// + /// The command used to find a blockset by its full hash and length. + /// + private SqliteCommand m_findblocksetCommand = null!; + /// + /// The command used to find a metadataset by its block hash and size. + /// + private SqliteCommand m_findfilesetCommand = null!; + /// + /// The command used to find a metadataset by its block hash and size. + /// + private SqliteCommand m_findmetadatasetCommand = null!; - private readonly IDbCommand m_insertblockCommand; + /// + /// The command used to insert a block into the database. + /// + private SqliteCommand m_insertblockCommand = null!; - private readonly IDbCommand m_insertfileCommand; + /// + /// The command used to insert a file into the database. + /// + private SqliteCommand m_insertfileCommand = null!; - private readonly IDbCommand m_insertblocksetCommand; - private readonly IDbCommand m_insertblocksetentryCommand; - private readonly IDbCommand m_insertblocklistHashesCommand; + /// + /// The command used to insert a blockset into the database. + /// + private SqliteCommand m_insertblocksetCommand = null!; + /// + /// The command used to insert a blockset entry into the database. + /// + private SqliteCommand m_insertblocksetentryCommand = null!; + /// + /// The command used to insert a blocklist hash into the database. + /// + private SqliteCommand m_insertblocklistHashesCommand = null!; - private readonly IDbCommand m_insertmetadatasetCommand; + /// + /// The command used to insert a metadataset into the database. + /// + private SqliteCommand m_insertmetadatasetCommand = null!; - private readonly IDbCommand m_findfileCommand; - private readonly IDbCommand m_selectfilelastmodifiedCommand; - private readonly IDbCommand m_selectfilelastmodifiedWithSizeCommand; - private readonly IDbCommand m_selectfileHashCommand; + /// + /// The command used to find a file in the database. + /// + private SqliteCommand m_findfileCommand = null!; + /// + /// The command used to select the last modified time of a file. + /// + private SqliteCommand m_selectfilelastmodifiedCommand = null!; + /// + /// The command used to select the last modified time and size of a file. + /// + private SqliteCommand m_selectfilelastmodifiedWithSizeCommand = null!; + /// + /// The command used to select the hash and size of a file's metadata. + /// + private SqliteCommand m_selectfileHashCommand = null!; - private readonly IDbCommand m_insertfileOperationCommand; - private readonly IDbCommand m_selectfilemetadatahashandsizeCommand; - private readonly IDbCommand m_getfirstfilesetwithblockinblockset; + /// + /// The command used to insert a file operation into the database. + /// + private SqliteCommand m_insertfileOperationCommand = null!; + /// + /// The command used to select the metadata hash and size of a file. + /// + private SqliteCommand m_selectfilemetadatahashandsizeCommand = null!; + /// + /// The command used to find the first fileset with a block in a blockset. + /// + private SqliteCommand m_getfirstfilesetwithblockinblockset = null!; - private HashSet m_blocklistHashes; + /// + /// HashSet of blocklist hashes to track whether a blocklist hash has been seen before. + /// + private HashSet m_blocklistHashes = []; /// /// The temporary table with deleted blocks that can be re-used; null if not table is used @@ -73,41 +137,262 @@ namespace Duplicati.Library.Main.Database /// /// The command used to move deleted blocks to the main block table; null if not used /// - private readonly IDbCommand? m_moveblockfromdeletedCommand; + private SqliteCommand? m_moveblockfromdeletedCommand; /// /// The command used to find blocks in the deleted blocks table; null if not used /// - private readonly IDbCommand? m_findindeletedCommand; + private SqliteCommand? m_findindeletedCommand; + /// + /// The ID of the fileset currently being processed. + /// private long m_filesetId; - private readonly bool m_logQueries; + /// + /// Indicates whether the database should log queries for profiling purposes. + /// + private bool m_logQueries; - public LocalBackupDatabase(string path, Options options) - : this(new LocalDatabase(path, "Backup", false, options.SqlitePageCache), options) + /// + /// Initializes a new instance of the class. + /// This constructor is private to enforce the use of the static and asynchronous CreateAsync methods for instantiation. + /// + /// The path to the database file. + /// The Duplicati options used by the current operation. + /// An optional existing instance of to reuse. Used when derived classes need to call the base constructor. + /// The cancellation token to monitor for cancellation requests. + /// A task that when awaited contains a new instance of . + public static async Task CreateAsync(string path, Options options, CancellationToken token, LocalBackupDatabase? dbnew = null) { - this.ShouldCloseConnection = true; + dbnew ??= new LocalBackupDatabase(); + + dbnew = (LocalBackupDatabase) + await CreateLocalDatabaseAsync(path, "Backup", false, dbnew, token) + .ConfigureAwait(false); + dbnew = await CreateAsync(dbnew, options, null, token).ConfigureAwait(false); + dbnew.ShouldCloseConnection = true; + + return dbnew; } - public LocalBackupDatabase(LocalDatabase db, Options options) - : base(db) + /// + /// Creates a new instance of using an existing parent database. + /// This method is used to create a new backup database based on an existing local database. + /// + /// The parent local database from which to create the new backup database. + /// The Duplicati options used by the current operation. + /// An optional existing instance of to reuse. Used when derived classes need to call the base constructor. + /// The cancellation token to monitor for cancellation requests. + /// A task that when awaited contains a new instance of . + public static async Task CreateAsync(LocalDatabase dbparent, Options options, LocalBackupDatabase? dbnew, CancellationToken token) { - m_logQueries = options.ProfileAllDatabaseQueries; + dbnew ??= new LocalBackupDatabase(); - m_findblockCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @hash AND ""Size"" = @Size"); - m_findblocksetCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Fullhash"" = @Fullhash AND ""Length"" = @Length"); - m_findmetadatasetCommand = m_connection.CreateCommand(@"SELECT ""A"".""ID"" FROM ""Metadataset"" A, ""BlocksetEntry"" B, ""Block"" C WHERE ""A"".""BlocksetID"" = ""B"".""BlocksetID"" AND ""B"".""BlockID"" = ""C"".""ID"" AND ""C"".""Hash"" = @Hash AND ""C"".""Size"" = @Size"); - m_findfilesetCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""FileLookup"" WHERE ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId AND ""Path"" = @Path AND ""PrefixID"" = @PrefixId"); - m_insertblockCommand = m_connection.CreateCommand(@"INSERT INTO ""Block"" (""Hash"", ""VolumeID"", ""Size"") VALUES (@Hash, @VolumeId, @Size); SELECT last_insert_rowid();"); - m_insertfileOperationCommand = m_connection.CreateCommand(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (@FilesetId, @FileId, @LastModified)"); - m_insertfileCommand = m_connection.CreateCommand(@"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"",""BlocksetID"", ""MetadataID"") VALUES (@PrefixId, @Path, @BlocksetId, @MetadataId); SELECT last_insert_rowid();"); - m_insertblocksetCommand = m_connection.CreateCommand(@"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (@Length, @Fullhash); SELECT last_insert_rowid();"); - m_insertblocksetentryCommand = m_connection.CreateCommand(@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") SELECT @BlocksetId AS A, @Index AS B, ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); - m_insertblocklistHashesCommand = m_connection.CreateCommand(@"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (@BlocksetId, @Index, @Hash)"); - m_insertmetadatasetCommand = m_connection.CreateCommand(@"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (@BlocksetId); SELECT last_insert_rowid();"); - m_selectfilelastmodifiedCommand = m_connection.CreateCommand(@"SELECT ""A"".""ID"", ""B"".""LastModified"" FROM (SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = @FilesetId"); - m_selectfilelastmodifiedWithSizeCommand = m_connection.CreateCommand(@"SELECT ""C"".""ID"", ""C"".""LastModified"", ""D"".""Length"" FROM (SELECT ""A"".""ID"", ""B"".""LastModified"", ""A"".""BlocksetID"" FROM (SELECT ""ID"", ""BlocksetID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = @FilesetId) AS ""C"", ""Blockset"" AS ""D"" WHERE ""C"".""BlocksetID"" == ""D"".""ID"" "); - m_selectfilemetadatahashandsizeCommand = m_connection.CreateCommand(@"SELECT ""Blockset"".""Length"", ""Blockset"".""FullHash"" FROM ""Blockset"", ""Metadataset"", ""File"" WHERE ""File"".""ID"" = @FileId AND ""Blockset"".""ID"" = ""Metadataset"".""BlocksetID"" AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" "); + dbnew = (LocalBackupDatabase)await CreateLocalDatabaseAsync(dbparent, dbnew, token) + .ConfigureAwait(false); + + dbnew.m_logQueries = options.ProfileAllDatabaseQueries; + + dbnew.m_findblockCommand = await dbnew.Connection.CreateCommandAsync(@" + SELECT ""ID"" + FROM ""Block"" + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size + ", token) + .ConfigureAwait(false); + + dbnew.m_findblocksetCommand = await dbnew.Connection.CreateCommandAsync(@" + SELECT ""ID"" + FROM ""Blockset"" + WHERE + ""Fullhash"" = @Fullhash + AND ""Length"" = @Length + ", token) + .ConfigureAwait(false); + + dbnew.m_findmetadatasetCommand = await dbnew.Connection.CreateCommandAsync(@" + SELECT ""A"".""ID"" + FROM + ""Metadataset"" ""A"", + ""BlocksetEntry"" ""B"", + ""Block"" ""C"" + WHERE + ""A"".""BlocksetID"" = ""B"".""BlocksetID"" + AND ""B"".""BlockID"" = ""C"".""ID"" + AND ""C"".""Hash"" = @Hash + AND ""C"".""Size"" = @Size + ", token) + .ConfigureAwait(false); + + dbnew.m_findfilesetCommand = await dbnew.Connection.CreateCommandAsync(@" + SELECT ""ID"" + FROM ""FileLookup"" + WHERE + ""BlocksetID"" = @BlocksetId + AND ""MetadataID"" = @MetadataId + AND ""Path"" = @Path + AND ""PrefixID"" = @PrefixId + ", token) + .ConfigureAwait(false); + + dbnew.m_insertblockCommand = await dbnew.Connection.CreateCommandAsync(@" + INSERT INTO ""Block"" ( + ""Hash"", + ""VolumeID"", + ""Size"" + ) + VALUES ( + @Hash, + @VolumeId, + @Size + ); + SELECT last_insert_rowid(); + ", token) + .ConfigureAwait(false); + + dbnew.m_insertfileOperationCommand = await dbnew.Connection.CreateCommandAsync(@" + INSERT INTO ""FilesetEntry"" ( + ""FilesetID"", + ""FileID"", + ""Lastmodified"" + ) + VALUES ( + @FilesetId, + @FileId, + @LastModified + ) + ", token) + .ConfigureAwait(false); + + dbnew.m_insertfileCommand = await dbnew.Connection.CreateCommandAsync(@" + INSERT INTO ""FileLookup"" ( + ""PrefixID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"" + ) + VALUES ( + @PrefixId, + @Path, + @BlocksetId, + @MetadataId + ); + SELECT last_insert_rowid(); + ", token) + .ConfigureAwait(false); + + dbnew.m_insertblocksetCommand = await dbnew.Connection.CreateCommandAsync(@" + INSERT INTO ""Blockset"" ( + ""Length"", + ""FullHash"" + ) + VALUES ( + @Length, + @Fullhash + ); + SELECT last_insert_rowid(); + ", token) + .ConfigureAwait(false); + + dbnew.m_insertblocksetentryCommand = await dbnew.Connection.CreateCommandAsync(@" + INSERT INTO ""BlocksetEntry"" ( + ""BlocksetID"", + ""Index"", + ""BlockID"" + ) + SELECT + @BlocksetId AS ""A"", + @Index AS ""B"", + ""ID"" + FROM ""Block"" + WHERE ""Hash"" = @Hash + AND ""Size"" = @Size + ", token) + .ConfigureAwait(false); + + dbnew.m_insertblocklistHashesCommand = await dbnew.Connection.CreateCommandAsync(@" + INSERT INTO ""BlocklistHash"" ( + ""BlocksetID"", + ""Index"", + ""Hash"" + ) + VALUES ( + @BlocksetId, + @Index, + @Hash + ) + ", token) + .ConfigureAwait(false); + + dbnew.m_insertmetadatasetCommand = await dbnew.Connection.CreateCommandAsync(@" + INSERT INTO ""Metadataset"" (""BlocksetID"") + VALUES (@BlocksetId); + SELECT last_insert_rowid(); + ", token) + .ConfigureAwait(false); + + dbnew.m_selectfilelastmodifiedCommand = await dbnew.Connection.CreateCommandAsync(@" + SELECT + ""A"".""ID"", + ""B"".""LastModified"" + FROM ( + SELECT ""ID"" + FROM ""FileLookup"" + WHERE ""PrefixID"" = @PrefixId + AND ""Path"" = @Path + ) ""A"" + CROSS JOIN ""FilesetEntry"" ""B"" + WHERE + ""A"".""ID"" = ""B"".""FileID"" + AND ""B"".""FilesetID"" = @FilesetId + ", token) + .ConfigureAwait(false); + + dbnew.m_selectfilelastmodifiedWithSizeCommand = await dbnew.Connection.CreateCommandAsync(@" + SELECT + ""C"".""ID"", + ""C"".""LastModified"", + ""D"".""Length"" + FROM + ( + SELECT + ""A"".""ID"", + ""B"".""LastModified"", + ""A"".""BlocksetID"" + FROM ( + SELECT + ""ID"", + ""BlocksetID"" + FROM ""FileLookup"" + WHERE + ""PrefixID"" = @PrefixId + AND ""Path"" = @Path + ) ""A"" + CROSS JOIN ""FilesetEntry"" ""B"" + WHERE + ""A"".""ID"" = ""B"".""FileID"" + AND ""B"".""FilesetID"" = @FilesetId + ) AS ""C"", + ""Blockset"" AS ""D"" + WHERE ""C"".""BlocksetID"" == ""D"".""ID"" + ", token) + .ConfigureAwait(false); + + dbnew.m_selectfilemetadatahashandsizeCommand = await dbnew.Connection.CreateCommandAsync(@" + SELECT + ""Blockset"".""Length"", + ""Blockset"".""FullHash"" + FROM + ""Blockset"", + ""Metadataset"", + ""File"" + WHERE + ""File"".""ID"" = @FileId + AND ""Blockset"".""ID"" = ""Metadataset"".""BlocksetID"" + AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" + ", token) + .ConfigureAwait(false); // Experimental toggling of the deleted block cache // If the value is less than zero, the lookup is disabled @@ -121,63 +406,141 @@ namespace Duplicati.Library.Main.Database if (deletedBlockCacheSizeLong >= 0) { - using (var cmd = m_connection.CreateCommand()) + await using var cmd = dbnew.Connection.CreateCommand(); + dbnew.m_tempDeletedBlockTable = "DeletedBlock-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + await cmd.SetCommandAndParameters($@" + CREATE TEMPORARY TABLE ""{dbnew.m_tempDeletedBlockTable}"" AS + SELECT + MAX(""ID"") AS ""ID"", + ""Hash"", + ""Size"" + FROM ""DeletedBlock"" + WHERE ""VolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""State"" NOT IN (@States) + ) + GROUP BY + ""Hash"", + ""Size"" + ") + .ExpandInClauseParameterMssqlite("@States", [ + RemoteVolumeState.Deleted, + RemoteVolumeState.Deleting + ]) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + var deletedBlocks = await cmd.ExecuteScalarInt64Async(@$" + SELECT COUNT(*) + FROM ""{dbnew.m_tempDeletedBlockTable}"" + ", 0, token) + .ConfigureAwait(false); + + // There are no deleted blocks, so we can drop the table + if (deletedBlocks == 0) { - m_tempDeletedBlockTable = "DeletedBlock-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - cmd.SetCommandAndParameters(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tempDeletedBlockTable}"" AS ") - + @$"SELECT MAX(""ID"") AS ""ID"", ""Hash"", ""Size"" FROM ""DeletedBlock"" WHERE ""VolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""State"" NOT IN (@States)) GROUP BY ""Hash"", ""Size""") - .ExpandInClauseParameter("@States", [RemoteVolumeState.Deleted, RemoteVolumeState.Deleting]) - .ExecuteNonQuery(); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{dbnew.m_tempDeletedBlockTable}""", token) + .ConfigureAwait(false); + dbnew.m_tempDeletedBlockTable = null; - var deletedBlocks = cmd.ExecuteScalarInt64(FormatInvariant(@$"SELECT COUNT(*) FROM ""{m_tempDeletedBlockTable}"""), 0); + } + // The deleted blocks are small enough to fit in memory + else if (deletedBlocks <= deletedBlockCacheSizeLong) + { + dbnew.m_deletedBlockLookup = new Dictionary>(); + cmd.SetCommandAndParameters(@$" + SELECT + ""ID"", + ""Hash"", + ""Size"" + FROM ""{dbnew.m_tempDeletedBlockTable}"" + ") + .ConfigureAwait(false); - // There are no deleted blocks, so we can drop the table - if (deletedBlocks == 0) - { - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE ""{m_tempDeletedBlockTable}""")); - m_tempDeletedBlockTable = null; + await using (var reader = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + while (await reader.ReadAsync(token).ConfigureAwait(false)) + { + var id = reader.ConvertValueToInt64(0); + var hash = reader.ConvertValueToString(1) ?? throw new Exception("Hash is null"); + var size = reader.ConvertValueToInt64(2); - } - // The deleted blocks are small enough to fit in memory - else if (deletedBlocks <= deletedBlockCacheSizeLong) - { - m_deletedBlockLookup = new Dictionary>(); - using (var reader = cmd.ExecuteReader(FormatInvariant(@$"SELECT ""ID"", ""Hash"", ""Size"" FROM ""{m_tempDeletedBlockTable}"""))) - while (reader.Read()) - { - var id = reader.ConvertValueToInt64(0); - var hash = reader.ConvertValueToString(1) ?? throw new Exception("Hash is null"); - var size = reader.ConvertValueToInt64(2); + if (!dbnew.m_deletedBlockLookup.TryGetValue(hash, out var sizes)) + dbnew.m_deletedBlockLookup[hash] = sizes = new Dictionary(); + sizes[size] = id; + } - if (!m_deletedBlockLookup.TryGetValue(hash, out var sizes)) - m_deletedBlockLookup[hash] = sizes = new Dictionary(); - sizes[size] = id; - } + await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{dbnew.m_tempDeletedBlockTable}""", token) + .ConfigureAwait(false); + dbnew.m_tempDeletedBlockTable = null; + } + // The deleted blocks are too large to fit in memory, so we use a temporary table + else + { + await cmd.ExecuteNonQueryAsync($@" + CREATE UNIQUE INDEX ""unique_{dbnew.m_tempDeletedBlockTable}"" + ON ""{dbnew.m_tempDeletedBlockTable}"" ( + ""Hash"", + ""Size"" + ) + ", token) + .ConfigureAwait(false); - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE ""{m_tempDeletedBlockTable}""")); - m_tempDeletedBlockTable = null; - } - // The deleted blocks are too large to fit in memory, so we use a temporary table - else - { - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE UNIQUE INDEX ""unique_{m_tempDeletedBlockTable}"" ON ""{m_tempDeletedBlockTable}"" (""Hash"", ""Size"")")); - m_findindeletedCommand = m_connection.CreateCommand(FormatInvariant($@"SELECT ""ID"" FROM ""{m_tempDeletedBlockTable}"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size")); - m_moveblockfromdeletedCommand = m_connection.CreateCommand(string.Join(";", - @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId LIMIT 1", - @"DELETE FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId", - FormatInvariant(@$"DELETE FROM {m_tempDeletedBlockTable} WHERE ""ID"" = @DeletedBlockId"), - "SELECT last_insert_rowid()")); + dbnew.m_findindeletedCommand = await dbnew.Connection.CreateCommandAsync($@" + SELECT ""ID"" + FROM ""{dbnew.m_tempDeletedBlockTable}"" + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size + ", token) + .ConfigureAwait(false); - } + dbnew.m_moveblockfromdeletedCommand = await dbnew.Connection.CreateCommandAsync(@$" + INSERT INTO ""Block"" ( + ""Hash"", + ""Size"", + ""VolumeID"" + ) + SELECT + ""Hash"", + ""Size"", + ""VolumeID"" + FROM ""DeletedBlock"" + WHERE ""ID"" = @DeletedBlockId LIMIT 1; - if (deletedBlocks > 0) - { - m_moveblockfromdeletedCommand = m_connection.CreateCommand(string.Join(";", - @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId LIMIT 1", - @"DELETE FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId", - "SELECT last_insert_rowid()")); - } + DELETE FROM ""DeletedBlock"" + WHERE ""ID"" = @DeletedBlockId; + DELETE FROM ""{dbnew.m_tempDeletedBlockTable}"" + WHERE ""ID"" = @DeletedBlockId; + + SELECT last_insert_rowid() + ", token) + .ConfigureAwait(false); + + } + + if (deletedBlocks > 0) + { + dbnew.m_moveblockfromdeletedCommand = await dbnew.m_connection.CreateCommandAsync(@$" + INSERT INTO ""Block"" ( + ""Hash"", + ""Size"", + ""VolumeID"" + ) + SELECT + ""Hash"", + ""Size"", + ""VolumeID"" + FROM ""DeletedBlock"" + WHERE ""ID"" = @DeletedBlockId LIMIT 1; + + DELETE FROM ""DeletedBlock"" + WHERE ""ID"" = @DeletedBlockId; + + SELECT last_insert_rowid() + ", token) + .ConfigureAwait(false); } } @@ -188,13 +551,13 @@ namespace Duplicati.Library.Main.Database if (testqueryversion != 0) Logging.Log.WriteWarningMessage(LOGTAG, "TestFileQuery", null, "Using performance test query version {0} as the TEST_QUERY_VERSION environment variable is set", testqueryversion); - // The original query (v==1) finds the most recent entry of the file in question, + // The original query (v==1) finds the most recent entry of the file in question, // but it requires some large joins to extract the required information. // To speed it up, we use a slightly simpler approach that only looks at the // previous fileset, and uses information here. // If there is a case where a file is sometimes there and sometimes not // (i.e. filter file, remove filter) we will not find the file. - // We currently use this faster version, + // We currently use this faster version, // but allow users to switch back via an environment variable // such that we can get performance feedback @@ -203,97 +566,197 @@ namespace Duplicati.Library.Main.Database { // The query used in Duplicati until 2.0.3.9 case 1: - findQuery = - @" SELECT ""FileLookup"".""ID"" AS ""FileID"", ""FilesetEntry"".""Lastmodified"", ""FileBlockset"".""Length"", ""MetaBlockset"".""Fullhash"" AS ""Metahash"", ""MetaBlockset"".""Length"" AS ""Metasize"" " + - @" FROM ""FileLookup"", ""FilesetEntry"", ""Fileset"", ""Blockset"" ""FileBlockset"", ""Metadataset"", ""Blockset"" ""MetaBlockset"" " + - @" WHERE ""FileLookup"".""PrefixID"" = @PrefixId AND ""FileLookup"".""Path"" = @Path " + - @" AND ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" " + - @" AND ""FileBlockset"".""ID"" = ""FileLookup"".""BlocksetID"" " + - @" AND ""Metadataset"".""ID"" = ""FileLookup"".""MetadataID"" AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"" " + - @" AND @FilesetId IS NOT NULL" + - @" ORDER BY ""Fileset"".""Timestamp"" DESC " + - @" LIMIT 1 "; + findQuery = @" + SELECT + ""FileLookup"".""ID"" AS ""FileID"", + ""FilesetEntry"".""Lastmodified"", + ""FileBlockset"".""Length"", + ""MetaBlockset"".""Fullhash"" AS ""Metahash"", + ""MetaBlockset"".""Length"" AS ""Metasize"" + FROM + ""FileLookup"", + ""FilesetEntry"", + ""Fileset"", + ""Blockset"" ""FileBlockset"", + ""Metadataset"", + ""Blockset"" ""MetaBlockset"" + WHERE + ""FileLookup"".""PrefixID"" = @PrefixId + AND ""FileLookup"".""Path"" = @Path + AND ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" + AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" + AND ""FileBlockset"".""ID"" = ""FileLookup"".""BlocksetID"" + AND ""Metadataset"".""ID"" = ""FileLookup"".""MetadataID"" + AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"" + AND @FilesetId IS NOT NULL + ORDER BY ""Fileset"".""Timestamp"" DESC + LIMIT 1 + "; break; // The fastest reported query in Duplicati 2.0.3.10, but with "LIMIT 1" added default: case 2: - var getLastFileEntryForPath = - @"SELECT ""A"".""ID"", ""B"".""LastModified"", ""A"".""BlocksetID"", ""A"".""MetadataID"" " + - @" FROM (SELECT ""ID"", ""BlocksetID"", ""MetadataID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path) ""A"" " + - @" CROSS JOIN ""FilesetEntry"" ""B"" " + - @" WHERE ""A"".""ID"" = ""B"".""FileID"" " + - @" AND ""B"".""FilesetID"" = @FilesetId "; + var getLastFileEntryForPath = @" + SELECT + ""A"".""ID"", + ""B"".""LastModified"", + ""A"".""BlocksetID"", + ""A"".""MetadataID"" + FROM ( + SELECT + ""ID"", + ""BlocksetID"", + ""MetadataID"" + FROM ""FileLookup"" + WHERE + ""PrefixID"" = @PrefixId + AND ""Path"" = @Path + ) ""A"" + CROSS JOIN ""FilesetEntry"" ""B"" + WHERE + ""A"".""ID"" = ""B"".""FileID"" + AND ""B"".""FilesetID"" = @FilesetId + "; - findQuery = FormatInvariant($@"SELECT ""C"".""ID"" AS ""FileID"", ""C"".""LastModified"", ""D"".""Length"", ""E"".""FullHash"" as ""Metahash"", ""E"".""Length"" AS ""Metasize"" -FROM ({getLastFileEntryForPath}) AS ""C"", ""Blockset"" AS ""D"", ""Blockset"" AS ""E"", ""Metadataset"" ""F"" -WHERE ""C"".""BlocksetID"" == ""D"".""ID"" AND ""C"".""MetadataID"" == ""F"".""ID"" AND ""F"".""BlocksetID"" = ""E"".""ID"" -LIMIT 1"); + findQuery = $@" + SELECT + ""C"".""ID"" AS ""FileID"", + ""C"".""LastModified"", + ""D"".""Length"", + ""E"".""FullHash"" as ""Metahash"", + ""E"".""Length"" AS ""Metasize"" + FROM + ({getLastFileEntryForPath}) AS ""C"", + ""Blockset"" AS ""D"", + ""Blockset"" AS ""E"", + ""Metadataset"" ""F"" + WHERE + ""C"".""BlocksetID"" == ""D"".""ID"" + AND ""C"".""MetadataID"" == ""F"".""ID"" + AND ""F"".""BlocksetID"" = ""E"".""ID"" + LIMIT 1 + "; break; // Potentially faster query: https://forum.duplicati.com/t/release-2-0-3-10-canary-2018-08-30/4497/25 case 3: - findQuery = - @" SELECT FileLookup.ID as FileID, FilesetEntry.Lastmodified, FileBlockset.Length, " + - @" MetaBlockset.FullHash AS Metahash, MetaBlockset.Length as Metasize " + - @" FROM FilesetEntry " + - @"INNER JOIN Fileset ON (FileSet.ID = FilesetEntry.FilesetID) " + - @"INNER JOIN FileLookup ON (FileLookup.ID = FilesetEntry.FileID) " + - @"INNER JOIN Metadataset ON (Metadataset.ID = FileLookup.MetadataID) " + - @"INNER JOIN Blockset AS MetaBlockset ON (MetaBlockset.ID = Metadataset.BlocksetID) " + - @" LEFT JOIN Blockset AS FileBlockset ON (FileBlockset.ID = FileLookup.BlocksetID) " + - @" WHERE FileLookup.PrefixID = @PrefixId AND FileLookup.Path = @Path AND FilesetID = @FilesetId " + - @" LIMIT 1 "; + findQuery = @" + SELECT + ""FileLookup"".""ID"" as ""FileID"", + ""FilesetEntry"".""Lastmodified"", + ""FileBlockset"".""Length"", + ""MetaBlockset"".""FullHash"" AS ""Metahash"", + ""MetaBlockset"".""Length"" as ""Metasize"" + FROM ""FilesetEntry"" + INNER JOIN ""Fileset"" + ON (""FileSet"".""ID"" = ""FilesetEntry"".FilesetID) + INNER JOIN ""FileLookup"" + ON (""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"") + INNER JOIN ""Metadataset"" + ON (""Metadataset"".""ID"" = ""FileLookup"".""MetadataID"") + INNER JOIN Blockset AS ""MetaBlockset"" + ON (""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"") + LEFT JOIN Blockset AS ""FileBlockset"" + ON (""FileBlockset"".""ID"" = ""FileLookup"".""BlocksetID"") + WHERE + ""FileLookup"".""PrefixID"" = @PrefixId + AND ""FileLookup"".""Path"" = @Path + AND FilesetID = @FilesetId + LIMIT 1 + "; break; // The slow query used in Duplicati 2.0.3.10, but with "LIMIT 1" added case 4: - findQuery = - @" SELECT ""FileLookup"".""ID"" AS ""FileID"", ""FilesetEntry"".""Lastmodified"", ""FileBlockset"".""Length"", ""MetaBlockset"".""Fullhash"" AS ""Metahash"", ""MetaBlockset"".""Length"" AS ""Metasize"" " + - @" FROM ""FileLookup"", ""FilesetEntry"", ""Fileset"", ""Blockset"" ""FileBlockset"", ""Metadataset"", ""Blockset"" ""MetaBlockset"" " + - @" WHERE ""FileLookup"".""PrefixID"" = @PrefixId AND ""FileLookup"".""Path"" = @Path " + - @" AND ""Fileset"".""ID"" = @FilesetId " + - @" AND ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" " + - @" AND ""FileBlockset"".""ID"" = ""FileLookup"".""BlocksetID"" " + - @" AND ""Metadataset"".""ID"" = ""FileLookup"".""MetadataID"" AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"" " + - @" LIMIT 1 "; + findQuery = @" + SELECT + ""FileLookup"".""ID"" AS ""FileID"", + ""FilesetEntry"".""Lastmodified"", + ""FileBlockset"".""Length"", + ""MetaBlockset"".""Fullhash"" AS ""Metahash"", + ""MetaBlockset"".""Length"" AS ""Metasize"" + FROM + ""FileLookup"", + ""FilesetEntry"", + ""Fileset"", + ""Blockset"" ""FileBlockset"", + ""Metadataset"", + ""Blockset"" ""MetaBlockset"" + WHERE + ""FileLookup"".""PrefixID"" = @PrefixId + AND ""FileLookup"".""Path"" = @Path + AND ""Fileset"".""ID"" = @FilesetId + AND ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" + AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" + AND ""FileBlockset"".""ID"" = ""FileLookup"".""BlocksetID"" + AND ""Metadataset"".""ID"" = ""FileLookup"".""MetadataID"" + AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"" + LIMIT 1 + "; break; } - m_findfileCommand = m_connection.CreateCommand(findQuery); + dbnew.m_findfileCommand = dbnew.m_connection.CreateCommand(findQuery); - m_selectfileHashCommand = m_connection.CreateCommand(@"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"", ""FileLookup"" WHERE ""Blockset"".""ID"" = ""FileLookup"".""BlocksetID"" AND ""FileLookup"".""ID"" = @FileId "); - m_getfirstfilesetwithblockinblockset = m_connection.CreateCommand(@"SELECT MIN(""FilesetEntry"".""FilesetID"") FROM ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" IN ( -SELECT ""File"".""ID"" FROM ""File"" WHERE ""File"".""BlocksetID"" IN( -SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""BlocklistHash"".""Hash"" = @Hash))"); + dbnew.m_selectfileHashCommand = dbnew.m_connection.CreateCommand(@" + SELECT ""Blockset"".""Fullhash"" + FROM + ""Blockset"", + ""FileLookup"" + WHERE + ""Blockset"".""ID"" = ""FileLookup"".""BlocksetID"" + AND ""FileLookup"".""ID"" = @FileId + "); - m_blocklistHashes = new HashSet(); + dbnew.m_getfirstfilesetwithblockinblockset = dbnew.m_connection.CreateCommand(@" + SELECT MIN(""FilesetEntry"".""FilesetID"") + FROM ""FilesetEntry"" + WHERE ""FilesetEntry"".""FileID"" IN ( + SELECT ""File"".""ID"" + FROM ""File"" + WHERE ""File"".""BlocksetID"" IN( + SELECT ""BlocklistHash"".""BlocksetID"" + FROM ""BlocklistHash"" + WHERE ""BlocklistHash"".""Hash"" = @Hash + ) + ) + "); + + dbnew.m_blocklistHashes = new HashSet(); + + return dbnew; } /// - /// Probes to see if a block already exists + /// Probes to see if a block already exists. /// - /// The block key - /// The size of the block - /// True if the block should be added to the current output - public long FindBlockID(string key, long size, IDbTransaction? transaction = null) + /// The block key. + /// The size of the block. + /// The cancellation token to monitor for cancellation requests. + /// A task that when awaited contains true if the block should be added to the current output. + public async Task FindBlockID(string key, long size, CancellationToken token) { - return m_findblockCommand.SetTransaction(transaction) + return await m_findblockCommand + .SetTransaction(m_rtr) .SetParameterValue("@Hash", key) .SetParameterValue("@Size", size) - .ExecuteScalarInt64(m_logQueries, -1); + .ExecuteScalarInt64Async(m_logQueries, -1, token) + .ConfigureAwait(false); } /// - /// Adds a block to the local database, returning a value indicating if the value presents a new block + /// Adds a block to the local database, returning a value indicating if the value presents a new block. /// - /// The block key - /// The size of the block - /// True if the block should be added to the current output - public bool AddBlock(string key, long size, long volumeid, IDbTransaction? transaction = null) + /// The block key. + /// The size of the block. + /// The ID of the volume to which the block belongs. + /// The cancellation token to monitor for cancellation requests. + /// A taskt that when awaited contains true if the block should be added to the current output. + public async Task AddBlock(string key, long size, long volumeid, CancellationToken token) { - var r = FindBlockID(key, size, transaction); + var r = await FindBlockID(key, size, token).ConfigureAwait(false); if (r == -1L) { if (m_moveblockfromdeletedCommand != null) @@ -303,9 +766,11 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist if (m_deletedBlockLookup.TryGetValue(key, out var sizes)) if (sizes.TryGetValue(size, out var id)) { - m_moveblockfromdeletedCommand.SetTransaction(transaction) + await m_moveblockfromdeletedCommand + .SetTransaction(m_rtr) .SetParameterValue("@DeletedBlockId", id) - .ExecuteNonQuery(m_logQueries); + .ExecuteNonQueryAsync(m_logQueries, token) + .ConfigureAwait(false); sizes.Remove(size); if (sizes.Count == 0) @@ -316,16 +781,20 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist else if (m_findindeletedCommand != null) { // No transaction on the temporary table - var id = m_findindeletedCommand + var id = await m_findindeletedCommand + .SetTransaction(m_rtr) .SetParameterValue("@Hash", key) .SetParameterValue("@Size", size) - .ExecuteScalarInt64(m_logQueries, -1); + .ExecuteScalarInt64Async(m_logQueries, -1, token) + .ConfigureAwait(false); if (id != -1) { - var c = m_moveblockfromdeletedCommand.SetTransaction(transaction) + var c = await m_moveblockfromdeletedCommand + .SetTransaction(m_rtr) .SetParameterValue("@DeletedBlockId", id) - .ExecuteNonQuery(m_logQueries); + .ExecuteNonQueryAsync(m_logQueries, token) + .ConfigureAwait(false); if (c != 2) throw new Exception($"Failed to move block {key} with size {size}, result count: {c}"); @@ -336,11 +805,14 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } } - var ins = m_insertblockCommand.SetTransaction(transaction) + var ins = await m_insertblockCommand + .SetTransaction(m_rtr) .SetParameterValue("@Hash", key) .SetParameterValue("@VolumeId", volumeid) .SetParameterValue("@Size", size) - .ExecuteNonQuery(m_logQueries); + .ExecuteNonQueryAsync(m_logQueries, token) + .ConfigureAwait(false); + if (ins != 1) throw new Exception($"Failed to insert block {key} with size {size}, result count: {ins}"); @@ -355,289 +827,413 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist /// - /// Adds a blockset to the database, returns a value indicating if the blockset is new + /// Adds a blockset to the database, returns a value indicating if the blockset is new. /// - /// The hash of the blockset - /// The size of the blockset - /// The list of hashes - /// The id of the blockset, new or old - /// True if the blockset was created, false otherwise - public bool AddBlockset(string filehash, long size, int blocksize, IEnumerable hashes, IEnumerable blocklistHashes, out long blocksetid, IDbTransaction? transaction = null) + /// The hash of the blockset. + /// The size of the blockset. + /// The size of the blocks in the blockset. + /// The list of hashes. + /// The list of hashes for the blocklist, or null if no blocklist is used. + /// The cancellation token to monitor for cancellation requests. + /// A task that when awaited contains a tuple with the first value indicating whether the blockset was created, and the second value being the blockset ID. + public async Task<(bool, long)> AddBlockset(string filehash, long size, int blocksize, IEnumerable hashes, IEnumerable blocklistHashes, CancellationToken token) { - blocksetid = m_findblocksetCommand.SetTransaction(transaction) + long blocksetid = await m_findblocksetCommand + .SetTransaction(m_rtr) .SetParameterValue("@Fullhash", filehash) .SetParameterValue("@Length", size) - .ExecuteScalarInt64(m_logQueries, null, -1); + .ExecuteScalarInt64Async(m_logQueries, -1, token) + .ConfigureAwait(false); + if (blocksetid != -1) - return false; //Found it + return (false, blocksetid); //Found it - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) + blocksetid = await m_insertblocksetCommand + .SetTransaction(m_rtr) + .SetParameterValue("@Length", size) + .SetParameterValue("@Fullhash", filehash) + .ExecuteScalarInt64Async(m_logQueries, token) + .ConfigureAwait(false); + + long ix = 0; + if (blocklistHashes != null) { - blocksetid = m_insertblocksetCommand.SetTransaction(tr.Parent) - .SetParameterValue("@Length", size) - .SetParameterValue("@Fullhash", filehash) - .ExecuteScalarInt64(m_logQueries); - - long ix = 0; - if (blocklistHashes != null) - { - m_insertblocklistHashesCommand.SetTransaction(tr.Parent) - .SetParameterValue("@BlocksetId", blocksetid); - - foreach (var bh in blocklistHashes) - { - var c = m_insertblocklistHashesCommand.SetParameterValue("@Index", ix) - .SetParameterValue("@Hash", bh) - .ExecuteNonQuery(m_logQueries); - if (c != 1) - throw new Exception($"Failed to insert blocklist hash {bh} for blockset {blocksetid}, result count: {c}"); - ix++; - } - } - - m_insertblocksetentryCommand.SetTransaction(tr.Parent) + m_insertblocklistHashesCommand + .SetTransaction(m_rtr) .SetParameterValue("@BlocksetId", blocksetid); - ix = 0; - long remainsize = size; - foreach (var h in hashes) + foreach (var bh in blocklistHashes) { - var exsize = remainsize < blocksize ? remainsize : blocksize; - var c = m_insertblocksetentryCommand.SetParameterValue("@Index", ix) - .SetParameterValue("@Hash", h) - .SetParameterValue("@Size", exsize) - .ExecuteNonQuery(m_logQueries); - if (c != 1) - { - Logging.Log.WriteErrorMessage(LOGTAG, "CheckingErrorsForIssue1400", null, "Checking errors, related to #1400. Unexpected result count: {0}, expected {1}, hash: {2}, size: {3}, blocksetid: {4}, ix: {5}, fullhash: {6}, fullsize: {7}", c, 1, h, exsize, blocksetid, ix, filehash, size); - using (var cmd = m_connection.CreateCommand(tr.Parent)) - { - var bid = cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash") - .SetParameterValue("@Hash", h) - .ExecuteScalarInt64(-1); - if (bid == -1) - throw new Exception($"Could not find any blocks with the given hash: {h}"); - cmd.SetCommandAndParameters(@"SELECT ""Size"" FROM ""Block"" WHERE ""Hash"" = @Hash") - .SetParameterValue("@Hash", h); - foreach (var rd in cmd.ExecuteReaderEnumerable()) - Logging.Log.WriteErrorMessage(LOGTAG, "FoundIssue1400Error", null, "Found block with ID {0} and hash {1} and size {2}", bid, h, rd.ConvertValueToInt64(0, -1)); - } + var c = await m_insertblocklistHashesCommand + .SetParameterValue("@Index", ix) + .SetParameterValue("@Hash", bh) + .ExecuteNonQueryAsync(m_logQueries, token) + .ConfigureAwait(false); - throw new Exception($"Unexpected result count: {c}, expected {1}, check log for more messages"); - } + if (c != 1) + throw new Exception($"Failed to insert blocklist hash {bh} for blockset {blocksetid}, result count: {c}"); ix++; - remainsize -= blocksize; } - - tr.Commit(); } - return true; + m_insertblocksetentryCommand + .SetTransaction(m_rtr) + .SetParameterValue("@BlocksetId", blocksetid); + + ix = 0; + long remainsize = size; + foreach (var h in hashes) + { + var exsize = remainsize < blocksize ? remainsize : blocksize; + var c = await m_insertblocksetentryCommand + .SetParameterValue("@Index", ix) + .SetParameterValue("@Hash", h) + .SetParameterValue("@Size", exsize) + .ExecuteNonQueryAsync(m_logQueries, token) + .ConfigureAwait(false); + + if (c != 1) + { + Logging.Log.WriteErrorMessage(LOGTAG, "CheckingErrorsForIssue1400", null, "Checking errors, related to #1400. Unexpected result count: {0}, expected {1}, hash: {2}, size: {3}, blocksetid: {4}, ix: {5}, fullhash: {6}, fullsize: {7}", c, 1, h, exsize, blocksetid, ix, filehash, size); + await using (var cmd = m_connection.CreateCommand(m_rtr)) + { + var bid = await cmd.SetCommandAndParameters(@" + SELECT ""ID"" + FROM ""Block"" + WHERE ""Hash"" = @Hash + ") + .SetParameterValue("@Hash", h) + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); + + if (bid == -1) + throw new Exception($"Could not find any blocks with the given hash: {h}"); + + cmd.SetCommandAndParameters(@" + SELECT ""Size"" + FROM ""Block"" + WHERE ""Hash"" = @Hash + ") + .SetParameterValue("@Hash", h); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) + Logging.Log.WriteErrorMessage(LOGTAG, "FoundIssue1400Error", null, "Found block with ID {0} and hash {1} and size {2}", bid, h, rd.ConvertValueToInt64(0, -1)); + } + + throw new Exception($"Unexpected result count: {c}, expected {1}, check log for more messages"); + } + + ix++; + remainsize -= blocksize; + } + + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); + + return (true, blocksetid); } /// - /// Gets the metadataset ID from the filehash + /// Gets the metadataset ID from the filehash. /// - /// true, if metadataset found, false if does not exist. /// The metadata hash. /// The size of the metadata. - /// The ID of the metadataset. - /// An optional transaction. - public bool GetMetadatasetID(string filehash, long size, out long metadataid, IDbTransaction? transaction = null) + /// The cancellation token to cancel the operation. + /// A task that when awaited contains a tuple with the first value indicating if the metadataset was found, and the second value being the metadataset ID. + public async Task<(bool, long)> GetMetadatasetID(string filehash, long size, CancellationToken token) { + long metadataid; + if (size > 0) { - metadataid = m_findmetadatasetCommand.SetTransaction(transaction) + metadataid = await m_findmetadatasetCommand + .SetTransaction(m_rtr) .SetParameterValue("@Hash", filehash) .SetParameterValue("@Size", size) - .ExecuteScalarInt64(m_logQueries, null, -1); - return metadataid != -1; + .ExecuteScalarInt64Async(m_logQueries, -1, token) + .ConfigureAwait(false); + + return (metadataid != -1, metadataid); } metadataid = -2; - return false; + return (false, metadataid); } /// - /// Adds a metadata set to the database, and returns a value indicating if the record was new + /// Adds a metadata set to the database, and returns a tuple indicating if the record was new and the ID of the metadata set. /// - /// The metadata hash - /// The size of the metadata - /// The transaction to execute under - /// The id of the blockset to add - /// The id of the metadata set - /// True if the set was added to the database, false otherwise - public bool AddMetadataset(string filehash, long size, long blocksetid, out long metadataid, IDbTransaction? transaction = null) + /// The metadata hash. + /// The size of the metadata. + /// The id of the blockset to add. + /// The cancellation token to cancel the operation. + /// A task that when awaited contains a tuple with the first value indicating if the metadata set was added, and the second value being the metadata ID. + public async Task<(bool, long)> AddMetadataset(string filehash, long size, long blocksetid, CancellationToken token) { - if (GetMetadatasetID(filehash, size, out metadataid, transaction)) - return false; + var (metadatafound, metadataid) = await GetMetadatasetID(filehash, size, token) + .ConfigureAwait(false); + if (metadatafound) + return (false, metadataid); - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - { - metadataid = m_insertmetadatasetCommand.SetTransaction(tr.Parent) - .SetParameterValue("@BlocksetId", blocksetid) - .ExecuteScalarInt64(m_logQueries); - tr.Commit(); - return true; - } + metadataid = await m_insertmetadatasetCommand + .SetTransaction(m_rtr) + .SetParameterValue("@BlocksetId", blocksetid) + .ExecuteScalarInt64Async(m_logQueries, token) + .ConfigureAwait(false); + + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); + + return (true, metadataid); } /// - /// Adds a file record to the database + /// Adds a file record to the database. /// - /// The path prefix ID - /// The path to the file - /// The time the file was modified - /// The ID of the hashkey for the file - /// The ID for the metadata - /// The transaction to use for insertion, or null for no transaction - public void AddFile(long pathprefixid, string filename, DateTime lastmodified, long blocksetID, long metadataID, IDbTransaction? transaction) + /// The path prefix ID. + /// The path to the file. + /// The time the file was modified. + /// The ID of the hashkey for the file. + /// The ID for the metadata. + /// The cancellation token to cancel the operation. + /// A task that completes when the file is added. + public async Task AddFile(long pathprefixid, string filename, DateTime lastmodified, long blocksetID, long metadataID, CancellationToken token) { - var fileidobj = m_findfilesetCommand.SetTransaction(transaction) + var fileidobj = await m_findfilesetCommand + .SetTransaction(m_rtr) .SetParameterValue("@BlocksetId", blocksetID) .SetParameterValue("@MetadataId", metadataID) .SetParameterValue("@Path", filename) .SetParameterValue("@PrefixId", pathprefixid) - .ExecuteScalarInt64(m_logQueries); + .ExecuteScalarInt64Async(m_logQueries, token) + .ConfigureAwait(false); if (fileidobj == -1) { - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - { - fileidobj = m_insertfileCommand.SetTransaction(tr.Parent) - .SetParameterValue("@PrefixId", pathprefixid) - .SetParameterValue("@Path", filename) - .SetParameterValue("@BlocksetId", blocksetID) - .SetParameterValue("@MetadataId", metadataID) - .ExecuteScalarInt64(m_logQueries); - tr.Commit(); - } + fileidobj = await m_insertfileCommand.SetTransaction(m_rtr) + .SetParameterValue("@PrefixId", pathprefixid) + .SetParameterValue("@Path", filename) + .SetParameterValue("@BlocksetId", blocksetID) + .SetParameterValue("@MetadataId", metadataID) + .ExecuteScalarInt64Async(m_logQueries, token) + .ConfigureAwait(false); + + await m_rtr.CommitAsync(token).ConfigureAwait(false); } - AddKnownFile(fileidobj, lastmodified, transaction); + await AddKnownFile(fileidobj, lastmodified, token).ConfigureAwait(false); } /// - /// Adds a file record to the database + /// Adds a file record to the database. /// - /// The path to the file - /// The time the file was modified - /// The ID of the hashkey for the file - /// The ID for the metadata - /// The transaction to use for insertion, or null for no transaction - public void AddFile(string filename, DateTime lastmodified, long blocksetID, long metadataID, IDbTransaction? transaction) + /// The path to the file. + /// The time the file was modified. + /// The ID of the hashkey for the file. + /// The ID for the metadata. + /// The cancellation token to cancel the operation. + /// A task that completes when the file is added. + public async Task AddFile(string filename, DateTime lastmodified, long blocksetID, long metadataID, CancellationToken token) { var split = SplitIntoPrefixAndName(filename); - AddFile(GetOrCreatePathPrefix(split.Key, transaction), split.Value, lastmodified, blocksetID, metadataID, transaction); + + await AddFile( + await GetOrCreatePathPrefix(split.Key, token).ConfigureAwait(false), + split.Value, + lastmodified, + blocksetID, + metadataID, + token + ) + .ConfigureAwait(false); } /// - /// Adds a known file to the fileset + /// Adds a known file to the fileset. /// - /// Id of the file - /// The time the file was modified - /// The transaction to use for insertion, or null for no transaction - public void AddKnownFile(long fileid, DateTime lastmodified, IDbTransaction? transaction = null) + /// Id of the file. + /// The time the file was modified. + /// The cancellation token to cancel the operation. + /// A task that completes when the file is added. + public async Task AddKnownFile(long fileid, DateTime lastmodified, CancellationToken token) { - m_insertfileOperationCommand.SetTransaction(transaction) + await m_insertfileOperationCommand + .SetTransaction(m_rtr) .SetParameterValue("@FilesetId", m_filesetId) .SetParameterValue("@FileId", fileid) .SetParameterValue("@LastModified", lastmodified.ToUniversalTime().Ticks) - .ExecuteNonQuery(m_logQueries); + .ExecuteNonQueryAsync(m_logQueries, token) + .ConfigureAwait(false); } - public void AddDirectoryEntry(string path, long metadataID, DateTime lastmodified, IDbTransaction? transaction = null) + /// + /// Adds a directory entry to the fileset. + /// + /// The path to the directory. + /// The ID for the metadata. + /// The time the directory was modified. + /// The cancellation token to cancel the operation. + /// A task that completes when the directory entry is added. + public async Task AddDirectoryEntry(string path, long metadataID, DateTime lastmodified, CancellationToken token) { - AddFile(path, lastmodified, FOLDER_BLOCKSET_ID, metadataID, transaction); + await AddFile(path, lastmodified, FOLDER_BLOCKSET_ID, metadataID, token) + .ConfigureAwait(false); } - public void AddSymlinkEntry(string path, long metadataID, DateTime lastmodified, IDbTransaction? transaction = null) + /// + /// Adds a symlink entry to the fileset. + /// + /// The path to the symlink. + /// The ID for the metadata. + /// The time the symlink was modified. + /// The cancellation token to cancel the operation. + /// A task that completes when the symlink entry is added. + public async Task AddSymlinkEntry(string path, long metadataID, DateTime lastmodified, CancellationToken token) { - AddFile(path, lastmodified, SYMLINK_BLOCKSET_ID, metadataID, transaction); + await AddFile(path, lastmodified, SYMLINK_BLOCKSET_ID, metadataID, token) + .ConfigureAwait(false); } - public long GetFileLastModified(long prefixid, string path, long filesetid, bool includeLength, out DateTime oldModified, out long length, IDbTransaction? transaction = null) + /// + /// Gets the ID, last modified time and size of a file in the fileset. + /// + /// The ID of the path prefix. + /// The path to the file. + /// The ID of the fileset. + /// Whether to include the file length in the result. + /// The cancellation token to monitor for cancellation requests. + /// A task that when awaited contains a tuple with the file ID, last modified time, and file length. + public async Task<(long, DateTime, long)> GetFileLastModified(long prefixid, string path, long filesetid, bool includeLength, CancellationToken token) { + DateTime oldModified; + long length; if (includeLength) { - m_selectfilelastmodifiedWithSizeCommand.SetTransaction(transaction) + m_selectfilelastmodifiedWithSizeCommand + .SetTransaction(m_rtr) .SetParameterValue("@PrefixId", prefixid) .SetParameterValue("@Path", path) .SetParameterValue("@FilesetId", filesetid); - using (var rd = m_selectfilelastmodifiedWithSizeCommand.ExecuteReader(m_logQueries, null)) - if (rd.Read()) - { - oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); - length = rd.ConvertValueToInt64(2); - return rd.ConvertValueToInt64(0); - } + + await using var rd = await m_selectfilelastmodifiedWithSizeCommand.ExecuteReaderAsync(m_logQueries, token).ConfigureAwait(false); + if (await rd.ReadAsync(token).ConfigureAwait(false)) + { + oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); + length = rd.ConvertValueToInt64(2); + return (rd.ConvertValueToInt64(0), oldModified, length); + } } else { - m_selectfilelastmodifiedCommand.SetTransaction(transaction) + m_selectfilelastmodifiedCommand.SetTransaction(m_rtr) .SetParameterValue("@PrefixId", prefixid) .SetParameterValue("@Path", path) .SetParameterValue("@FilesetId", filesetid); - using (var rd = m_selectfilelastmodifiedCommand.ExecuteReader(m_logQueries, null)) - if (rd.Read()) - { - length = -1; - oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); - return rd.ConvertValueToInt64(0); - } + await using var rd = await m_selectfilelastmodifiedCommand.ExecuteReaderAsync(m_logQueries, token).ConfigureAwait(false); + if (await rd.ReadAsync(token).ConfigureAwait(false)) + { + length = -1; + oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); + return (rd.ConvertValueToInt64(0), oldModified, length); + } } + oldModified = new DateTime(0, DateTimeKind.Utc); length = -1; - return -1; + return (-1, oldModified, length); } - - public long GetFileEntry(long prefixid, string path, long filesetid, out DateTime oldModified, out long lastFileSize, out string? oldMetahash, out long oldMetasize, IDbTransaction transaction) + /// + /// Gets the file entry for a given path in the fileset. + /// + /// The ID of the path prefix. + /// The path to the file. + /// The ID of the fileset. + /// The cancellation token to monitor for cancellation requests. + /// A task that when awaited contains a tuple with the file ID, last modified time, file size, metadata hash, and metadata size. + /// + public async Task<(long, DateTime, long, string?, long)> GetFileEntry(long prefixid, string path, long filesetid, CancellationToken token) { - m_findfileCommand.SetTransaction(transaction) + DateTime oldModified; + long lastFileSize; + string? oldMetahash; + long oldMetasize; + + m_findfileCommand + .SetTransaction(m_rtr) .SetParameterValue("@PrefixId", prefixid) .SetParameterValue("@Path", path) .SetParameterValue("@FilesetId", filesetid); - using (var rd = m_findfileCommand.ExecuteReader()) - if (rd.Read()) - { - oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); - lastFileSize = rd.ConvertValueToInt64(2); - oldMetahash = rd.ConvertValueToString(3); - oldMetasize = rd.ConvertValueToInt64(4); - return rd.ConvertValueToInt64(0); - } - else - { - oldModified = new DateTime(0, DateTimeKind.Utc); - lastFileSize = -1; - oldMetahash = null; - oldMetasize = -1; - return -1; - } + await using var rd = await m_findfileCommand.ExecuteReaderAsync(token).ConfigureAwait(false); + if (await rd.ReadAsync(token).ConfigureAwait(false)) + { + oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); + lastFileSize = rd.ConvertValueToInt64(2); + oldMetahash = rd.ConvertValueToString(3); + oldMetasize = rd.ConvertValueToInt64(4); + + return ( + rd.ConvertValueToInt64(0), + oldModified, + lastFileSize, + oldMetahash, + oldMetasize + ); + } + else + { + oldModified = new DateTime(0, DateTimeKind.Utc); + lastFileSize = -1; + oldMetahash = null; + oldMetasize = -1; + + return ( + -1, + oldModified, + lastFileSize, + oldMetahash, + oldMetasize + ); + } } - public (long Size, string MetadataHash)? GetMetadataHashAndSizeForFile(long fileid, IDbTransaction transaction) + /// + /// Gets the metadata hash and size for a file. + /// + /// The ID of the file. + /// A cancellation token to monitor for cancellation requests. + /// A task that when awaited contains a tuple with the metadata hash and size, or null if the file does not exist. + public async Task<(string MetadataHash, long Size)?> GetMetadataHashAndSizeForFile(long fileid, CancellationToken token) { - m_selectfilemetadatahashandsizeCommand.SetTransaction(transaction) + m_selectfilemetadatahashandsizeCommand + .SetTransaction(m_rtr) .SetParameterValue("@FileId", fileid); - using (var rd = m_selectfilemetadatahashandsizeCommand.ExecuteReader()) - if (rd.Read()) - return (rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? throw new InvalidOperationException("Metadata hash is null")); + await using var rd = await m_selectfilemetadatahashandsizeCommand.ExecuteReaderAsync(token).ConfigureAwait(false); + if (await rd.ReadAsync(token).ConfigureAwait(false)) + return ( + rd.ConvertValueToString(1) ?? throw new InvalidOperationException("Metadata hash is null"), + rd.ConvertValueToInt64(0) + ); return null; } - - public string? GetFileHash(long fileid, IDbTransaction transaction) + /// + /// Gets the hash of a file. + /// + /// The ID of the file. + /// The cancellation token to monitor for cancellation requests. + /// A task that when awaited contains the hash of the file, or null if the file does not exist. + public async Task GetFileHash(long fileid, CancellationToken token) { - var r = m_selectfileHashCommand.SetTransaction(transaction) + var r = await m_selectfileHashCommand + .SetTransaction(m_rtr) .SetParameterValue("@FileId", fileid) - .ExecuteScalar(m_logQueries, null); + .ExecuteScalarAsync(m_logQueries, token) + .ConfigureAwait(false); + if (r == null || r == DBNull.Value) return null; @@ -645,396 +1241,689 @@ SELECT ""BlocklistHash"".""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Blocklist } public override void Dispose() + { + this.DisposeAsync().AsTask().Await(); + } + + public override async ValueTask DisposeAsync() { if (!string.IsNullOrWhiteSpace(m_tempDeletedBlockTable)) try { - using (var cmd = m_connection.CreateCommand()) - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE ""{m_tempDeletedBlockTable}""")); + await using (var cmd = m_connection.CreateCommand(m_rtr)) + await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{m_tempDeletedBlockTable}""", default) + .ConfigureAwait(false); + await m_rtr.CommitAsync().ConfigureAwait(false); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "DropTempTableFailed", ex, "Failed to drop temporary table {0}: {1}", m_tempDeletedBlockTable, ex.Message); } - base.Dispose(); + await base.DisposeAsync().ConfigureAwait(false); } - public long GetLastWrittenDBlockVolumeSize(IDbTransaction? transaction) + /// + /// Gets the size of the last written DBlock volume. + /// + /// The cancellation token to monitor for cancellation requests. + /// A task that when awaited contains the size of the last written DBlock volume, or -1 if no such volume exists. + public async Task GetLastWrittenDBlockVolumeSize(CancellationToken token) { - using (var cmd = m_connection.CreateCommand(transaction)) - return cmd.SetCommandAndParameters(@"SELECT ""Size"" FROM ""RemoteVolume"" WHERE ""State"" = @State AND ""Type"" = @Type ORDER BY ""ID"" DESC LIMIT 1") - .SetParameterValue("@State", RemoteVolumeState.Uploaded.ToString()) - .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) - .ExecuteScalarInt64(-1); + await using var cmd = m_connection.CreateCommand(m_rtr); + return await cmd.SetCommandAndParameters(@" + SELECT ""Size"" + FROM ""RemoteVolume"" + WHERE + ""State"" = @State + AND ""Type"" = @Type + ORDER BY ""ID"" DESC + LIMIT 1 + ") + .SetParameterValue("@State", RemoteVolumeState.Uploaded.ToString()) + .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); } - private long GetPreviousFilesetID(IDbCommand cmd) + /// + /// Gets the ID of the previous fileset based on the operation timestamp and current fileset ID. + /// + /// The command to use for the query. + /// The cancellation token to cancel the operation. + /// A task that when awaited contains the ID of the previous fileset, or -1 if no such fileset exists. + private async Task GetPreviousFilesetID(SqliteCommand cmd, CancellationToken token) { - return GetPreviousFilesetID(cmd, OperationTimestamp, m_filesetId, cmd.Transaction); + return await GetPreviousFilesetID(cmd, OperationTimestamp, m_filesetId, token) + .ConfigureAwait(false); } - private long GetPreviousFilesetID(IDbCommand cmd, DateTime timestamp, long filesetid, IDbTransaction? transaction) + /// + /// Gets the ID of the previous fileset based on the operation timestamp and current fileset ID. + /// + /// The command to use for the query. + /// The timestamp to use for the query. + /// The current fileset ID. + /// The cancellation token to cancel the operation. + /// A task that when awaited contains the ID of the previous fileset, or -1 if no such fileset exists. + private async Task GetPreviousFilesetID(SqliteCommand cmd, DateTime timestamp, long filesetid, CancellationToken token) { - return cmd.SetTransaction(transaction) - .SetCommandAndParameters(@"SELECT ""ID"" FROM ""Fileset"" WHERE ""Timestamp"" < @Timestamp AND ""ID"" != @FilesetId ORDER BY ""Timestamp"" DESC ") + return await cmd + .SetTransaction(m_rtr) + .SetCommandAndParameters(@" + SELECT ""ID"" + FROM ""Fileset"" + WHERE + ""Timestamp"" < @Timestamp + AND ""ID"" != @FilesetId + ORDER BY ""Timestamp"" DESC + ") .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(timestamp)) .SetParameterValue("@FilesetId", filesetid) - .ExecuteScalarInt64(-1); - } - - internal Tuple GetLastBackupFileCountAndSize() - { - using (var cmd = m_connection.CreateCommand()) - { - var lastFilesetId = cmd.ExecuteScalarInt64(@"SELECT ""ID"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC LIMIT 1"); - var count = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""FileLookup"" INNER JOIN ""FilesetEntry"" ON ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""FileLookup"".""BlocksetID"" NOT IN (@FolderBlocksetId, @SymlinkBlocksetId)") - .SetParameterValue("@FilesetId", lastFilesetId) - .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) - .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) - .ExecuteScalarInt64(-1); - - var size = cmd.SetCommandAndParameters(@"SELECT SUM(""Blockset"".""Length"") FROM ""FileLookup"", ""FilesetEntry"", ""Blockset"" WHERE ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""FileLookup"".""BlocksetID"" NOT IN (@FolderBlocksetId, @SymlinkBlocksetId)") - .SetParameterValue("@FilesetId", lastFilesetId) - .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) - .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) - .ExecuteScalarInt64(-1); - - return new Tuple(count, size); - } - } - - internal void UpdateChangeStatistics(BackupResults results, IDbTransaction transaction) - { - using (var cmd = m_connection.CreateCommand(transaction)) - { - var prevFileSetId = GetPreviousFilesetID(cmd); - ChangeStatistics.UpdateChangeStatistics(cmd, results, m_filesetId, prevFileSetId); - } + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); } /// - /// Populates FilesetEntry table with files from previous fileset, which aren't - /// yet part of the new fileset, and which aren't on the (optional) list of deleted paths. + /// Gets the count and size of files in the last backup fileset. /// - /// Transaction - /// List of deleted paths, or null - public void AppendFilesFromPreviousSet(IDbTransaction transaction, IEnumerable? deleted = null) + /// The cancellation token to cancel the operation. + /// A task that when awaited contains a tuple with the count of files and the total size of files in the last backup fileset. + internal async Task> GetLastBackupFileCountAndSize(CancellationToken token) { - AppendFilesFromPreviousSet(transaction, deleted, m_filesetId, -1, OperationTimestamp); + await using var cmd = m_connection.CreateCommand(m_rtr); + var lastFilesetId = await cmd.ExecuteScalarInt64Async(@" + SELECT ""ID"" + FROM ""Fileset"" + ORDER BY ""Timestamp"" DESC + LIMIT 1 + ", token) + .ConfigureAwait(false); + + var count = await cmd.SetCommandAndParameters(@" + SELECT COUNT(*) + FROM ""FileLookup"" + INNER JOIN ""FilesetEntry"" + ON ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" + WHERE + ""FilesetEntry"".""FilesetID"" = @FilesetId + AND ""FileLookup"".""BlocksetID"" NOT IN ( + @FolderBlocksetId, + @SymlinkBlocksetId + ) + ") + .SetParameterValue("@FilesetId", lastFilesetId) + .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) + .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); + + var size = await cmd.SetCommandAndParameters(@" + SELECT SUM(""Blockset"".""Length"") + FROM + ""FileLookup"", + ""FilesetEntry"", + ""Blockset"" + WHERE + ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND + ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" AND + ""FilesetEntry"".""FilesetID"" = @FilesetId AND + ""FileLookup"".""BlocksetID"" NOT IN ( + @FolderBlocksetId, + @SymlinkBlocksetId + ) + ") + .SetParameterValue("@FilesetId", lastFilesetId) + .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) + .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); + + return new Tuple(count, size); } /// - /// Populates FilesetEntry table with files from previous fileset, which aren't + /// Updates the change statistics for the current fileset based on the results of a backup operation. + /// + /// The results of the backup operation. + /// The cancellation token to cancel the operation. + /// A task that completes when the change statistics are updated. + internal async Task UpdateChangeStatistics(BackupResults results, CancellationToken token) + { + await using var cmd = m_connection.CreateCommand(m_rtr); + var prevFileSetId = await GetPreviousFilesetID(cmd, token) + .ConfigureAwait(false); + await ChangeStatistics.UpdateChangeStatistics(cmd, results, m_filesetId, prevFileSetId, token) + .ConfigureAwait(false); + } + + /// + /// Populates FilesetEntry table with files from previous fileset, which aren't /// yet part of the new fileset, and which aren't on the (optional) list of deleted paths. /// - /// Transaction - /// List of deleted paths, or null - /// Current file-set ID - /// Source file-set ID - /// If filesetid == -1, used to locate previous file-set - public void AppendFilesFromPreviousSet(IDbTransaction transaction, IEnumerable? deleted, long filesetid, long prevId, DateTime timestamp) + /// List of deleted paths, or null." + /// The cancellation token to cancel the operation. + /// A task that completes when the files are appended. + public async Task AppendFilesFromPreviousSet(IEnumerable? deleted, CancellationToken token) { - using (var cmd = m_connection.CreateCommand()) - using (var cmdDelete = m_connection.CreateCommand()) - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) + await AppendFilesFromPreviousSet(deleted, m_filesetId, -1, OperationTimestamp, token) + .ConfigureAwait(false); + } + + /// + /// Populates FilesetEntry table with files from previous fileset, which aren't + /// yet part of the new fileset, and which aren't on the (optional) list of deleted paths. + /// + /// List of deleted paths, or null. + /// Current file-set ID. + /// Source file-set ID. + /// If filesetid == -1, used to locate previous file-set. + /// The cancellation token to cancel the operation. + /// A task that completes when the files are appended. + public async Task AppendFilesFromPreviousSet(IEnumerable? deleted, long filesetid, long prevId, DateTime timestamp, CancellationToken token) + { + await using var cmd = m_connection.CreateCommand(); + await using var cmdDelete = m_connection.CreateCommand(); + long lastFilesetId = prevId < 0 ? + await GetPreviousFilesetID(cmd, timestamp, filesetid, token) + .ConfigureAwait(false) + : + prevId; + + await cmd.SetTransaction(m_rtr) + .SetCommandAndParameters(@" + INSERT INTO ""FilesetEntry"" ( + ""FilesetID"", + ""FileID"", + ""Lastmodified"" + ) + SELECT + @CurrentFilesetId AS ""FilesetID"", + ""FileID"", + ""Lastmodified"" + FROM ( + SELECT DISTINCT + ""FilesetID"", + ""FileID"", + ""Lastmodified"" + FROM ""FilesetEntry"" + WHERE + ""FilesetID"" = @PreviousFilesetId + AND ""FileID"" NOT IN ( + SELECT ""FileID"" + FROM ""FilesetEntry"" + WHERE ""FilesetID"" = @CurrentFilesetId + ) + ) + ") + .SetParameterValue("@CurrentFilesetId", filesetid) + .SetParameterValue("@PreviousFilesetId", lastFilesetId) + .ExecuteNonQueryAsync(m_logQueries, token) + .ConfigureAwait(false); + + if (deleted != null) { - long lastFilesetId = prevId < 0 ? GetPreviousFilesetID(cmd, timestamp, filesetid, tr.Parent) : prevId; + await using var tmplist = await TemporaryDbValueList + .CreateAsync(this, deleted, token) + .ConfigureAwait(false); - cmd.SetTransaction(tr.Parent) - .SetCommandAndParameters(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") SELECT @CurrentFilesetId AS ""FilesetID"", ""FileID"", ""Lastmodified"" FROM (SELECT DISTINCT ""FilesetID"", ""FileID"", ""Lastmodified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @PreviousFilesetId AND ""FileID"" NOT IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @CurrentFilesetId)) ") - .SetParameterValue("@CurrentFilesetId", filesetid) - .SetParameterValue("@PreviousFilesetId", lastFilesetId) - .ExecuteNonQuery(m_logQueries); - - if (deleted != null) - { - using var tmplist = new TemporaryDbValueList(m_connection, tr.Parent, deleted); - cmdDelete.SetTransaction(tr.Parent) - .SetCommandAndParameters(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId AND ""FileID"" IN (SELECT ""ID"" FROM ""File"" WHERE ""Path"" IN (@Paths)) ") + await ( + await cmdDelete.SetTransaction(m_rtr) + .SetCommandAndParameters(@" + DELETE FROM ""FilesetEntry"" + WHERE + ""FilesetID"" = @FilesetId + AND ""FileID"" IN ( + SELECT ""ID"" + FROM ""File"" + WHERE ""Path"" IN (@Paths) + ) + ") .SetParameterValue("@FilesetId", filesetid) - .ExpandInClauseParameter("@Paths", tmplist) - .ExecuteNonQuery(m_logQueries); - } - - tr.Commit(); + .ExpandInClauseParameterMssqliteAsync("@Paths", tmplist, token) + .ConfigureAwait(false) + ) + .ExecuteNonQueryAsync(m_logQueries, token) + .ConfigureAwait(false); } + + await m_rtr.CommitAsync(token).ConfigureAwait(false); } /// - /// Populates FilesetEntry table with files from previous fileset, which aren't - /// yet part of the new fileset, and which aren't excluded by the (optional) exclusion + /// Populates FilesetEntry table with files from previous fileset, which aren't + /// yet part of the new fileset, and which aren't excluded by the (optional) exclusion /// predicate. /// - /// Transaction - /// Optional exclusion predicate (true = exclude file) - public void AppendFilesFromPreviousSetWithPredicate(IDbTransaction transaction, Func exclusionPredicate) + /// Optional exclusion predicate (true = exclude file). + /// The cancellation token to cancel the operation. + /// A task that completes when the files are appended. + public async Task AppendFilesFromPreviousSetWithPredicate(Func exclusionPredicate, CancellationToken token) { - AppendFilesFromPreviousSetWithPredicate(transaction, exclusionPredicate, m_filesetId, -1, OperationTimestamp); + await AppendFilesFromPreviousSetWithPredicate(exclusionPredicate, m_filesetId, -1, OperationTimestamp, token) + .ConfigureAwait(false); } /// - /// Populates FilesetEntry table with files from previous fileset, which aren't - /// yet part of the new fileset, and which aren't excluded by the (optional) exclusion + /// Populates FilesetEntry table with files from previous fileset, which aren't + /// yet part of the new fileset, and which aren't excluded by the (optional) exclusion /// predicate. /// - /// Transaction /// Optional exclusion predicate (true = exclude file) /// Current fileset ID /// Source fileset ID /// If prevFileSetId == -1, used to locate previous fileset - public void AppendFilesFromPreviousSetWithPredicate(IDbTransaction transaction, - Func exclusionPredicate, long fileSetId, long prevFileSetId, DateTime timestamp) + /// The cancellation token to cancel the operation. + /// A task that completes when the files are appended. + public async Task AppendFilesFromPreviousSetWithPredicate(Func exclusionPredicate, long fileSetId, long prevFileSetId, DateTime timestamp, CancellationToken token) { if (exclusionPredicate == null) { - AppendFilesFromPreviousSet(transaction, null, fileSetId, prevFileSetId, timestamp); + await AppendFilesFromPreviousSet(null, fileSetId, prevFileSetId, timestamp, token) + .ConfigureAwait(false); return; } - using (var cmd = m_connection.CreateCommand()) - using (var cmdDelete = m_connection.CreateCommand()) - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) + await using var cmd = m_connection.CreateCommand(); + await using var cmdDelete = m_connection.CreateCommand(); + long lastFilesetId = prevFileSetId < 0 ? + await GetPreviousFilesetID(cmd, timestamp, fileSetId, token) + .ConfigureAwait(false) + : + prevFileSetId; + + // copy entries from previous file set into a temporary table, except those file IDs already added by the current backup + var tempFileSetTable = "FilesetEntry-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + await cmd + .SetTransaction(m_rtr) + .SetCommandAndParameters($@" + CREATE TEMPORARY TABLE ""{tempFileSetTable}"" AS + SELECT + ""FileID"", + ""Lastmodified"" + FROM ( + SELECT DISTINCT + ""FilesetID"", + ""FileID"", + ""Lastmodified"" + FROM ""FilesetEntry"" + WHERE + ""FilesetID"" = @PreviousFilesetId + AND ""FileID"" NOT IN ( + SELECT ""FileID"" + FROM ""FilesetEntry"" + WHERE ""FilesetID"" = @CurrentFilesetId + ) + ) + ") + .SetParameterValue("@PreviousFilesetId", lastFilesetId) + .SetParameterValue("@CurrentFilesetId", fileSetId) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + // now we need to remove, from the above, any entries that were enumerated by the + // UNC-driven backup + cmdDelete.SetTransaction(m_rtr) + .SetCommandAndParameters($@" + DELETE FROM ""{tempFileSetTable}"" + WHERE ""FileID"" = @FileId + "); + + // enumerate files from new temporary file set, and remove any entries handled by UNC + cmd.SetCommandAndParameters($@" + SELECT + ""f"".""Path"", + ""fs"".""FileID"", + ""fs"".""Lastmodified"", + COALESCE(""bs"".""Length"", -1) + FROM ( + SELECT DISTINCT + ""FileID"", + ""Lastmodified"" + FROM ""{tempFileSetTable}"" + ) AS ""fs"" + LEFT JOIN ""File"" AS ""f"" + ON ""fs"".""FileID"" = ""f"".""ID"" + LEFT JOIN ""Blockset"" AS ""bs"" + ON ""f"".""BlocksetID"" = ""bs"".""ID""; + "); + + await foreach (var row in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) { - long lastFilesetId = prevFileSetId < 0 ? GetPreviousFilesetID(cmd, timestamp, fileSetId, tr.Parent) : prevFileSetId; + var path = row.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for path"); + var size = row.ConvertValueToInt64(3); - // copy entries from previous file set into a temporary table, except those file IDs already added by the current backup - var tempFileSetTable = "FilesetEntry-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - cmd.Transaction = tr.Parent; - cmd.SetCommandAndParameters(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tempFileSetTable}"" AS SELECT ""FileID"", ""Lastmodified"" FROM (SELECT DISTINCT ""FilesetID"", ""FileID"", ""Lastmodified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @PreviousFilesetId AND ""FileID"" NOT IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @CurrentFilesetId))")) - .SetParameterValue("@PreviousFilesetId", lastFilesetId) - .SetParameterValue("@CurrentFilesetId", fileSetId) - .ExecuteNonQuery(); - - // now we need to remove, from the above, any entries that were enumerated by the - // UNC-driven backup - cmdDelete.SetTransaction(tr.Parent) - .SetCommandAndParameters(FormatInvariant($@"DELETE FROM ""{tempFileSetTable}"" WHERE ""FileID"" = @FileId")); - - // enumerate files from new temporary file set, and remove any entries handled by UNC - cmd.Transaction = tr.Parent; - foreach (var row in cmd.ExecuteReaderEnumerable(FormatInvariant( - $@"SELECT f.""Path"", fs.""FileID"", fs.""Lastmodified"", COALESCE(bs.""Length"", -1) - FROM (SELECT DISTINCT ""FileID"", ""Lastmodified"" FROM ""{tempFileSetTable}"") AS fs - LEFT JOIN ""File"" AS f ON fs.""FileID"" = f.""ID"" - LEFT JOIN ""Blockset"" AS bs ON f.""BlocksetID"" = bs.""ID"";"))) - { - var path = row.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for path"); - var size = row.ConvertValueToInt64(3); - - if (exclusionPredicate(path, size)) - cmdDelete.SetParameterValue("@FileId", row.ConvertValueToInt64(1)) - .ExecuteNonQuery(); - } - - // now copy the temporary table into the FileSetEntry table - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") - SELECT @FilesetId, ""FileID"", ""Lastmodified"" FROM ""{tempFileSetTable}""")) - .SetParameterValue("@FilesetId", fileSetId) - .ExecuteNonQuery(); - - tr.Commit(); + if (exclusionPredicate(path, size)) + await cmdDelete.SetParameterValue("@FileId", row.ConvertValueToInt64(1)) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); } + + // now copy the temporary table into the FileSetEntry table + await cmd.SetCommandAndParameters($@" + INSERT INTO ""FilesetEntry"" ( + ""FilesetID"", + ""FileID"", + ""Lastmodified"" + ) + SELECT + @FilesetId, + ""FileID"", + ""Lastmodified"" + FROM ""{tempFileSetTable}"" + ") + .SetParameterValue("@FilesetId", fileSetId) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + await m_rtr.CommitAsync(token).ConfigureAwait(false); } /// /// Creates a timestamped backup operation to correctly associate the fileset with the time it was created. /// - /// The ID of the fileset volume to update - /// The timestamp of the operation to create - /// An optional external transaction - public override long CreateFileset(long volumeid, DateTime timestamp, IDbTransaction? transaction = null) + /// The ID of the fileset volume to update. + /// The timestamp of the operation to create. + /// The cancellation token to cancel the operation. + /// A task that when awaited contains the ID of the created fileset. + public override async Task CreateFileset(long volumeid, DateTime timestamp, CancellationToken token) { - return m_filesetId = base.CreateFileset(volumeid, timestamp, transaction); + return m_filesetId = await base.CreateFileset(volumeid, timestamp, token) + .ConfigureAwait(false); } - public IEnumerable GetTemporaryFilelistVolumeNames(bool latestOnly, IDbTransaction? transaction = null) + /// + /// Retrieves the names of temporary fileset volumes that are incomplete. + /// + /// If true, only the latest incomplete fileset volume will be returned. + /// The cancellation token to cancel the operation. + /// A task that when awaited contains a list of volume names. + public async Task> GetTemporaryFilelistVolumeNames(bool latestOnly, CancellationToken token) { - var incompleteFilesetIDs = GetIncompleteFilesets(transaction).OrderBy(x => x.Value).Select(x => x.Key).ToArray(); + var incompleteFilesetIDs = GetIncompleteFilesets(token).OrderBy(x => x.Value).Select(x => x.Key); - if (!incompleteFilesetIDs.Any()) - return Enumerable.Empty(); + if (!await incompleteFilesetIDs.AnyAsync(token).ConfigureAwait(false)) + return []; if (latestOnly) - incompleteFilesetIDs = new long[] { incompleteFilesetIDs.Last() }; + incompleteFilesetIDs = new long[] { + await incompleteFilesetIDs.LastAsync(token).ConfigureAwait(false) + } + .ToAsyncEnumerable(); var volumeNames = new List(); - foreach (var filesetID in incompleteFilesetIDs) - volumeNames.Add(GetRemoteVolumeFromFilesetID(filesetID).Name); + await foreach (var filesetID in incompleteFilesetIDs.ConfigureAwait(false)) + volumeNames.Add(( + await GetRemoteVolumeFromFilesetID(filesetID, token) + .ConfigureAwait(false) + ).Name); return volumeNames; } - public IEnumerable GetMissingIndexFiles(IDbTransaction transaction) + /// + /// Retrieves the names of remote volumes that are missing index files. + /// + /// The cancellation token to cancel the operation. + /// An asynchronous enumerable of volume names that are missing index files. + public async IAsyncEnumerable GetMissingIndexFiles([EnumeratorCancellation] CancellationToken token) { - using var cmd = m_connection.CreateCommand(transaction) - .SetCommandAndParameters(@"SELECT ""Name"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND NOT ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"") AND ""State"" IN (@States)") + await using var cmd = m_connection.CreateCommand(m_rtr) + .SetCommandAndParameters(@" + SELECT ""Name"" + FROM ""RemoteVolume"" + WHERE + ""Type"" = @Type + AND NOT ""ID"" IN ( + SELECT ""BlockVolumeID"" + FROM ""IndexBlockLink"" + ) + AND ""State"" IN (@States) + ") .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) - .ExpandInClauseParameter("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); + .ExpandInClauseParameterMssqlite("@States", [ + RemoteVolumeState.Uploaded.ToString(), + RemoteVolumeState.Verified.ToString() + ]); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - yield return rd.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for volume name"); - } - - public void MoveBlockToVolume(string blockkey, long size, long sourcevolumeid, long targetvolumeid, IDbTransaction transaction) - { - using (var cmd = m_connection.CreateCommand()) - { - cmd.Transaction = transaction; - var c = cmd.SetCommandAndParameters(@"UPDATE ""Block"" SET ""VolumeID"" = @NewVolumeId WHERE ""Hash"" = @Hash AND ""Size"" = @Size AND ""VolumeID"" = @PreviousVolumeId ") - .SetParameterValue("@NewVolumeId", targetvolumeid) - .SetParameterValue("@Hash", blockkey) - .SetParameterValue("@Size", size) - .SetParameterValue("@PreviousVolumeId", sourcevolumeid) - .ExecuteNonQuery(); - if (c != 1) - throw new Exception($"Failed to move block {blockkey}:{size} from volume {sourcevolumeid}, count: {c}"); - } - } - - public void SafeDeleteRemoteVolume(string name, IDbTransaction transaction) - { - var volumeid = GetRemoteVolumeID(name, transaction); - - using (var cmd = m_connection.CreateCommand(transaction)) - { - var c = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""Block"" WHERE ""VolumeID"" = @VolumeId ") - .SetParameterValue("@VolumeId", volumeid) - .ExecuteScalarInt64(-1); - if (c != 0) - throw new Exception($"Failed to safe-delete volume {name}, blocks: {c}"); - - RemoveRemoteVolume(name, transaction); - } - } - - public string[] GetBlocklistHashes(string name, IDbTransaction transaction) - { - var volumeid = GetRemoteVolumeID(name, transaction); - using (var cmd = m_connection.CreateCommand(transaction)) - { - // Grab the strings and return as array to avoid concurrent access to the IEnumerable - cmd.SetCommandAndParameters(@"SELECT DISTINCT ""Block"".""Hash"" FROM ""Block"" WHERE ""Block"".""VolumeID"" = @VolumeId AND ""Block"".""Hash"" IN (SELECT ""Hash"" FROM ""BlocklistHash"")") - .SetParameterValue("@VolumeId", volumeid); - return cmd.ExecuteReaderEnumerable() - .Select(x => x.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for blocklist hash")) - .ToArray(); - } - } - - public string? GetFirstPath() - { - using (var cmd = m_connection.CreateCommand(@"SELECT ""Path"" FROM ""File"" ORDER BY LENGTH(""Path"") DESC LIMIT 1")) - { - var v0 = cmd.ExecuteScalar(); - if (v0 == null || v0 == DBNull.Value) - return null; - - return v0.ToString(); - } + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + yield return rd.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for volume name"); } /// - /// Retrieves change journal data for file set + /// Moves a block from one volume to another. /// - /// Fileset-ID - public IEnumerable GetChangeJournalData(long fileSetId) + /// The hash of the block to move. + /// The size of the block to move. + /// The ID of the source volume. + /// The ID of the target volume. + /// The cancellation token to cancel the operation. + /// A task that completes when the block is moved. + public async Task MoveBlockToVolume(string blockkey, long size, long sourcevolumeid, long targetvolumeid, CancellationToken token) + { + await using var cmd = m_connection.CreateCommand(m_rtr); + var c = await cmd.SetCommandAndParameters(@" + UPDATE ""Block"" + SET ""VolumeID"" = @NewVolumeId + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size + AND ""VolumeID"" = @PreviousVolumeId + ") + .SetParameterValue("@NewVolumeId", targetvolumeid) + .SetParameterValue("@Hash", blockkey) + .SetParameterValue("@Size", size) + .SetParameterValue("@PreviousVolumeId", sourcevolumeid) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + if (c != 1) + throw new Exception($"Failed to move block {blockkey}:{size} from volume {sourcevolumeid}, count: {c}"); + } + + /// + /// Safely deletes a remote volume by checking if it has any associated blocks. + /// If it does, an exception is thrown; otherwise, the volume is removed. + /// + /// The name of the remote volume to delete. + /// The cancellation token to cancel the operation. + /// A task that completes when the remote volume is safely deleted. + /// Thrown if the volume has associated blocks. + public async Task SafeDeleteRemoteVolume(string name, CancellationToken token) + { + var volumeid = await GetRemoteVolumeID(name, token).ConfigureAwait(false); + + await using var cmd = m_connection.CreateCommand(m_rtr); + var c = await cmd.SetCommandAndParameters(@" + SELECT COUNT(*) + FROM ""Block"" + WHERE ""VolumeID"" = @VolumeId + ") + .SetParameterValue("@VolumeId", volumeid) + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); + + if (c != 0) + throw new Exception($"Failed to safe-delete volume {name}, blocks: {c}"); + + await RemoveRemoteVolume(name, token).ConfigureAwait(false); + } + + /// + /// Retrieves the hashes of blocks that are on the blocklist for a given volume. + /// + /// The name of the volume to check. + /// The cancellation token to cancel the operation. + /// A task that when awaited contains an array of blocklist hashes. + public async Task GetBlocklistHashes(string name, CancellationToken token) + { + var volumeid = GetRemoteVolumeID(name, token); + await using var cmd = m_connection.CreateCommand(m_rtr); + // Grab the strings and return as array to avoid concurrent access to the IEnumerable + cmd.SetCommandAndParameters(@" + SELECT DISTINCT ""Block"".""Hash"" + FROM ""Block"" + WHERE + ""Block"".""VolumeID"" = @VolumeId + AND ""Block"".""Hash"" IN ( + SELECT ""Hash"" + FROM ""BlocklistHash"" + ) + ") + .SetParameterValue("@VolumeId", volumeid); + + return await cmd.ExecuteReaderEnumerableAsync(token) + .Select(x => x.ConvertValueToString(0) ?? throw new Exception("Unexpected null value for blocklist hash")) + .ToArrayAsync(cancellationToken: token) + .ConfigureAwait(false); + } + + /// + /// Retrieves the first path in the database, ordered by length in descending order. + /// + /// The cancellation token to cancel the operation. + /// A task that when awaited contains the first path, or null if no paths exist. + public async Task GetFirstPath(CancellationToken token) + { + await using var cmd = m_connection.CreateCommand(m_rtr); + var v0 = await cmd.ExecuteScalarAsync(@" + SELECT ""Path"" + FROM ""File"" + ORDER BY LENGTH(""Path"") DESC + LIMIT 1 + ", token) + .ConfigureAwait(false); + + if (v0 == null || v0 == DBNull.Value) + return null; + + return v0.ToString(); + } + + /// + /// Retrieves the change journal data for file set. + /// + /// The Fileset-ID. + /// An asynchronous enumerable of USN journal data entries. + public async IAsyncEnumerable GetChangeJournalData(long fileSetId, [EnumeratorCancellation] CancellationToken token) { var data = new List(); - using var cmd = m_connection.CreateCommand(@"SELECT ""VolumeName"", ""JournalID"", ""NextUSN"", ""ConfigHash"" FROM ""ChangeJournalData"" WHERE ""FilesetID"" = @FilesetId") + await using var cmd = m_connection.CreateCommand(@" + SELECT + ""VolumeName"", + ""JournalID"", + ""NextUSN"", + ""ConfigHash"" + FROM ""ChangeJournalData"" + WHERE ""FilesetID"" = @FilesetId + ") + .SetTransaction(m_rtr) .SetParameterValue("@FilesetId", fileSetId); - using (var rd = cmd.ExecuteReader()) + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) { - while (rd.Read()) + yield return new Interface.USNJournalDataEntry { - data.Add(new Interface.USNJournalDataEntry - { - Volume = rd.ConvertValueToString(0), - JournalId = rd.ConvertValueToInt64(1), - NextUsn = rd.ConvertValueToInt64(2), - ConfigHash = rd.ConvertValueToString(3) - }); - } - - } - - return data; - } - - /// - /// Adds NTFS change journal data for file set and volume - /// - /// Data to add - /// An optional external transaction - public void CreateChangeJournalData(IEnumerable data, IDbTransaction? transaction = null) - { - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - { - foreach (var entry in data) - { - using (var cmd = m_connection.CreateCommand()) - { - cmd.Transaction = tr.Parent; - var c = cmd.SetCommandAndParameters( - @"INSERT INTO ""ChangeJournalData"" (""FilesetID"", ""VolumeName"", ""JournalID"", ""NextUSN"", ""ConfigHash"") VALUES (@FilesetId, @VolumeName, @JournalId, @NextUsn, @ConfigHash);") - .SetParameterValue("@FilesetId", m_filesetId) - .SetParameterValue("@VolumeName", entry.Volume) - .SetParameterValue("@JournalId", entry.JournalId) - .SetParameterValue("@NextUsn", entry.NextUsn) - .SetParameterValue("@ConfigHash", entry.ConfigHash) - .ExecuteNonQuery(); - - if (c != 1) - throw new Exception("Unable to add change journal entry"); - } - } - - tr.Commit(); + Volume = rd.ConvertValueToString(0), + JournalId = rd.ConvertValueToInt64(1), + NextUsn = rd.ConvertValueToInt64(2), + ConfigHash = rd.ConvertValueToString(3) + }; } } /// - /// Adds NTFS change journal data for file set and volume + /// Adds NTFS change journal data for file set and volume. /// - /// Data to add - /// Existing file set to update - /// An optional external transaction - public void UpdateChangeJournalData(IEnumerable data, long fileSetId, IDbTransaction? transaction = null) + /// Data to add. + /// The cancellation token to cancel the operation. + /// A task that completes when the data is added. + /// Thrown if unable to add change journal entry. + public async Task CreateChangeJournalData(IEnumerable data, CancellationToken token) { - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) + foreach (var entry in data) { - foreach (var entry in data) - { - using (var cmd = m_connection.CreateCommand()) - { - cmd.SetTransaction(tr.Parent) - .SetCommandAndParameters( - @"UPDATE ""ChangeJournalData"" SET ""NextUSN"" = @NextUsn WHERE ""FilesetID"" = @FilesetId AND ""VolumeName"" = @VolumeName AND ""JournalID"" = @JournalId;") - .SetParameterValue("@NextUsn", entry.NextUsn) - .SetParameterValue("@FilesetId", fileSetId) - .SetParameterValue("@VolumeName", entry.Volume) - .SetParameterValue("@JournalId", entry.JournalId) - .ExecuteNonQuery(); - } - } + await using var cmd = m_connection.CreateCommand(m_rtr); + var c = await cmd.SetCommandAndParameters(@" + INSERT INTO ""ChangeJournalData"" ( + ""FilesetID"", + ""VolumeName"", + ""JournalID"", + ""NextUSN"", + ""ConfigHash"" + ) + VALUES ( + @FilesetId, + @VolumeName, + @JournalId, + @NextUsn, + @ConfigHash + ); + ") + .SetParameterValue("@FilesetId", m_filesetId) + .SetParameterValue("@VolumeName", entry.Volume) + .SetParameterValue("@JournalId", entry.JournalId) + .SetParameterValue("@NextUsn", entry.NextUsn) + .SetParameterValue("@ConfigHash", entry.ConfigHash) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); - tr.Commit(); + if (c != 1) + throw new Exception("Unable to add change journal entry"); } + + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } /// - /// Checks if a blocklist hash is known + /// Adds NTFS change journal data for file set and volume. /// - /// The hash to check - /// An optional external transaction - /// True if the hash is known, false otherwise - public bool IsBlocklistHashKnown(string hash, IDbTransaction transaction) + /// Data to add. + /// Existing file set to update. + /// The cancellation token to cancel the operation. + /// A task that completes when the data is added. + public async Task UpdateChangeJournalData(IEnumerable data, long fileSetId, CancellationToken token) { - var res = m_getfirstfilesetwithblockinblockset.SetTransaction(transaction) + foreach (var entry in data) + { + await using var cmd = m_connection.CreateCommand(); + await cmd.SetCommandAndParameters(@" + UPDATE ""ChangeJournalData"" + SET ""NextUSN"" = @NextUsn + WHERE + ""FilesetID"" = @FilesetId + AND ""VolumeName"" = @VolumeName + AND ""JournalID"" = @JournalId; + ") + .SetTransaction(m_rtr) + .SetParameterValue("@NextUsn", entry.NextUsn) + .SetParameterValue("@FilesetId", fileSetId) + .SetParameterValue("@VolumeName", entry.Volume) + .SetParameterValue("@JournalId", entry.JournalId) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } + + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); + } + + /// + /// Checks if a blocklist hash is known. + /// + /// The hash to check. + /// The cancellation token to cancel the operation. + /// A task that when awaited returns true if the hash is known, false otherwise. + public async Task IsBlocklistHashKnown(string hash, CancellationToken token) + { + var res = await m_getfirstfilesetwithblockinblockset + .SetTransaction(m_rtr) .SetParameterValue("@Hash", hash) - .ExecuteScalarInt64(); + .ExecuteScalarInt64Async(token) + .ConfigureAwait(false); + if (res != -1 && res != m_filesetId) return true; else diff --git a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs index 73a2ba126..aeb5415e2 100644 --- a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs @@ -1,31 +1,37 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable using System; +using System.Threading; +using System.Threading.Tasks; using Duplicati.Library.Common.IO; namespace Duplicati.Library.Main.Database { + + /// + /// A local database for bug reports, which obfuscates sensitive data before generating a bug report. + /// internal class LocalBugReportDatabase : LocalDatabase { /// @@ -33,66 +39,144 @@ namespace Duplicati.Library.Main.Database /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalBugReportDatabase)); - public LocalBugReportDatabase(string path, long pagecachesize) - : base(path, "BugReportCreate", false, pagecachesize) + /// + /// Creates a new instance of the class. + /// + /// The path to the database file. + /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains a new instance of . + public static async Task CreateAsync(string path, LocalBugReportDatabase? dbnew, CancellationToken token) { - ShouldCloseConnection = true; + dbnew ??= new LocalBugReportDatabase(); + + dbnew = (LocalBugReportDatabase) + await CreateLocalDatabaseAsync(path, "BugReportCreate", false, dbnew, token) + .ConfigureAwait(false); + dbnew.ShouldCloseConnection = true; + + return dbnew; } - public void Fix() + /// + /// Obfuscates sensitive data in the database, readying it for a bug report. + /// + /// A cancellation token to cancel the operation. + /// A task that completes when the obfuscation is finished. + public async Task Fix(CancellationToken token) { - using (var tr = m_connection.BeginTransactionSafe()) - using (var cmd = m_connection.CreateCommand()) + await using var cmd = m_connection.CreateCommand(m_rtr); + var tablename = "PathMap-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + + // TODO: Rewrite this to use PathPrefix + // TODO: Needs to be much faster + await using (var upcmd = m_connection.CreateCommand(m_rtr)) { - cmd.Transaction = tr; - var tablename = "PathMap-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + await upcmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{tablename}"" ( + ""ID"" INTEGER PRIMARY KEY, + ""RealPath"" TEXT NOT NULL, + ""Obfuscated"" TEXT NULL + ) + ", token) + .ConfigureAwait(false); - // TODO: Rewrite this to use PathPrefix - // TODO: Needs to be much faster - using (var upcmd = m_connection.CreateCommand()) - { + await upcmd.ExecuteNonQueryAsync($@" + INSERT INTO ""{tablename}"" (""RealPath"") + SELECT DISTINCT ""Path"" + FROM ""File"" + ORDER BY ""Path"" + ", token) + .ConfigureAwait(false); - upcmd.Transaction = tr; - upcmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" (""ID"" INTEGER PRIMARY KEY, ""RealPath"" TEXT NOT NULL, ""Obfuscated"" TEXT NULL)")); - upcmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""{tablename}"" (""RealPath"") SELECT DISTINCT ""Path"" FROM ""File"" ORDER BY ""Path"" ")); - upcmd.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{tablename}"" SET ""Obfuscated"" = @StartPath || length(""RealPath"") || @DirSep || ""ID"" || (CASE WHEN substr(""RealPath"", length(""RealPath"")) = @DirSep THEN @DirSep ELSE @FileExt END) ")) - .SetParameterValue("@StartPath", !OperatingSystem.IsWindows() ? "/" : "X:\\") - .SetParameterValue("@DirSep", Util.DirectorySeparatorString) - .SetParameterValue("@FileExt", ".bin") - .ExecuteNonQuery(); + await upcmd.SetCommandAndParameters($@" + UPDATE ""{tablename}"" + SET ""Obfuscated"" = + @StartPath + || length(""RealPath"") + || @DirSep + || ""ID"" + || ( + CASE + WHEN substr(""RealPath"", length(""RealPath"")) = @DirSep + THEN @DirSep + ELSE @FileExt + END + ) + ") + .SetParameterValue("@StartPath", !OperatingSystem.IsWindows() ? "/" : "X:\\") + .SetParameterValue("@DirSep", Util.DirectorySeparatorString) + .SetParameterValue("@FileExt", ".bin") + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); - /*long id = 1; - using(var rd = cmd.ExecuteReader(string.Format(@"SELECT ""RealPath"", ""Obfuscated"" FROM ""{0}"" ", tablename))) - while(rd.Read()) - { - upcmd.SetCommandAndParameters(@"UPDATE ""LogData"" SET ""Message"" = replace(""Message"", @RealPath, @Obfuscated), ""Exception"" = replace(""Exception"", @RealPath, @Obfuscated)") - .SetParameterValue("@RealPath", rd.GetValue(0)) - .SetParameterValue("@Obfuscated", rd.GetValue(1)) - .ExecuteNonQuery(); - id++; - } - */ - } - - cmd.ExecuteNonQuery(@"UPDATE ""LogData"" SET ""Message"" = 'ERASED!' WHERE ""Message"" LIKE '%/%' OR ""Message"" LIKE '%:\%' "); - cmd.ExecuteNonQuery(@"UPDATE ""LogData"" SET ""Exception"" = 'ERASED!' WHERE ""Exception"" LIKE '%/%' OR ""Exception"" LIKE '%:\%' "); - - cmd.ExecuteNonQuery(@"UPDATE ""Configuration"" SET ""Value"" = 'ERASED!' WHERE ""Key"" = 'passphrase' "); - - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TABLE ""FixedFile"" AS SELECT ""B"".""ID"" AS ""ID"", ""A"".""Obfuscated"" AS ""Path"", ""B"".""BlocksetID"" AS ""BlocksetID"", ""B"".""MetadataID"" AS ""MetadataID"" FROM ""{tablename}"" ""A"", ""File"" ""B"" WHERE ""A"".""RealPath"" = ""B"".""Path"" ")); - cmd.ExecuteNonQuery(@"DROP VIEW ""File"" "); - cmd.ExecuteNonQuery(@"DROP TABLE ""FileLookup"" "); - cmd.ExecuteNonQuery(@"DROP TABLE ""PathPrefix"" "); - - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tablename}"" ")); - - using (new Logging.Timer(LOGTAG, "CommitUpdateBugReport", "CommitUpdateBugReport")) - tr.Commit(); - - cmd.Transaction = null; - - cmd.ExecuteNonQuery("VACUUM"); + /*long id = 1; + using(var rd = cmd.ExecuteReader(string.Format(@"SELECT ""RealPath"", ""Obfuscated"" FROM ""{0}"" ", tablename))) + while(rd.Read()) + { + upcmd.SetCommandAndParameters(@"UPDATE ""LogData"" SET ""Message"" = replace(""Message"", @RealPath, @Obfuscated), ""Exception"" = replace(""Exception"", @RealPath, @Obfuscated)") + .SetParameterValue("@RealPath", rd.GetValue(0)) + .SetParameterValue("@Obfuscated", rd.GetValue(1)) + .ExecuteNonQuery(); + id++; + } + */ } + + await cmd.ExecuteNonQueryAsync(@" + UPDATE ""LogData"" + SET ""Message"" = 'ERASED!' + WHERE + ""Message"" LIKE '%/%' + OR ""Message"" LIKE '%:\%' + ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync(@" + UPDATE ""LogData"" + SET ""Exception"" = 'ERASED!' + WHERE + ""Exception"" LIKE '%/%' + OR ""Exception"" LIKE '%:\%' + ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync(@" + UPDATE ""Configuration"" + SET ""Value"" = 'ERASED!' + WHERE ""Key"" = 'passphrase' + ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@" + CREATE TABLE ""FixedFile"" AS + SELECT + ""B"".""ID"" AS ""ID"", + ""A"".""Obfuscated"" AS ""Path"", + ""B"".""BlocksetID"" AS ""BlocksetID"", + ""B"".""MetadataID"" AS ""MetadataID"" + FROM + ""{tablename}"" ""A"", + ""File"" ""B"" + WHERE ""A"".""RealPath"" = ""B"".""Path"" + ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync(@"DROP VIEW ""File"" ", token) + .ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync(@"DROP TABLE ""FileLookup"" ", token) + .ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync(@"DROP TABLE ""PathPrefix"" ", token) + .ConfigureAwait(false); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tablename}"" ", token) + .ConfigureAwait(false); + + using (new Logging.Timer(LOGTAG, "CommitUpdateBugReport", "CommitUpdateBugReport")) + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); + + cmd.SetTransaction(m_rtr); + await cmd.ExecuteNonQueryAsync("VACUUM", token).ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 1f22b6976..70d47b4b1 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable @@ -24,21 +24,28 @@ using System; using System.Collections.Generic; using System.Data; -using System.Linq; -using System.Text; using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading.Tasks; using Duplicati.Library.Modules.Builtin.ResultSerialization; using Duplicati.Library.Utility; -using System.Runtime.CompilerServices; using Duplicati.Library.Interface; - +using Microsoft.Data.Sqlite; +using System.Threading; // Expose internal classes to UnitTests, so that Database classes can be tested [assembly: InternalsVisibleTo("Duplicati.UnitTest")] namespace Duplicati.Library.Main.Database { - internal class LocalDatabase : IDisposable + /// + /// Represents a local database for Duplicati operations. + /// This class provides methods to interact with the local SQLite database, including + /// managing remote volumes, logging operations, and handling transactions. + /// + internal class LocalDatabase : IDisposable, IAsyncDisposable { /// /// The tag used for logging @@ -51,45 +58,140 @@ namespace Duplicati.Library.Main.Database /// SQLite has a limit of 999 parameters in a single statement public const int CHUNK_SIZE = 128; - protected readonly IDbConnection m_connection; - protected readonly long m_operationid = -1; - protected readonly long m_pagecachesize; + // All of the required fields have been set to null! to ignore the compiler warning, as they will be initialized in the Create* factory methods. + /// + /// The SQLite connection to the local database. + /// + protected SqliteConnection m_connection = null!; + /// + /// The operation ID for the current operation. + /// + protected long m_operationid = -1; + /// + /// The size of the SQLite page cache. + /// + protected long m_pagecachesize; + /// + /// Indicates whether the database has executed a vacuum operation. + /// private bool m_hasExecutedVacuum; + /// + /// The reusable transaction for the current operation. + /// + protected ReusableTransaction m_rtr = null!; + /// + /// A read-only property that provides access to the current transaction. + /// + public ReusableTransaction Transaction { get { return m_rtr; } } - private readonly IDbCommand m_updateremotevolumeCommand; - private readonly IDbCommand m_selectremotevolumesCommand; - private readonly IDbCommand m_selectremotevolumeCommand; - private readonly IDbCommand m_removeremotevolumeCommand; - private readonly IDbCommand m_removedeletedremotevolumeCommand; - private readonly IDbCommand m_selectremotevolumeIdCommand; - private readonly IDbCommand m_createremotevolumeCommand; - private readonly IDbCommand m_selectduplicateRemoteVolumesCommand; + /// + /// The command used to update a remote volume in the database. + /// + private SqliteCommand m_updateremotevolumeCommand = null!; + /// + /// The command used to select remote volumes from the database. + /// + private SqliteCommand m_selectremotevolumesCommand = null!; + /// + /// The command used to select a specific remote volume by name. + /// + private SqliteCommand m_selectremotevolumeCommand = null!; + /// + /// The command used to remove a remote volume from the database. + /// + private SqliteCommand m_removeremotevolumeCommand = null!; + /// + /// The command used to remove deleted remote volumes from the database. + /// + private SqliteCommand m_removedeletedremotevolumeCommand = null!; + /// + /// The command used to select the ID of a remote volume by name. + /// + private SqliteCommand m_selectremotevolumeIdCommand = null!; + /// + /// The command used to create a new remote volume in the database. + /// + private SqliteCommand m_createremotevolumeCommand = null!; + /// + /// The command used to select duplicate remote volumes from the database. + /// + private SqliteCommand m_selectduplicateRemoteVolumesCommand = null!; - private readonly IDbCommand m_insertlogCommand; - private readonly IDbCommand m_insertremotelogCommand; - private readonly IDbCommand m_insertIndexBlockLink; + /// + /// The command used to insert log data into the database. + /// + private SqliteCommand m_insertlogCommand = null!; + /// + /// The command used to insert remote operation logs into the database. + /// + private SqliteCommand m_insertremotelogCommand = null!; + /// + /// The command used to insert index block links into the database. + /// + private SqliteCommand m_insertIndexBlockLink = null!; - private readonly IDbCommand m_findpathprefixCommand; - private readonly IDbCommand m_insertpathprefixCommand; + /// + /// The command used to find a path prefix in the database. + /// + private SqliteCommand m_findpathprefixCommand = null!; + /// + /// The command used to insert a new path prefix into the database. + /// + private SqliteCommand m_insertpathprefixCommand = null!; + /// + /// A constant representing the blockset ID for a folder. + /// public const long FOLDER_BLOCKSET_ID = -100; + /// + /// A constant representing the blockset ID for a symlink. + /// public const long SYMLINK_BLOCKSET_ID = -200; + /// + /// The timestamp of the operation being performed. + /// public DateTime OperationTimestamp { get; private set; } - internal IDbConnection Connection { get { return m_connection; } } + /// + /// A read-only property that provides access to the internal SQLite connection. + /// + internal SqliteConnection Connection { get { return m_connection; } } + /// + /// Indicates whether the database has been disposed. + /// public bool IsDisposed { get; private set; } + /// + /// Indicates whether the connection should be closed when the database is disposed. + /// public bool ShouldCloseConnection { get; set; } - protected static IDbConnection CreateConnection(string path, long pagecachesize) + // Constructor is private to force use of CreateLocalDatabaseAsync + protected LocalDatabase() { } + + // Arguments are not used, but are required to match the constructor signatures + [Obsolete("Calling the constructor will throw an exception. Use the CreateLocalDatabaseAsync or CreateLocalDatabase functions instead")] + public LocalDatabase(object? _ignore1 = null, object? _ignore2 = null, object? _ignore3 = null, object? _ignore4 = null) + { + throw new NotImplementedException("Use the CreateLocalDatabaseAsync or CreateLocalDatabase functions instead"); + } + + /// + /// Creates a new SQLite connection to the specified database path with the given page cache size. + /// This method ensures that the directory for the database exists and upgrades the database schema if necessary. + /// + /// The path to the SQLite database file. + /// A task that, when awaited, returns a new to the specified database. + protected static async Task CreateConnectionAsync(string path) { path = Path.GetFullPath(path); if (!Directory.Exists(Path.GetDirectoryName(path))) Directory.CreateDirectory(Path.GetDirectoryName(path) ?? throw new DirectoryNotFoundException("Path was a root folder.")); - var c = SQLiteHelper.SQLiteLoader.LoadConnection(path, pagecachesize); + var c = await SQLiteHelper.SQLiteLoader.LoadConnectionAsync(path) + .ConfigureAwait(false); try { @@ -98,7 +200,7 @@ namespace Duplicati.Library.Main.Database catch { //Don't leak database connections when something goes wrong - c.Dispose(); + await c.DisposeAsync().ConfigureAwait(false); throw; } @@ -106,125 +208,362 @@ namespace Duplicati.Library.Main.Database } /// - /// Formats the string using the invariant culture + /// Creates a new instance of with the specified parameters. + /// This method initializes the database connection, sets the operation timestamp, and prepares the necessary commands for database operations. /// - /// The formattable string - /// The formatted string - public static string FormatInvariant(FormattableString formattable) - => Library.Utility.Utility.FormatInvariant(formattable); - - public static bool Exists(string path) + /// The path to the SQLite database file. + /// The description of the operation being performed. + /// Indicates whether the connection should be closed when the database is disposed. + /// An optional existing instance to use. If not provided, a new instance will be created. + /// Cancellation token to monitor for cancellation requests. + /// A task that, when awaited, returns a new instance of . + public static async Task CreateLocalDatabaseAsync(string path, string operation, bool shouldclose, LocalDatabase? db, CancellationToken token) { - return File.Exists(path); + db ??= new LocalDatabase(); + + var connection = await CreateConnectionAsync(path) + .ConfigureAwait(false); + db = await CreateLocalDatabaseAsync(connection, operation, db, token) + .ConfigureAwait(false); + + db.ShouldCloseConnection = shouldclose; + + return db; } /// - /// Creates a new database instance and starts a new operation + /// Creates a new instance of based on an existing database instance. + /// This method copies the connection and transaction from the parent database and initializes the new database with the same operation timestamp and ID. /// - /// The path to the database - /// The name of the operation. If null, continues last operation - /// Should the connection be closed when this object is disposed - /// The page cache size - public LocalDatabase(string path, string operation, bool shouldclose, long pagecachesize) - : this(CreateConnection(path, pagecachesize), operation) + /// The parent instance to copy from. + /// An optional existing instance to use. If not provided, a new instance will be created. + /// Cancellation token to monitor for cancellation requests. + /// A task that, when awaited, returns a new instance of . + public static async Task CreateLocalDatabaseAsync(LocalDatabase dbparent, LocalDatabase? dbnew, CancellationToken token) { - ShouldCloseConnection = shouldclose; - m_pagecachesize = pagecachesize; + dbnew ??= new LocalDatabase(); + + dbnew.m_connection = dbparent.m_connection; + dbnew.m_rtr = dbparent.m_rtr; + dbnew = await CreateLocalDatabaseAsync(dbparent.m_connection, dbnew, token) + .ConfigureAwait(false); + + dbnew.OperationTimestamp = dbparent.OperationTimestamp; + dbnew.m_operationid = dbparent.m_operationid; + dbnew.m_pagecachesize = dbparent.m_pagecachesize; + + return dbnew; } /// - /// Creates a new database instance and starts a new operation + /// Creates a new instance of with the specified SQLite connection and operation description. + /// This method initializes the database connection, sets the operation timestamp, and prepares the necessary commands for database operations. /// - public LocalDatabase(LocalDatabase db) - : this(db.m_connection) + /// The SQLite connection to use for the database operations. + /// The description of the operation being performed. + /// An optional existing instance to use. If not provided, a new instance will be created. + /// Cancellation token to monitor for cancellation requests. + /// A task that, when awaited, returns a new instance of . + public static async Task CreateLocalDatabaseAsync(SqliteConnection connection, string operation, LocalDatabase? dbnew, CancellationToken token) { - OperationTimestamp = db.OperationTimestamp; - m_connection = db.m_connection; - m_operationid = db.m_operationid; - m_pagecachesize = db.m_pagecachesize; - } + dbnew ??= new LocalDatabase(); + dbnew = await CreateLocalDatabaseAsync(connection, dbnew, token) + .ConfigureAwait(false); - /// - /// Creates a new database instance and starts a new operation - /// - /// The name of the operation. If null, continues last operation - public LocalDatabase(IDbConnection connection, string operation) - : this(connection) - { - OperationTimestamp = DateTime.UtcNow; - m_connection = connection; + dbnew.OperationTimestamp = DateTime.UtcNow; - if (m_connection.State != ConnectionState.Open) - m_connection.Open(); + if (dbnew.m_connection.State != ConnectionState.Open) + await dbnew.m_connection.OpenAsync(token).ConfigureAwait(false); + await using var cmd = dbnew.m_connection.CreateCommand() + .SetTransaction(dbnew.m_rtr); if (operation != null) { - using (var cmd = m_connection.CreateCommand()) - m_operationid = cmd.SetCommandAndParameters(@"INSERT INTO ""Operation"" (""Description"", ""Timestamp"") VALUES (@Description, @Timestamp); SELECT last_insert_rowid();") - .SetParameterValue("@Description", operation) - .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(OperationTimestamp)) - .ExecuteScalarInt64(-1); + dbnew.m_operationid = await cmd.SetCommandAndParameters(@" + INSERT INTO ""Operation"" ( + ""Description"", + ""Timestamp"" + ) + VALUES ( + @Description, + @Timestamp + ); + SELECT last_insert_rowid(); + ") + .SetParameterValue("@Description", operation) + .SetParameterValue("@Timestamp", dbnew.OperationTimestamp) + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); } else { // Get last operation - using (var cmd = m_connection.CreateCommand()) - using (var rd = cmd.ExecuteReader(@"SELECT ""ID"", ""Timestamp"" FROM ""Operation"" ORDER BY ""Timestamp"" DESC LIMIT 1")) - { - if (!rd.Read()) - throw new Exception("LocalDatabase does not contain a previous operation."); + await using var rd = await cmd.ExecuteReaderAsync(@" + SELECT + ""ID"", + ""Timestamp"" + FROM ""Operation"" + ORDER BY ""Timestamp"" + DESC LIMIT 1 + ", token) + .ConfigureAwait(false); - m_operationid = rd.ConvertValueToInt64(0); - OperationTimestamp = ParseFromEpochSeconds(rd.ConvertValueToInt64(1)); - } + if (!await rd.ReadAsync(token).ConfigureAwait(false)) + throw new Exception("LocalDatabase does not contain a previous operation."); + + dbnew.m_operationid = rd.ConvertValueToInt64(0); + dbnew.OperationTimestamp = ParseFromEpochSeconds(rd.ConvertValueToInt64(1)); } - } - private LocalDatabase(IDbConnection connection) - { - m_connection = connection; - m_insertlogCommand = connection.CreateCommand(@"INSERT INTO ""LogData"" (""OperationID"", ""Timestamp"", ""Type"", ""Message"", ""Exception"") VALUES (@OperationID, @Timestamp, @Type, @Message, @Exception)"); - m_insertremotelogCommand = connection.CreateCommand(@"INSERT INTO ""RemoteOperation"" (""OperationID"", ""Timestamp"", ""Operation"", ""Path"", ""Data"") VALUES (@OperationID, @Timestamp, @Operation, @Path, @Data)"); - m_updateremotevolumeCommand = connection.CreateCommand(@"UPDATE ""Remotevolume"" SET ""OperationID"" = @OperationID, ""State"" = @State, ""Hash"" = @Hash, ""Size"" = @Size WHERE ""Name"" = @Name"); - m_selectremotevolumesCommand = connection.CreateCommand(@"SELECT ""ID"", ""Name"", ""Type"", ""Size"", ""Hash"", ""State"", ""DeleteGraceTime"", ""ArchiveTime"" FROM ""Remotevolume"""); - m_selectremotevolumeCommand = connection.CreateCommand(m_selectremotevolumesCommand.CommandText + @" WHERE ""Name"" = @Name"); - m_selectduplicateRemoteVolumesCommand = connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""Name"", ""State"" FROM ""Remotevolume"" WHERE ""Name"" IN (SELECT ""Name"" FROM ""Remotevolume"" WHERE ""State"" IN ('{RemoteVolumeState.Deleted.ToString()}', '{RemoteVolumeState.Deleting.ToString()}')) AND NOT ""State"" IN ('{RemoteVolumeState.Deleted.ToString()}', '{RemoteVolumeState.Deleting.ToString()}')")); - m_removeremotevolumeCommand = connection.CreateCommand(@"DELETE FROM ""Remotevolume"" WHERE ""Name"" = @Name AND (""DeleteGraceTime"" < @Now OR ""State"" != @State)"); - m_removedeletedremotevolumeCommand = connection.CreateCommand(FormatInvariant($@"DELETE FROM ""Remotevolume"" WHERE ""State"" == '{RemoteVolumeState.Deleted.ToString()}' AND (""DeleteGraceTime"" < @Now OR LENGTH(""DeleteGraceTime"") > 12) ")); // >12 is to handle removal of old records that were in ticks - m_selectremotevolumeIdCommand = connection.CreateCommand(@"SELECT ""ID"" FROM ""Remotevolume"" WHERE ""Name"" = @Name"); - m_createremotevolumeCommand = connection.CreateCommand(@"INSERT INTO ""Remotevolume"" (""OperationID"", ""Name"", ""Type"", ""State"", ""Size"", ""VerificationCount"", ""DeleteGraceTime"", ""ArchiveTime"") VALUES (@OperationID, @Name, @Type, @State, @Size, @VerificationCount, @DeleteGraceTime, @ArchiveTime); SELECT last_insert_rowid();"); - m_insertIndexBlockLink = connection.CreateCommand(@"INSERT INTO ""IndexBlockLink"" (""IndexVolumeID"", ""BlockVolumeID"") VALUES (@IndexVolumeId, @BlockVolumeId)"); - m_findpathprefixCommand = connection.CreateCommand(@"SELECT ""ID"" FROM ""PathPrefix"" WHERE ""Prefix"" = @Prefix"); - m_insertpathprefixCommand = connection.CreateCommand(@"INSERT INTO ""PathPrefix"" (""Prefix"") VALUES (@Prefix); SELECT last_insert_rowid(); "); + return dbnew; } /// - /// Creates a DateTime instance by adding the specified number of seconds to the EPOCH value - /// + /// Creates a new instance of with the specified SQLite connection. + /// This method initializes the database connection, sets up the reusable transaction, and prepares the necessary commands for database operations. + /// + /// The SQLite connection to use for the database operations. + /// An optional existing instance to use. If not provided, a new instance will be created. + /// Cancellation token to monitor for cancellation requests. + /// A task that, when awaited, returns a new instance of . + private static async Task CreateLocalDatabaseAsync(SqliteConnection connection, LocalDatabase? dbnew, CancellationToken token) + { + dbnew ??= new LocalDatabase(); + dbnew.m_connection = connection; + dbnew.m_rtr ??= new ReusableTransaction(connection); + + var selectremotevolumes_sql = @" + SELECT + ""ID"", + ""Name"", + ""Type"", + ""Size"", + ""Hash"", + ""State"", + ""DeleteGraceTime"", + ""ArchiveTime"" + FROM ""Remotevolume"" + "; + + dbnew.m_insertlogCommand = await connection.CreateCommandAsync(@" + INSERT INTO ""LogData"" ( + ""OperationID"", + ""Timestamp"", + ""Type"", + ""Message"", + ""Exception"" + ) + VALUES ( + @OperationID, + @Timestamp, + @Type, + @Message, + @Exception + ) + ", token) + .ConfigureAwait(false); + + dbnew.m_insertremotelogCommand = await connection.CreateCommandAsync(@" + INSERT INTO ""RemoteOperation"" ( + ""OperationID"", + ""Timestamp"", + ""Operation"", + ""Path"", + ""Data"" + ) + VALUES ( + @OperationID, + @Timestamp, + @Operation, + @Path, + @Data + ) + ", token) + .ConfigureAwait(false); + + dbnew.m_updateremotevolumeCommand = await connection.CreateCommandAsync(@" + UPDATE ""Remotevolume"" + SET + ""OperationID"" = @OperationID, + ""State"" = @State, + ""Hash"" = @Hash, + ""Size"" = @Size + WHERE ""Name"" = @Name + ", token) + .ConfigureAwait(false); + + dbnew.m_selectremotevolumesCommand = + await connection.CreateCommandAsync(selectremotevolumes_sql, token) + .ConfigureAwait(false); + + dbnew.m_selectremotevolumeCommand = await connection.CreateCommandAsync(@$" + {selectremotevolumes_sql} + WHERE ""Name"" = @Name + ", token) + .ConfigureAwait(false); + + dbnew.m_selectduplicateRemoteVolumesCommand = await connection.CreateCommandAsync($@" + SELECT DISTINCT + ""Name"", + ""State"" + FROM ""Remotevolume"" + WHERE + ""Name"" IN ( + SELECT ""Name"" + FROM ""Remotevolume"" + WHERE ""State"" IN ('{RemoteVolumeState.Deleted}', '{RemoteVolumeState.Deleting}') + ) + AND NOT ""State"" IN ('{RemoteVolumeState.Deleted}', '{RemoteVolumeState.Deleting}') + ", token) + .ConfigureAwait(false); + + dbnew.m_removeremotevolumeCommand = await connection.CreateCommandAsync(@" + DELETE FROM ""Remotevolume"" + WHERE + ""Name"" = @Name + AND ( + ""DeleteGraceTime"" < @Now + OR ""State"" != @State + ) + ", token) + .ConfigureAwait(false); + + // >12 is to handle removal of old records that were in ticks + dbnew.m_removedeletedremotevolumeCommand = await connection.CreateCommandAsync($@" + DELETE FROM ""Remotevolume"" + WHERE + ""State"" == '{RemoteVolumeState.Deleted}' + AND ( + ""DeleteGraceTime"" < @Now + OR LENGTH(""DeleteGraceTime"") > 12 + ) + ", token) + .ConfigureAwait(false); + + dbnew.m_selectremotevolumeIdCommand = await connection.CreateCommandAsync(@" + SELECT ""ID"" + FROM ""Remotevolume"" + WHERE ""Name"" = @Name + ", token) + .ConfigureAwait(false); + + dbnew.m_createremotevolumeCommand = await connection.CreateCommandAsync(@" + INSERT INTO ""Remotevolume"" ( + ""OperationID"", + ""Name"", + ""Type"", + ""State"", + ""Size"", + ""VerificationCount"", + ""DeleteGraceTime"", + ""ArchiveTime"" + ) + VALUES ( + @OperationID, + @Name, + @Type, + @State, + @Size, + @VerificationCount, + @DeleteGraceTime, + @ArchiveTime + ); + SELECT last_insert_rowid(); + ", token) + .ConfigureAwait(false); + + dbnew.m_insertIndexBlockLink = await connection.CreateCommandAsync(@" + INSERT INTO ""IndexBlockLink"" ( + ""IndexVolumeID"", + ""BlockVolumeID"" + ) + VALUES ( + @IndexVolumeId, + @BlockVolumeId + ) + ", token) + .ConfigureAwait(false); + + dbnew.m_findpathprefixCommand = await connection.CreateCommandAsync(@" + SELECT ""ID"" + FROM ""PathPrefix"" + WHERE ""Prefix"" = @Prefix + ", token) + .ConfigureAwait(false); + + dbnew.m_insertpathprefixCommand = await connection.CreateCommandAsync(@" + INSERT INTO ""PathPrefix"" (""Prefix"") + VALUES (@Prefix); + SELECT last_insert_rowid(); + ", token) + .ConfigureAwait(false); + + return dbnew; + } + + /// + /// Creates a DateTime instance by adding the specified number of seconds to the EPOCH value. + /// + /// The number of seconds since the EPOCH (January 1, 1970). + /// A DateTime instance representing the specified number of seconds since the EPOCH. public static DateTime ParseFromEpochSeconds(long seconds) { return Library.Utility.Utility.EPOCH.AddSeconds(seconds); } - public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, IDbTransaction? transaction = null) + /// + /// Updates the state of a remote volume in the database. + /// + /// The name of the remote volume to update. + /// The new state of the remote volume. + /// The size of the remote volume in bytes. + /// The hash of the remote volume, or null if not applicable. + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the remote volume has been updated. + public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, CancellationToken token) { - UpdateRemoteVolume(name, state, size, hash, false, transaction); + await UpdateRemoteVolume(name, state, size, hash, false, token) + .ConfigureAwait(false); } - public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, bool suppressCleanup, IDbTransaction? transaction = null) + /// + /// Updates the state of a remote volume in the database. + /// + /// The name of the remote volume to update. + /// The new state of the remote volume. + /// The size of the remote volume in bytes. + /// The hash of the remote volume, or null if not applicable. + /// If true, suppresses cleanup of the remote volume after updating. + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the remote volume has been updated. + public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, bool suppressCleanup, CancellationToken token) { - UpdateRemoteVolume(name, state, size, hash, suppressCleanup, new TimeSpan(0), null, transaction); + await UpdateRemoteVolume(name, state, size, hash, suppressCleanup, new TimeSpan(0), null, token) + .ConfigureAwait(false); } - public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, bool suppressCleanup, TimeSpan deleteGraceTime, bool? setArchived, IDbTransaction? transaction = null) + /// + /// Updates the state of a remote volume in the database. + /// + /// The name of the remote volume to update. + /// The new state of the remote volume. + /// The size of the remote volume in bytes. + /// The hash of the remote volume, or null if not applicable. + /// If true, suppresses cleanup of the remote volume after updating. + /// The time after which the remote volume can be deleted. + /// If true, sets the remote volume as archived. + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the remote volume has been updated. + public async Task UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string? hash, bool suppressCleanup, TimeSpan deleteGraceTime, bool? setArchived, CancellationToken token) { - m_updateremotevolumeCommand.Transaction = transaction; - var c = m_updateremotevolumeCommand.SetParameterValue("@OperationID", m_operationid) + var c = await m_updateremotevolumeCommand.SetTransaction(m_rtr) + .SetParameterValue("@OperationID", m_operationid) .SetParameterValue("@State", state.ToString()) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) .SetParameterValue("@Name", name) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); if (c != 1) { @@ -233,52 +572,92 @@ namespace Duplicati.Library.Main.Database if (deleteGraceTime.Ticks > 0) { - using (var cmd = m_connection.CreateCommand(transaction)) - { - c = cmd.SetCommandAndParameters(@"UPDATE ""RemoteVolume"" SET ""DeleteGraceTime"" = @DeleteGraceTime WHERE ""Name"" = @Name ") - .SetParameterValue("@DeleteGraceTime", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow + deleteGraceTime)) - .SetParameterValue("@Name", name) - .ExecuteNonQuery(); + await using var cmd = await m_connection.CreateCommandAsync(@" + UPDATE ""RemoteVolume"" + SET ""DeleteGraceTime"" = @DeleteGraceTime + WHERE ""Name"" = @Name + ", token) + .ConfigureAwait(false); - if (c != 1) - throw new Exception($"Unexpected number of updates when recording remote volume updates: {c}!"); - } + c = await cmd.SetTransaction(m_rtr) + .SetParameterValue("@DeleteGraceTime", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow + deleteGraceTime)) + .SetParameterValue("@Name", name) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + if (c != 1) + throw new Exception($"Unexpected number of updates when recording remote volume updates: {c}!"); } if (setArchived.HasValue) { - using (var cmd = m_connection.CreateCommand(transaction)) - { - c = cmd.SetCommandAndParameters(@"UPDATE ""RemoteVolume"" SET ""ArchiveTime"" = @ArchiveTime WHERE ""Name"" = @Name ") - .SetParameterValue("@ArchiveTime", setArchived.Value ? Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow) : 0) - .SetParameterValue("@Name", name) - .ExecuteNonQuery(); + await using var cmd = await m_connection.CreateCommandAsync(@" + UPDATE ""RemoteVolume"" + SET ""ArchiveTime"" = @ArchiveTime + WHERE ""Name"" = @Name + ", token) + .ConfigureAwait(false); - if (c != 1) - throw new Exception($"Unexpected number of updates when recording remote volume archive-time updates: {c}!"); - } + c = await cmd.SetTransaction(m_rtr) + .SetParameterValue("@ArchiveTime", setArchived.Value ? Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow) : 0) + .SetParameterValue("@Name", name) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + if (c != 1) + throw new Exception($"Unexpected number of updates when recording remote volume archive-time updates: {c}!"); } if (!suppressCleanup && state == RemoteVolumeState.Deleted) { - RemoveRemoteVolume(name, transaction); + await RemoveRemoteVolume(name, token).ConfigureAwait(false); } } - public IEnumerable> FilesetTimes + /// + /// Gets the ID and timestamp of all filesets in the database, ordered by timestamp in descending order. + /// + /// Cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of key-value pairs, where each pair contains the fileset ID and its timestamp. + public async IAsyncEnumerable> FilesetTimes([EnumeratorCancellation] CancellationToken token) { - get - { - using (var cmd = m_connection.CreateCommand()) - using (var rd = cmd.ExecuteReader(@"SELECT ""ID"", ""Timestamp"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC")) - while (rd.Read()) - yield return new KeyValuePair(rd.ConvertValueToInt64(0), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime()); - } + await using var cmd = await m_connection.CreateCommandAsync(@" + SELECT + ""ID"", + ""Timestamp"" + FROM ""Fileset"" + ORDER BY ""Timestamp"" DESC + ", token) + .ConfigureAwait(false); + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + yield return new KeyValuePair( + rd.ConvertValueToInt64(0), + ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime() + ); } - public (string Query, Dictionary Values) GetFilelistWhereClause(DateTime time, long[]? versions, IEnumerable>? filesetslist = null, bool singleTimeMatch = false) + /// + /// Generates a SQL WHERE clause for filtering file lists based on a specified time and versions. + /// + /// The time to filter files by. If Ticks is 0, it will not be used in the query. + /// An array of versions to filter files by. If null or empty, it will not be used in the query. + /// An optional list of filesets to use for filtering. If null, it will fetch all filesets from the database. + /// If true, matches files with the exact timestamp; otherwise, matches files with timestamps less than or equal to the specified time. + /// Cancellation token to monitor for cancellation requests. + /// A task that, when awaited, returns a tuple containing the SQL WHERE clause and a dictionary of parameter values. + /// Thrown if the provided time is unspecified. + public async Task<(string Query, Dictionary Values)> GetFilelistWhereClause(DateTime time, long[]? versions, IEnumerable>? filesetslist, bool singleTimeMatch, CancellationToken token) { - var filesets = (filesetslist ?? FilesetTimes).ToArray(); + KeyValuePair[] filesets; + if (filesetslist != null) + filesets = [.. filesetslist]; + else + filesets = await FilesetTimes(token) + .ToArrayAsync(cancellationToken: token) + .ConfigureAwait(false); + var query = new StringBuilder(); var args = new Dictionary(); if (time.Ticks > 0 || (versions != null && versions.Length > 0)) @@ -305,7 +684,7 @@ namespace Duplicati.Library.Main.Database var argName = "@Fileset" + v; args.Add(argName, filesets[v].Key); qs.Append(argName); - qs.Append(","); + qs.Append(','); } else Logging.Log.WriteWarningMessage(LOGTAG, "SkipInvalidVersion", null, "Skipping invalid version: {0}", v); @@ -329,32 +708,63 @@ namespace Duplicati.Library.Main.Database return (query.ToString(), args); } - public long GetRemoteVolumeID(string file, IDbTransaction? transaction = null) + /// + /// Gets the ID of a remote volume by its name. + /// + /// The name of the remote volume. + /// Cancellation token to monitor for cancellation requests. + /// A task that, when awaited, returns the ID of the remote volume. If the volume does not exist, it returns -1. + public async Task GetRemoteVolumeID(string file, CancellationToken token) { - m_selectremotevolumeIdCommand.Transaction = transaction; - return m_selectremotevolumeIdCommand.SetParameterValue("@Name", file).ExecuteScalarInt64(-1); + return await m_selectremotevolumeIdCommand + .SetTransaction(m_rtr) + .SetParameterValue("@Name", file) + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); } - public IEnumerable> GetRemoteVolumeIDs(IEnumerable files, IDbTransaction? transaction = null) + /// + /// Gets the IDs of remote volumes for a list of files. + /// + /// An enumerable collection of file names. + /// Cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of key-value pairs, where each pair contains the file name and its corresponding remote volume ID. + public async IAsyncEnumerable> GetRemoteVolumeIDs(IEnumerable files, [EnumeratorCancellation] CancellationToken token) { - using (var cmd = m_connection.CreateCommand(transaction)) - { - using var tmptable = new TemporaryDbValueList(m_connection, transaction, files); - cmd.SetCommandAndParameters(@"SELECT ""Name"", ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Name)") - .ExpandInClauseParameter("@Name", tmptable); + await using var cmd = await m_connection.CreateCommandAsync(@" + SELECT + ""Name"", + ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" IN (@Name) + ", token) + .ConfigureAwait(false); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); - } + cmd.SetTransaction(m_rtr); + await using var tmptable = await TemporaryDbValueList.CreateAsync(this, files, token) + .ConfigureAwait(false); + await cmd.ExpandInClauseParameterMssqliteAsync("@Name", tmptable, token) + .ConfigureAwait(false); + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); } - public RemoteVolumeEntry GetRemoteVolume(string file, IDbTransaction? transaction = null) + /// + /// Gets a remote volume entry by its file name. + /// + /// The name of the remote volume file. + /// Cancellation token to monitor for cancellation requests. + /// A task that, when awaited, returns a representing the remote volume. If the volume does not exist, it returns an empty entry. + public async Task GetRemoteVolume(string file, CancellationToken token) { - m_selectremotevolumeCommand.Transaction = transaction; - m_selectremotevolumeCommand.SetParameterValue("@Name", file); - using (var rd = m_selectremotevolumeCommand.ExecuteReader()) - if (rd.Read()) + m_selectremotevolumeCommand + .SetTransaction(m_rtr) + .SetParameterValue("@Name", file); + + await using (var rd = await m_selectremotevolumeCommand.ExecuteReaderAsync(token).ConfigureAwait(false)) + if (await rd.ReadAsync(token).ConfigureAwait(false)) return new RemoteVolumeEntry( rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), @@ -369,707 +779,1352 @@ namespace Duplicati.Library.Main.Database return RemoteVolumeEntry.Empty; } - public IEnumerable> DuplicateRemoteVolumes(IDbTransaction? transaction) + /// + /// Gets remote volumes that are duplicates, meaning they have the same name but different states. + /// + /// Cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of key-value pairs, where each pair contains the name of the remote volume and its state. + public async IAsyncEnumerable> DuplicateRemoteVolumes([EnumeratorCancellation] CancellationToken token) { - foreach (var rd in m_selectduplicateRemoteVolumesCommand.SetTransaction(transaction).ExecuteReaderEnumerable()) + m_selectduplicateRemoteVolumesCommand.SetTransaction(m_rtr); + + await foreach (var rd in m_selectduplicateRemoteVolumesCommand.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) { yield return new KeyValuePair( rd.ConvertValueToString(0) ?? throw new Exception("Name was null"), - (RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.ConvertValueToString(1) ?? "") + (RemoteVolumeState)Enum.Parse( + typeof(RemoteVolumeState), rd.ConvertValueToString(1) ?? "" + ) ); } } - public IEnumerable GetRemoteVolumes(IDbTransaction? transaction = null) + /// + /// Gets all remote volumes from the database. + /// + /// Cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of representing all remote volumes. + public async IAsyncEnumerable GetRemoteVolumes([EnumeratorCancellation] CancellationToken token) { - m_selectremotevolumesCommand.Transaction = transaction; - using (var rd = m_selectremotevolumesCommand.ExecuteReader()) + m_selectremotevolumesCommand.SetTransaction(m_rtr); + await using var rd = await m_selectremotevolumesCommand + .ExecuteReaderAsync(token) + .ConfigureAwait(false); + + while (await rd.ReadAsync(token).ConfigureAwait(false)) { - while (rd.Read()) - { - yield return new RemoteVolumeEntry( - rd.ConvertValueToInt64(0), - rd.ConvertValueToString(1), - rd.ConvertValueToString(4), - rd.ConvertValueToInt64(3, -1), - (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), rd.ConvertValueToString(2) ?? ""), - (RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.ConvertValueToString(5) ?? ""), - ParseFromEpochSeconds(rd.ConvertValueToInt64(6, 0)), - ParseFromEpochSeconds(rd.ConvertValueToInt64(7, 0)) - ); - } + yield return new RemoteVolumeEntry( + rd.ConvertValueToInt64(0), + rd.ConvertValueToString(1), + rd.ConvertValueToString(4), + rd.ConvertValueToInt64(3, -1), + (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), rd.ConvertValueToString(2) ?? ""), + (RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.ConvertValueToString(5) ?? ""), + ParseFromEpochSeconds(rd.ConvertValueToInt64(6, 0)), + ParseFromEpochSeconds(rd.ConvertValueToInt64(7, 0)) + ); } } /// - /// Log an operation performed on the remote backend + /// Log an operation performed on the remote backend. /// - /// The operation performed - /// The path involved - /// Any data relating to the operation - public void LogRemoteOperation(string operation, string path, string? data, IDbTransaction? transaction) + /// The operation performed. + /// The path involved. + /// Any data relating to the operation. + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the operation log has been recorded. + public async Task LogRemoteOperation(string operation, string path, string? data, CancellationToken token) { - m_insertremotelogCommand + await m_insertremotelogCommand + .SetTransaction(m_rtr) .SetParameterValue("@OperationID", m_operationid) .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow)) .SetParameterValue("@Operation", operation) .SetParameterValue("@Path", path) .SetParameterValue("@Data", data) - .ExecuteNonQuery(transaction); + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); } /// - /// Log a debug message + /// Log a debug message. /// - /// The message type - /// The message - /// An optional exception - public void LogMessage(string type, string message, Exception? exception, IDbTransaction? transaction) + /// The message type. + /// The message. + /// An optional exception. + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the log message has been recorded. + public async Task LogMessage(string type, string message, Exception? exception, CancellationToken token) { - m_insertlogCommand.SetParameterValue("@OperationID", m_operationid) + await m_insertlogCommand + .SetTransaction(m_rtr) + .SetParameterValue("@OperationID", m_operationid) .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow)) .SetParameterValue("@Type", type) .SetParameterValue("@Message", message) .SetParameterValue("@Exception", exception?.ToString()) - .ExecuteNonQuery(transaction); + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); } - public void UnlinkRemoteVolume(string name, RemoteVolumeState state, IDbTransaction? transaction = null) + /// + /// Unlinks a remote volume from the database. + /// This operation removes the remote volume entry from the database without deleting the actual volume. + /// + /// The name of the remote volume to unlink. + /// The state of the remote volume to unlink. + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the remote volume has been unlinked. + /// Thrown if the number of unlinked remote volumes is not equal to 1. + public async Task UnlinkRemoteVolume(string name, RemoteVolumeState state, CancellationToken token) { - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - using (var cmd = m_connection.CreateCommand(tr.Parent)) - { - var c = cmd.SetCommandAndParameters(@"DELETE FROM ""RemoteVolume"" WHERE ""Name"" = @Name AND ""State"" = @State ") - .SetParameterValue("@Name", name) - .SetParameterValue("@State", state.ToString()) - .ExecuteNonQuery(); + await using var cmd = await m_connection.CreateCommandAsync(@" + DELETE FROM ""RemoteVolume"" + WHERE + ""Name"" = @Name + AND ""State"" = @State + ", token) + .ConfigureAwait(false); - if (c != 1) - throw new Exception($"Unexpected number of remote volumes deleted: {c}, expected {1}"); + var c = await cmd.SetTransaction(m_rtr) + .SetParameterValue("@Name", name) + .SetParameterValue("@State", state.ToString()) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); - tr.Commit(); - } + if (c != 1) + throw new Exception($"Unexpected number of remote volumes deleted: {c}, expected {1}"); } - public void RemoveRemoteVolume(string name, IDbTransaction? transaction = null) + /// + /// Removes a remote volume from the database. + /// + /// The name of the remote volume to remove. + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the remote volume has been removed. + public async Task RemoveRemoteVolume(string name, CancellationToken token) { - RemoveRemoteVolumes([name], transaction); + await RemoveRemoteVolumes([name], token).ConfigureAwait(false); } - public void RemoveRemoteVolumes(IEnumerable names, IDbTransaction? transaction = null) + /// + /// Removes multiple remote volumes from the database. + /// + /// An enumerable collection of names of remote volumes to remove. + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the remote volumes have been removed. + public async Task RemoveRemoteVolumes(IEnumerable names, CancellationToken token) { if (names == null || !names.Any()) return; - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - using (var deletecmd = m_connection.CreateCommand(tr.Parent)) + await using var deletecmd = m_connection.CreateCommand(m_rtr); + string temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + var volidstable = "DelVolSetIds-" + temptransguid; + var blocksetidstable = "DelBlockSetIds-" + temptransguid; + var filesetidstable = "DelFilesetIds-" + temptransguid; + + // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. + await deletecmd.ExecuteNonQueryAsync($@" + CREATE TEMP TABLE ""{volidstable}"" ( + ""ID"" INTEGER PRIMARY KEY + ) + ", token) + .ConfigureAwait(false); + + await deletecmd.SetCommandAndParameters($@" + INSERT OR IGNORE INTO ""{volidstable}"" + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" IN (@VolumeNames) + ") + .ExpandInClauseParameterMssqlite("@VolumeNames", [.. names]) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + var volIdsSubQuery = $@" + SELECT ""ID"" + FROM ""{volidstable}"" + "; + + var bsIdsSubQuery = @$" + SELECT DISTINCT ""BlocksetEntry"".""BlocksetID"" + FROM + ""BlocksetEntry"", + ""Block"" + WHERE + ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + AND ""Block"".""VolumeID"" IN ({volIdsSubQuery}) + UNION ALL + SELECT DISTINCT ""BlocksetID"" + FROM ""BlocklistHash"" + WHERE ""Hash"" IN ( + SELECT ""Hash"" + FROM ""Block"" + WHERE ""VolumeID"" IN ({volIdsSubQuery}) + ) + "; + + // Create a temporary table to cache subquery result, as it might take long (SQLite does not cache at all). + await deletecmd.ExecuteNonQueryAsync($@" + CREATE TEMP TABLE ""{blocksetidstable}"" ( + ""ID"" INTEGER PRIMARY KEY + ) + ", token) + .ConfigureAwait(false); + + await deletecmd.ExecuteNonQueryAsync($@" + INSERT OR IGNORE INTO ""{blocksetidstable}"" (""ID"") + {bsIdsSubQuery} + ", token) + .ConfigureAwait(false); + + bsIdsSubQuery = $@" + SELECT DISTINCT ""ID"" + FROM ""{blocksetidstable}"" + "; + + // Create a temp table to associate metadata that is being deleted to a fileset + var metadataFilesetQuery = $@" + SELECT + ""Metadataset"".""ID"", + ""FilesetEntry"".""FilesetID"" + FROM ""Metadataset"" + INNER JOIN ""FileLookup"" + ON ""FileLookup"".""MetadataID"" = ""Metadataset"".""ID"" + INNER JOIN ""FilesetEntry"" + ON ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" + WHERE ""Metadataset"".""BlocksetID"" IN ({bsIdsSubQuery}) + OR ""Metadataset"".""ID"" IN ( + SELECT ""MetadataID"" + FROM ""FileLookup"" + WHERE ""BlocksetID"" IN ({bsIdsSubQuery}) + ) + "; + + var metadataFilesetTable = $"DelMetadataFilesetIds-{temptransguid}"; + await deletecmd.ExecuteNonQueryAsync($@" + CREATE TEMP TABLE ""{metadataFilesetTable}"" ( + ""MetadataID"" INTEGER PRIMARY KEY, + ""FilesetID"" INTEGER + ) + ", token) + .ConfigureAwait(false); + + await deletecmd.ExecuteNonQueryAsync($@" + INSERT OR IGNORE INTO ""{metadataFilesetTable}"" ( + ""MetadataID"", + ""FilesetID"" + ) + {metadataFilesetQuery} + ", token) + .ConfigureAwait(false); + + // Delete FilesetEntry rows that had their metadata deleted + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""FilesetEntry"" + WHERE + ""FilesetEntry"".""FilesetID"" IN ( + SELECT DISTINCT ""FilesetID"" + FROM ""{metadataFilesetTable}"" + ) + AND ""FilesetEntry"".""FileID"" IN ( + SELECT ""FilesetEntry"".""FileID"" + FROM ""FilesetEntry"" + INNER JOIN ""FileLookup"" + ON ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" + WHERE ""FileLookup"".""MetadataID"" IN ( + SELECT ""MetadataID"" + FROM ""{metadataFilesetTable}"" + ) + ) + ", token) + .ConfigureAwait(false); + + // Delete FilesetEntry rows that had their blocks deleted + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""FilesetEntry"" + WHERE ""FilesetEntry"".""FileID"" IN ( + SELECT ""ID"" + FROM ""FileLookup"" + WHERE ""FileLookup"".""BlocksetID"" IN ({bsIdsSubQuery}) + ) + ", token) + .ConfigureAwait(false); + + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""FileLookup"" + WHERE ""FileLookup"".""MetadataID"" IN ( + SELECT ""MetadataID"" + FROM ""{metadataFilesetTable}"" + ) + ", token) + .ConfigureAwait(false); + + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""Metadataset"" + WHERE ""BlocksetID"" IN ({bsIdsSubQuery}) + ", token) + .ConfigureAwait(false); + + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""FileLookup"" + WHERE ""BlocksetID"" IN ({bsIdsSubQuery}) + ", token) + .ConfigureAwait(false); + + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""Blockset"" + WHERE ""ID"" IN ({bsIdsSubQuery}) + ", token) + .ConfigureAwait(false); + + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""BlocksetEntry"" + WHERE ""BlocksetID"" IN ({bsIdsSubQuery}) + ", token) + .ConfigureAwait(false); + + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""BlocklistHash"" + WHERE ""BlocklistHash"".""BlocksetID"" IN ({bsIdsSubQuery}) + ", token) + .ConfigureAwait(false); + + // If the volume is a block or index volume, this will update the crosslink table, otherwise nothing will happen + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""IndexBlockLink"" + WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) + OR ""IndexVolumeID"" IN ({volIdsSubQuery}) + ", token) + .ConfigureAwait(false); + + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""Block"" + WHERE ""VolumeID"" IN ({volIdsSubQuery}) + ", token) + .ConfigureAwait(false); + + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""DeletedBlock"" + WHERE ""VolumeID"" IN ({volIdsSubQuery}) + ", token) + .ConfigureAwait(false); + + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""ChangeJournalData"" + WHERE ""FilesetID"" IN ( + SELECT ""ID"" + FROM ""Fileset"" + WHERE ""VolumeID"" IN ({volIdsSubQuery}) + ) + ", token) + .ConfigureAwait(false); + + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""FilesetEntry"" + WHERE ""FilesetID"" IN ( + SELECT ""ID"" + FROM ""Fileset"" + WHERE ""VolumeID"" IN ({volIdsSubQuery}) + ) + ", token) + .ConfigureAwait(false); + + await deletecmd.ExecuteNonQueryAsync($@" + CREATE TABLE ""{filesetidstable}"" ( + ""ID"" INTEGER PRIMARY KEY + ) + ", token) + .ConfigureAwait(false); + + await deletecmd.ExecuteNonQueryAsync($@" + INSERT OR IGNORE INTO ""{filesetidstable}"" + SELECT ""ID"" + FROM ""Fileset"" + WHERE ""VolumeID"" IN ({volIdsSubQuery}) + ", token) + .ConfigureAwait(false); + + // Delete from Fileset if FilesetEntry rows were deleted by related metadata and there are no references in FilesetEntry anymore + await deletecmd.ExecuteNonQueryAsync($@" + INSERT OR IGNORE INTO ""{filesetidstable}"" + SELECT ""ID"" + FROM ""Fileset"" + WHERE + ""Fileset"".""ID"" IN ( + SELECT DISTINCT ""FilesetID"" + FROM ""{metadataFilesetTable}"" + ) + AND ""Fileset"".""ID"" NOT IN ( + SELECT DISTINCT ""FilesetID"" + FROM ""FilesetEntry"" + ) + ", token) + .ConfigureAwait(false); + + // Since we are deleting the fileset, we also need to mark the remote volume as deleting so it will be cleaned up later + await deletecmd.SetCommandAndParameters($@" + UPDATE ""RemoteVolume"" + SET ""State"" = @NewState + WHERE + ""ID"" IN ( + SELECT DISTINCT ""VolumeID"" + FROM ""Fileset"" + WHERE ""Fileset"".""ID"" IN ( + SELECT ""ID"" + FROM ""{filesetidstable}"" + ) + ) + AND ""State"" IN (@AllowedStates) + ") + .SetParameterValue("@NewState", RemoteVolumeState.Deleting.ToString()) + .ExpandInClauseParameterMssqlite("@AllowedStates", [ + RemoteVolumeState.Uploading.ToString(), + RemoteVolumeState.Uploaded.ToString(), + RemoteVolumeState.Verified.ToString(), + RemoteVolumeState.Temporary.ToString() + ]) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""Fileset"" + WHERE ""ID"" IN ( + SELECT ""ID"" + FROM ""{filesetidstable}"" + ) + ", token) + .ConfigureAwait(false); + + // Clean up temp tables for subqueries. We truncate content and then try to delete. + // Drop in try-block, as it fails in nested transactions (SQLite problem) + // SQLite.SQLiteException (0x80004005): database table is locked + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{blocksetidstable}"" ", token) + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{volidstable}"" ", token) + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{metadataFilesetTable}"" ", token) + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@"DELETE FROM ""{filesetidstable}"" ", token) + .ConfigureAwait(false); + try { - string temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var volidstable = "DelVolSetIds-" + temptransguid; - var blocksetidstable = "DelBlockSetIds-" + temptransguid; - var filesetidstable = "DelFilesetIds-" + temptransguid; - - // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. - deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{volidstable}"" (""ID"" INTEGER PRIMARY KEY)")); - deletecmd.SetCommandAndParameters(FormatInvariant($@"INSERT OR IGNORE INTO ""{volidstable}"" SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@VolumeNames)")) - .ExpandInClauseParameter("@VolumeNames", names.ToArray()) - .ExecuteNonQuery(); - - var volIdsSubQuery = FormatInvariant($@"SELECT ""ID"" FROM ""{volidstable}"" "); - deletecmd.Parameters.Clear(); - - var bsIdsSubQuery = FormatInvariant(@$" -SELECT DISTINCT ""BlocksetEntry"".""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" -WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" IN ({volIdsSubQuery}) -UNION ALL -SELECT DISTINCT ""BlocksetID"" FROM ""BlocklistHash"" -WHERE ""Hash"" IN (SELECT ""Hash"" FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery}))"); - - // Create a temporary table to cache subquery result, as it might take long (SQLite does not cache at all). - deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{blocksetidstable}"" (""ID"" INTEGER PRIMARY KEY)")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"INSERT OR IGNORE INTO ""{blocksetidstable}"" (""ID"") {bsIdsSubQuery}")); - bsIdsSubQuery = FormatInvariant($@"SELECT DISTINCT ""ID"" FROM ""{blocksetidstable}"" "); - deletecmd.Parameters.Clear(); - - // Create a temp table to associate metadata that is being deleted to a fileset - var metadataFilesetQuery = FormatInvariant($@"SELECT Metadataset.ID, FilesetEntry.FilesetID -FROM Metadataset -INNER JOIN FileLookup ON FileLookup.MetadataID = Metadataset.ID -INNER JOIN FilesetEntry ON FilesetEntry.FileID = FileLookup.ID -WHERE Metadataset.BlocksetID IN ({bsIdsSubQuery}) -OR Metadataset.ID IN (SELECT MetadataID FROM FileLookup WHERE BlocksetID IN ({bsIdsSubQuery}))"); - - var metadataFilesetTable = @"DelMetadataFilesetIds-" + temptransguid; - deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{metadataFilesetTable}"" (MetadataID INTEGER PRIMARY KEY, FilesetID INTEGER)")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"INSERT OR IGNORE INTO ""{metadataFilesetTable}"" (MetadataID, FilesetID) {metadataFilesetQuery}")); - - // Delete FilesetEntry rows that had their metadata deleted - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM FilesetEntry -WHERE FilesetEntry.FilesetID IN (SELECT DISTINCT FilesetID FROM ""{metadataFilesetTable}"") -AND FilesetEntry.FileID IN ( - SELECT FilesetEntry.FileID - FROM FilesetEntry - INNER JOIN FileLookup ON FileLookup.ID = FilesetEntry.FileID - WHERE FileLookup.MetadataID IN (SELECT MetadataID FROM ""{metadataFilesetTable}""))")); - - // Delete FilesetEntry rows that had their blocks deleted - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM FilesetEntry WHERE FilesetEntry.FileID IN ( -SELECT ID FROM FileLookup -WHERE FileLookup.BlocksetID IN ({bsIdsSubQuery}))")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM FileLookup WHERE FileLookup.MetadataID IN (SELECT MetadataID FROM ""{metadataFilesetTable}"")")); - - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""FileLookup"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Blockset"" WHERE ""ID"" IN ({bsIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ({bsIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""BlocklistHash"" WHERE ""BlocklistHash"".""BlocksetID"" IN ({bsIdsSubQuery})")); - - // If the volume is a block or index volume, this will update the crosslink table, otherwise nothing will happen - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""ChangeJournalData"" WHERE ""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ({volIdsSubQuery}))")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM FilesetEntry WHERE FilesetID IN (SELECT ID FROM Fileset WHERE VolumeID IN ({volIdsSubQuery}))")); - - deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TABLE ""{filesetidstable}"" (""ID"" INTEGER PRIMARY KEY)")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"INSERT OR IGNORE INTO ""{filesetidstable}"" SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); - // Delete from Fileset if FilesetEntry rows were deleted by related metadata and there are no references in FilesetEntry anymore - deletecmd.ExecuteNonQuery(FormatInvariant($@"INSERT OR IGNORE INTO ""{filesetidstable}"" SELECT ""ID"" FROM ""Fileset"" WHERE ""Fileset"".""ID"" IN -(SELECT DISTINCT ""FilesetID"" FROM ""{metadataFilesetTable}"") -AND ""Fileset"".""ID"" NOT IN - (SELECT DISTINCT ""FilesetID"" FROM FilesetEntry)")); - - // Since we are deleting the fileset, we also need to mark the remote volume as deleting so it will be cleaned up later - deletecmd.SetCommandAndParameters(FormatInvariant($@"UPDATE ""RemoteVolume"" SET ""State"" = @NewState WHERE ""ID"" IN (SELECT DISTINCT ""VolumeID"" FROM ""Fileset"" WHERE ""Fileset"".""ID"" IN (SELECT ""ID"" FROM ""{filesetidstable}"")) AND ""State"" IN (@AllowedStates)")) - .SetParameterValue("@NewState", RemoteVolumeState.Deleting.ToString()) - .ExpandInClauseParameter("@AllowedStates", [RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString(), RemoteVolumeState.Temporary.ToString()]) - .ExecuteNonQuery(); - - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Fileset"" WHERE ""ID"" IN (SELECT ""ID"" FROM ""{filesetidstable}"")")); - - - // Clean up temp tables for subqueries. We truncate content and then try to delete. - // Drop in try-block, as it fails in nested transactions (SQLite problem) - // SQLite.SQLiteException (0x80004005): database table is locked - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{blocksetidstable}"" ")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{volidstable}"" ")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{metadataFilesetTable}"" ")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{filesetidstable}"" ")); - try - { - deletecmd.CommandTimeout = 2; - deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{blocksetidstable}"" ")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{volidstable}"" ")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{metadataFilesetTable}"" ")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{filesetidstable}"" ")); - } - catch { /* Ignore, will be deleted on close anyway. */ } - - m_removeremotevolumeCommand.Transaction = tr.Parent; - m_removeremotevolumeCommand.SetParameterValue("@Now", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow)); - m_removeremotevolumeCommand.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); - foreach (var name in names) - { - m_removeremotevolumeCommand.SetParameterValue("@Name", name); - m_removeremotevolumeCommand.ExecuteNonQuery(); - } - - // Validate before commiting changes - var nonAttachedFiles = deletecmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FileID"" NOT IN (SELECT ""ID"" FROM ""FileLookup"")"); - if (nonAttachedFiles > 0) - throw new ConstraintException($"Detected {nonAttachedFiles} file(s) in FilesetEntry without corresponding FileLookup entry"); - - tr.Commit(); + deletecmd.CommandTimeout = 2; + await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{blocksetidstable}"" ", token) + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{volidstable}"" ", token) + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{metadataFilesetTable}"" ", token) + .ConfigureAwait(false); + await deletecmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filesetidstable}"" ", token) + .ConfigureAwait(false); } + catch { /* Ignore, will be deleted on close anyway. */ } + + m_removeremotevolumeCommand + .SetTransaction(m_rtr) + .SetParameterValue("@Now", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(DateTime.UtcNow)) + .SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); + foreach (var name in names) + { + await m_removeremotevolumeCommand + .SetTransaction(m_rtr) + .SetParameterValue("@Name", name) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } + + // Validate before commiting changes + var nonAttachedFiles = await deletecmd.ExecuteScalarInt64Async(@" + SELECT COUNT(*) + FROM ""FilesetEntry"" + WHERE ""FileID"" NOT IN ( + SELECT ""ID"" + FROM ""FileLookup"" + ) + ", token) + .ConfigureAwait(false); + + if (nonAttachedFiles > 0) + throw new ConstraintException($"Detected {nonAttachedFiles} file(s) in FilesetEntry without corresponding FileLookup entry"); } - public void Vacuum() + /// + /// Performs a VACUUM operation on the database to reclaim unused space. + /// This operation can help optimize the database performance by defragmenting it. + /// Note: This operation can take a significant amount of time depending on the size of the database. + /// + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the VACUUM operation has finished. + public async Task Vacuum(CancellationToken token) { m_hasExecutedVacuum = true; - using (var cmd = m_connection.CreateCommand()) - cmd.ExecuteNonQuery("VACUUM"); + await using var cmd = m_connection.CreateCommand(); + + await cmd.ExecuteNonQueryAsync("VACUUM", token) + .ConfigureAwait(false); } - public long RegisterRemoteVolume(string name, RemoteVolumeType type, long size, RemoteVolumeState state) + /// + /// Registers a new remote volume in the database. + /// + /// The name of the remote volume. + /// The type of the remote volume. + /// The size of the remote volume in bytes. Use -1 for unknown size. + /// The state of the remote volume. + /// Cancellation token to monitor for cancellation requests. + /// A task that, when awaited, returns the ID of the newly registered remote volume. + public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, long size, RemoteVolumeState state, CancellationToken token) { - return RegisterRemoteVolume(name, type, state, size, new TimeSpan(0), null); + return await RegisterRemoteVolume(name, type, state, size, new TimeSpan(0), token) + .ConfigureAwait(false); } - public long RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, IDbTransaction? transaction) + /// + /// Registers a new remote volume in the database. + /// + /// The name of the remote volume. + /// The type of the remote volume. + /// The state of the remote volume. + /// Cancellation token to monitor for cancellation requests. + /// A task that, when awaited, returns the ID of the newly registered remote volume. + public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, CancellationToken token) { - return RegisterRemoteVolume(name, type, state, new TimeSpan(0), transaction); + return await RegisterRemoteVolume(name, type, state, new TimeSpan(0), token) + .ConfigureAwait(false); } - public long RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, TimeSpan deleteGraceTime, IDbTransaction? transaction) + /// + /// Registers a new remote volume in the database. + /// + /// The name of the remote volume. + /// The type of the remote volume. + /// The state of the remote volume. + /// The time after which the remote volume can be deleted. + /// Cancellation token to monitor for cancellation requests. + /// A task that, when awaited, returns the ID of the newly registered remote volume. + public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, TimeSpan deleteGraceTime, CancellationToken token) { - return RegisterRemoteVolume(name, type, state, -1, deleteGraceTime, transaction); + return await RegisterRemoteVolume(name, type, state, -1, deleteGraceTime, token) + .ConfigureAwait(false); } - public long RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, long size, TimeSpan deleteGraceTime, IDbTransaction? transaction) + /// + /// Registers a new remote volume in the database. + /// + /// The name of the remote volume. + /// The type of the remote volume. + /// The state of the remote volume. + /// The size of the remote volume in bytes. Use -1 for unknown size. + /// The time after which the remote volume can be deleted. + /// Cancellation token to monitor for cancellation requests. + /// A task that, when awaited, returns the ID of the newly registered remote volume. + public async Task RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, long size, TimeSpan deleteGraceTime, CancellationToken token) { - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - { - var r = m_createremotevolumeCommand.SetParameterValue("@OperationId", m_operationid) - .SetParameterValue("@Name", name) - .SetParameterValue("@Type", type.ToString()) - .SetParameterValue("@State", state.ToString()) - .SetParameterValue("@Size", size) - .SetParameterValue("@VerificationCount", 0) - .SetParameterValue("@DeleteGraceTime", deleteGraceTime.Ticks <= 0 ? 0 : (DateTime.UtcNow + deleteGraceTime).Ticks) - .SetParameterValue("@ArchiveTime", 0) - .ExecuteScalarInt64(tr.Parent); + var r = await m_createremotevolumeCommand + .SetTransaction(m_rtr) + .SetParameterValue("@OperationID", m_operationid) + .SetParameterValue("@Name", name) + .SetParameterValue("@Type", type.ToString()) + .SetParameterValue("@State", state.ToString()) + .SetParameterValue("@Size", size) + .SetParameterValue("@VerificationCount", 0) + .SetParameterValue("@DeleteGraceTime", deleteGraceTime.Ticks <= 0 ? 0 : (DateTime.UtcNow + deleteGraceTime).Ticks) + .SetParameterValue("@ArchiveTime", 0) + .ExecuteScalarInt64Async(token) + .ConfigureAwait(false); - tr.Commit(); - return r; - } + return r; } - public IEnumerable GetFilesetIDs(DateTime restoretime, long[]? versions, bool singleTimeMatch = false) + /// + /// Retrieves the IDs of filesets that match a specific restore time and optional versions. + /// If no filesets match the criteria, it returns the newest fileset ID. + /// + /// The time to restore from. + /// Optional array of versions to match against the filesets. + /// If true, only match filesets that exactly match the restore time. + /// Cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of fileset IDs that match the criteria. + /// Thrown if the provided DateTime is unspecified. + /// Thrown if no backups are found at the specified date. + public async IAsyncEnumerable GetFilesetIDs(DateTime restoretime, long[]? versions, bool singleTimeMatch, [EnumeratorCancellation] CancellationToken token) { if (restoretime.Kind == DateTimeKind.Unspecified) throw new Exception("Invalid DateTime given, must be either local or UTC"); - (var query, var values) = GetFilelistWhereClause(restoretime, versions, singleTimeMatch: singleTimeMatch); + (var wherequery, var values) = + await GetFilelistWhereClause(restoretime, versions, null, singleTimeMatch, token) + .ConfigureAwait(false); var res = new List(); - using (var cmd = m_connection.CreateCommand()) + await using var cmd = m_connection.CreateCommand(); + cmd.SetCommandAndParameters($@" + SELECT ""ID"" + FROM ""Fileset"" + {wherequery} + ORDER BY ""Timestamp"" DESC + ") + .SetParameterValues(values); + + await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) + res.Add(rd.ConvertValueToInt64(0)); + + if (res.Count == 0) { - using (var rd = cmd.ExecuteReader($@"SELECT ""ID"" FROM ""Fileset"" {query} ORDER BY ""Timestamp"" DESC", values)) - while (rd.Read()) + cmd.SetCommandAndParameters(@" + SELECT ""ID"" + FROM ""Fileset"" + ORDER BY ""Timestamp"" DESC + "); + await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) res.Add(rd.ConvertValueToInt64(0)); if (res.Count == 0) - { - cmd.Parameters.Clear(); - using (var rd = cmd.ExecuteReader(@"SELECT ""ID"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC ")) - while (rd.Read()) - res.Add(rd.ConvertValueToInt64(0)); - - if (res.Count == 0) - throw new Duplicati.Library.Interface.UserInformationException("No backup at the specified date", "NoBackupAtDate"); - else - Logging.Log.WriteWarningMessage(LOGTAG, "RestoreTimeNoMatch", null, "Restore time or version did not match any existing backups, selecting newest backup"); - } - - return res; + throw new Duplicati.Library.Interface.UserInformationException("No backup at the specified date", "NoBackupAtDate"); + else + Logging.Log.WriteWarningMessage(LOGTAG, "RestoreTimeNoMatch", null, "Restore time or version did not match any existing backups, selecting newest backup"); } + + foreach (var el in res) + yield return el; } - public IEnumerable FindMatchingFilesets(DateTime restoretime, long[]? versions) + /// + /// Finds filesets that match a specific restore time and optional versions. + /// + /// The time to restore from. + /// Optional array of versions to match against the filesets. + /// Cancellation token to monitor for cancellation requests. + /// An asynchronous task that returns a collection of fileset IDs that match the criteria. + public async Task> FindMatchingFilesets(DateTime restoretime, long[]? versions, CancellationToken token) { if (restoretime.Kind == DateTimeKind.Unspecified) throw new Exception("Invalid DateTime given, must be either local or UTC"); - var tmp = GetFilelistWhereClause(restoretime, versions, singleTimeMatch: true); - string query = tmp.Query; - var args = tmp.Values; + var (wherequery, args) = + await GetFilelistWhereClause(restoretime, versions, null, true, token) + .ConfigureAwait(false); var res = new List(); - using (var cmd = m_connection.CreateCommand()) - using (var rd = cmd.ExecuteReader(@"SELECT ""ID"" FROM ""Fileset"" " + query + @" ORDER BY ""Timestamp"" DESC", args)) - while (rd.Read()) + await using var cmd = m_connection.CreateCommand(); + cmd.SetCommandAndParameters(@$" + SELECT ""ID"" + FROM ""Fileset"" + {wherequery} + ORDER BY ""Timestamp"" DESC + ") + .SetParameterValues(args); + await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) res.Add(rd.ConvertValueToInt64(0)); return res; } - public bool IsFilesetFullBackup(DateTime filesetTime, IDbTransaction? transaction) + /// + /// Checks if a fileset is a full backup. + /// A fileset is considered a full backup if its "IsFullBackup" field is set to FULL_BACKUP. + /// + /// The timestamp of the fileset to check. + /// Cancellation token to monitor for cancellation requests. + /// A task that, when awaited, returns true if the fileset is a full backup, otherwise false. + public async Task IsFilesetFullBackup(DateTime filesetTime, CancellationToken token) { - using (var cmd = m_connection.CreateCommand(transaction)) - using (var rd = cmd.SetCommandAndParameters($@"SELECT ""IsFullBackup"" FROM ""Fileset"" WHERE ""Timestamp"" = @Timestamp").SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(filesetTime)).ExecuteReader()) - { - if (!rd.Read()) - return false; - var isFullBackup = rd.GetInt32(0); - return isFullBackup == BackupType.FULL_BACKUP; - } - } + await using var cmd = m_connection.CreateCommand(); + cmd.SetCommandAndParameters($@" + SELECT ""IsFullBackup"" + FROM ""Fileset"" + WHERE ""Timestamp"" = @Timestamp + ") + .SetTransaction(m_rtr) + .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(filesetTime)); - // TODO: Remove this - public IDbTransaction BeginTransaction() - { - return m_connection.BeginTransactionSafe(); - } - - protected class TemporaryTransactionWrapper : IDisposable - { - private readonly IDbTransaction m_parent; - private readonly bool m_isTemporary; - - public TemporaryTransactionWrapper(IDbConnection connection, IDbTransaction? transaction) - { - if (transaction != null) - { - m_parent = transaction; - m_isTemporary = false; - } - else - { - m_parent = connection.BeginTransactionSafe(); - m_isTemporary = true; - } - } - - public void Commit() - { - if (m_isTemporary) - m_parent.Commit(); - } - - public void Dispose() - { - if (m_isTemporary) - m_parent.Dispose(); - } - - public IDbTransaction Parent { get { return m_parent; } } - } - - private IEnumerable> GetDbOptionList(IDbTransaction? transaction = null) - { - using (var cmd = m_connection.CreateCommand(transaction)) - using (var rd = cmd.ExecuteReader(@"SELECT ""Key"", ""Value"" FROM ""Configuration"" ")) - while (rd.Read()) - yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? ""); - } - - public IDictionary GetDbOptions(IDbTransaction? transaction = null) - { - return GetDbOptionList(transaction).ToDictionary(x => x.Key, x => x.Value); + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + if (!await rd.ReadAsync(token).ConfigureAwait(false)) + return false; + var isFullBackup = rd.GetInt32(0); + return isFullBackup == BackupType.FULL_BACKUP; } /// - /// Updates a database option + /// Retrieves a list of database options from the "Configuration" table. /// - /// The key to update - /// The value to set - private void UpdateDbOption(string key, bool value) + /// Cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of key-value pairs representing the database options. + private async IAsyncEnumerable> GetDbOptionList([EnumeratorCancellation] CancellationToken token) { - var opts = GetDbOptions(); + await using var cmd = m_connection.CreateCommand(@" + SELECT + ""Key"", + ""Value"" + FROM ""Configuration"" + ") + .SetTransaction(m_rtr); + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + yield return new KeyValuePair( + rd.ConvertValueToString(0) ?? "", + rd.ConvertValueToString(1) ?? "" + ); + } + + /// + /// Retrieves all database options as a dictionary. + /// + /// Cancellation token to monitor for cancellation requests. + /// A task that, when awaited, returns a dictionary containing all database options. + public async Task> GetDbOptions(CancellationToken token) + { + var res = await GetDbOptionList(token) + .ToDictionaryAsync(x => x.Key, x => x.Value, cancellationToken: token) + .ConfigureAwait(false); + + return res; + } + + /// + /// Updates a database option. + /// + /// The key to update. + /// The value to set. + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the option has been updated. + private async Task UpdateDbOption(string key, bool value, CancellationToken token) + { + var opts = await GetDbOptions(token).ConfigureAwait(false); if (value) opts[key] = "true"; else opts.Remove(key); - SetDbOptions(opts); + await SetDbOptions(opts, token).ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } /// - /// Flag indicating if a repair is in progress + /// Flag indicating if a repair is in progress. /// - public bool RepairInProgress + /// Cancellation token to monitor for cancellation requests. + /// Optional value to set the flag to. If null, the current value in the database is returned. + /// A task that, when awaited, returns true if a repair is in progress, otherwise false. + public async Task RepairInProgress(CancellationToken token, bool? value = null) { - get => GetDbOptions().ContainsKey("repair-in-progress"); - set => UpdateDbOption("repair-in-progress", value); - } - - /// - /// Flag indicating if a repair is in progress - /// - public bool PartiallyRecreated - { - get => GetDbOptions().ContainsKey("partially-recreated"); - set => UpdateDbOption("partially-recreated", value); - } - - /// - /// Flag indicating if the database can contain partial uploads - /// - public bool TerminatedWithActiveUploads - { - get => GetDbOptions().ContainsKey("terminated-with-active-uploads"); - set => UpdateDbOption("terminated-with-active-uploads", value); - } - - /// - /// Sets the database options - /// - /// The options to set - /// An optional transaction - public void SetDbOptions(IDictionary options, IDbTransaction? transaction = null) - { - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - using (var cmd = m_connection.CreateCommand(tr.Parent)) + if (value is bool v) { - cmd.ExecuteNonQuery(@"DELETE FROM ""Configuration"" "); - foreach (var kp in options) - cmd.SetCommandAndParameters(@"INSERT INTO ""Configuration"" (""Key"", ""Value"") VALUES (@Key, @Value) ") - .SetParameterValue("@Key", kp.Key) - .SetParameterValue("@Value", kp.Value) - .ExecuteNonQuery(); + await UpdateDbOption("repair-in-progress", v, token) + .ConfigureAwait(false); - tr.Commit(); + return v; + } + + var opts = await GetDbOptions(token).ConfigureAwait(false); + return opts.ContainsKey("repair-in-progress"); + } + + /// + /// Flag indicating if the database has been partially recreated. + /// + /// Cancellation token to monitor for cancellation requests. + /// Optional value to set the flag to. If null, the current value in the database is returned. + /// A task that, when awaited, returns true if the database has been partially recreated, otherwise false. + public async Task PartiallyRecreated(CancellationToken token, bool? value = null) + { + if (value is bool v) + { + await UpdateDbOption("partially-recreated", v, token) + .ConfigureAwait(false); + + return v; + } + + var opts = await GetDbOptions(token).ConfigureAwait(false); + return opts.ContainsKey("partially-recreated"); + } + + /// + /// Flag indicating if the database has been terminated with active uploads. + /// + /// Cancellation token to monitor for cancellation requests. + /// Optional value to set the flag to. If null, the current value in the database is returned. + /// A task that, when awaited, returns true if the database has been terminated with active uploads, otherwise false. + public async Task TerminatedWithActiveUploads(CancellationToken token, bool? value = null) + { + if (value is bool v) + { + await UpdateDbOption("terminated-with-active-uploads", v, token) + .ConfigureAwait(false); + + return v; + } + + var opts = await GetDbOptions(token).ConfigureAwait(false); + return opts.ContainsKey("terminated-with-active-uploads"); + } + + /// + /// Sets the database options. + /// + /// The options to set. + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the options have been set. + public async Task SetDbOptions(IDictionary options, CancellationToken token) + { + await using var cmd = m_connection.CreateCommand(); + await cmd.ExecuteNonQueryAsync(@" + DELETE FROM ""Configuration"" + ", token) + .ConfigureAwait(false); + + foreach (var kp in options) + { + await cmd.SetCommandAndParameters(@" + INSERT INTO ""Configuration"" ( + ""Key"", + ""Value"" + ) + VALUES ( + @Key, + @Value + ) + ") + .SetParameterValue("@Key", kp.Key) + .SetParameterValue("@Value", kp.Value) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); } } - public long GetBlocksLargerThan(long fhblocksize) + /// + /// Counts the number of blocks larger than a specified size. + /// + /// The size in bytes to compare against. + /// Cancellation token to monitor for cancellation requests. + /// A task that, when awaited, returns the count of blocks larger than the specified size. + public async Task GetBlocksLargerThan(long fhblocksize, CancellationToken token) { - using (var cmd = m_connection.CreateCommand()) - return cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""Block"" WHERE ""Size"" > @Size") - .SetParameterValue("@Size", fhblocksize) - .ExecuteScalarInt64(-1); + await using var cmd = await m_connection.CreateCommandAsync(@" + SELECT COUNT(*) + FROM ""Block"" + WHERE ""Size"" > @Size + ", token) + .ConfigureAwait(false); + + return await cmd + .SetParameterValue("@Size", fhblocksize) + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); } /// - /// Verifies the consistency of the database + /// Verifies the consistency of the database. /// - /// The block size in bytes - /// The hash size in byts - /// Also verify filelists (can be slow) - /// The transaction to run in - public void VerifyConsistency(long blocksize, long hashsize, bool verifyfilelists, IDbTransaction? transaction) - => VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, false, transaction); + /// The block size in bytes. + /// The hash size in bytes. + /// Also verify filelists (can be slow). + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the consistency check is finished. + public async Task VerifyConsistency(long blocksize, long hashsize, bool verifyfilelists, CancellationToken token) + => await VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, false, token) + .ConfigureAwait(false); /// - /// Verifies the consistency of the database prior to repair + /// Verifies the consistency of the database prior to repair. /// - /// The block size in bytes - /// The hash size in byts - /// Also verify filelists (can be slow) - /// The transaction to run in - public void VerifyConsistencyForRepair(long blocksize, long hashsize, bool verifyfilelists, IDbTransaction? transaction) - => VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, true, transaction); + /// The block size in bytes. + /// The hash size in bytes. + /// Also verify filelists (can be slow). + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the consistency check is finished. + public async Task VerifyConsistencyForRepair(long blocksize, long hashsize, bool verifyfilelists, CancellationToken token) + => await VerifyConsistencyInner(blocksize, hashsize, verifyfilelists, true, token) + .ConfigureAwait(false); /// - /// Verifies the consistency of the database + /// Verifies the consistency of the database. /// - /// The block size in bytes - /// The hash size in byts - /// Also verify filelists (can be slow) - /// Disable verify for errors that will be fixed by repair - /// The transaction to run in - private void VerifyConsistencyInner(long blocksize, long hashsize, bool verifyfilelists, bool laxVerifyForRepair, IDbTransaction? transaction) + /// The block size in bytes. + /// The hash size in bytes. + /// Also verify filelists (can be slow). + /// Disable verify for errors that will be fixed by repair. + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the consistency check is finished. + private async Task VerifyConsistencyInner(long blocksize, long hashsize, bool verifyfilelists, bool laxVerifyForRepair, CancellationToken token) { - using (var cmd = m_connection.CreateCommand(transaction)) + await using var cmd = m_connection.CreateCommand() + .SetTransaction(m_rtr); + // Calculate the lengths for each blockset + var combinedLengths = @" + SELECT + ""A"".""ID"" AS ""BlocksetID"", + IFNULL(""B"".""CalcLen"", 0) AS ""CalcLen"", + ""A"".""Length"" + FROM ""Blockset"" ""A"" + LEFT OUTER JOIN ( + SELECT + ""BlocksetEntry"".""BlocksetID"", + SUM(""Block"".""Size"") AS ""CalcLen"" + FROM ""BlocksetEntry"" + LEFT OUTER JOIN ""Block"" + ON ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" + GROUP BY ""BlocksetEntry"".""BlocksetID"" + ) ""B"" + ON ""A"".""ID"" = ""B"".""BlocksetID"" + "; + + // For each blockset with wrong lengths, fetch the file path + var reportDetails = @$" + SELECT + ""CalcLen"", + ""Length"", ""A"".""BlocksetID"", + ""File"".""Path"" + FROM + ({combinedLengths}) ""A"", + ""File"" + WHERE + ""A"".""BlocksetID"" = ""File"".""BlocksetID"" + AND ""A"".""CalcLen"" != ""A"".""Length"" + "; + + await using (var rd = await cmd.ExecuteReaderAsync(reportDetails, token).ConfigureAwait(false)) + if (await rd.ReadAsync(token).ConfigureAwait(false)) + { + var sb = new StringBuilder(); + sb.AppendLine("Found inconsistency in the following files while validating database: "); + var c = 0; + do + { + if (c < 5) + sb.AppendFormat("{0}, actual size {1}, dbsize {2}, blocksetid: {3}{4}", rd.GetValue(3), rd.GetValue(1), rd.GetValue(0), rd.GetValue(2), Environment.NewLine); + c++; + } while (await rd.ReadAsync(token).ConfigureAwait(false)); + + c -= 5; + if (c > 0) + sb.AppendFormat("... and {0} more", c); + + sb.Append(". Run repair to fix it."); + throw new DatabaseInconsistencyException(sb.ToString()); + } + + var real_count = await cmd.ExecuteScalarInt64Async(@" + SELECT COUNT(*) + FROM ""BlocklistHash"" + ", 0, token) + .ConfigureAwait(false); + + var unique_count = await cmd.ExecuteScalarInt64Async(@" + SELECT COUNT(*) + FROM ( + SELECT DISTINCT + ""BlocksetID"", + ""Index"" + FROM ""BlocklistHash"" + ) + ", 0, token) + .ConfigureAwait(false); + + if (real_count != unique_count) + throw new DatabaseInconsistencyException($"Found {real_count} blocklist hashes, but there should be {unique_count}. Run repair to fix it."); + + var itemswithnoblocklisthash = await cmd.ExecuteScalarInt64Async($@" + SELECT COUNT(*) + FROM ( + SELECT * + FROM ( + SELECT + ""N"".""BlocksetID"", + ((""N"".""BlockCount"" + {blocksize / hashsize} - 1) / {blocksize / hashsize}) AS ""BlocklistHashCountExpected"", + CASE + WHEN ""G"".""BlocklistHashCount"" IS NULL + THEN 0 + ELSE ""G"".""BlocklistHashCount"" + END AS ""BlocklistHashCountActual"" + FROM ( + SELECT + ""BlocksetID"", + COUNT(*) AS ""BlockCount"" + FROM ""BlocksetEntry"" + GROUP BY ""BlocksetID"" + ) ""N"" + LEFT OUTER JOIN ( + SELECT + ""BlocksetID"", + COUNT(*) AS ""BlocklistHashCount"" + FROM ""BlocklistHash"" + GROUP BY ""BlocksetID"" + ) ""G"" + ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" + WHERE ""N"".""BlockCount"" > 1 + ) + WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"" + ) + ", 0, token) + .ConfigureAwait(false); + + if (itemswithnoblocklisthash != 0) + throw new DatabaseInconsistencyException($"Found {itemswithnoblocklisthash} file(s) with missing blocklist hashes"); + + cmd.SetCommandAndParameters(@" + SELECT COUNT(*) + FROM ""Blockset"" + WHERE + ""Length"" > 0 + AND ""ID"" NOT IN ( + SELECT ""BlocksetId"" + FROM ""BlocksetEntry"" + ) + "); + + if (await cmd.ExecuteScalarInt64Async(token).ConfigureAwait(false) != 0) + throw new DatabaseInconsistencyException("Detected non-empty blocksets with no associated blocks!"); + + cmd.SetCommandAndParameters(@" + SELECT COUNT(*) + FROM ""FileLookup"" + WHERE + ""BlocksetID"" != @FolderBlocksetId + AND ""BlocksetID"" != @SymlinkBlocksetId + AND NOT ""BlocksetID"" IN ( + SELECT ""ID"" + FROM ""Blockset"" + ) + "); + cmd.SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID); + cmd.SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID); + if (await cmd.ExecuteScalarInt64Async(0, token).ConfigureAwait(false) != 0) + throw new DatabaseInconsistencyException("Detected files associated with non-existing blocksets!"); + + if (!laxVerifyForRepair) { - // Calculate the lengths for each blockset - var combinedLengths = @" -SELECT - ""A"".""ID"" AS ""BlocksetID"", - IFNULL(""B"".""CalcLen"", 0) AS ""CalcLen"", - ""A"".""Length"" -FROM - ""Blockset"" A -LEFT OUTER JOIN - ( - SELECT - ""BlocksetEntry"".""BlocksetID"", - SUM(""Block"".""Size"") AS ""CalcLen"" - FROM - ""BlocksetEntry"" - LEFT OUTER JOIN - ""Block"" - ON - ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" - GROUP BY ""BlocksetEntry"".""BlocksetID"" - ) B -ON - ""A"".""ID"" = ""B"".""BlocksetID"" + cmd.SetCommandAndParameters(@" + SELECT COUNT(*) + FROM ""Fileset"" + WHERE ""VolumeID"" NOT IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE + ""Type"" = @Type + AND ""State"" != @State + ) + "); + cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); + cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); + var filesetsMissingVolumes = + await cmd.ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false); -"; - // For each blockset with wrong lengths, fetch the file path - var reportDetails = @"SELECT ""CalcLen"", ""Length"", ""A"".""BlocksetID"", ""File"".""Path"" FROM (" + combinedLengths + @") A, ""File"" WHERE ""A"".""BlocksetID"" = ""File"".""BlocksetID"" AND ""A"".""CalcLen"" != ""A"".""Length"" "; - - using (var rd = cmd.ExecuteReader(reportDetails)) - if (rd.Read()) - { - var sb = new StringBuilder(); - sb.AppendLine("Found inconsistency in the following files while validating database: "); - var c = 0; - do - { - if (c < 5) - sb.AppendFormat("{0}, actual size {1}, dbsize {2}, blocksetid: {3}{4}", rd.GetValue(3), rd.GetValue(1), rd.GetValue(0), rd.GetValue(2), Environment.NewLine); - c++; - } while (rd.Read()); - - c -= 5; - if (c > 0) - sb.AppendFormat("... and {0} more", c); - - sb.Append(". Run repair to fix it."); - throw new DatabaseInconsistencyException(sb.ToString()); - } - - var real_count = cmd.ExecuteScalarInt64(@"SELECT Count(*) FROM ""BlocklistHash""", 0); - var unique_count = cmd.ExecuteScalarInt64(@"SELECT Count(*) FROM (SELECT DISTINCT ""BlocksetID"", ""Index"" FROM ""BlocklistHash"")", 0); - - if (real_count != unique_count) - throw new DatabaseInconsistencyException($"Found {real_count} blocklist hashes, but there should be {unique_count}. Run repair to fix it."); - - var itemswithnoblocklisthash = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM (SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {blocksize / hashsize} - 1) / {blocksize / hashsize}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"")"), 0); - if (itemswithnoblocklisthash != 0) - throw new DatabaseInconsistencyException($"Found {itemswithnoblocklisthash} file(s) with missing blocklist hashes"); - - if (cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Blockset"" WHERE ""Length"" > 0 AND ""ID"" NOT IN (SELECT ""BlocksetId"" FROM ""BlocksetEntry"")") != 0) - throw new DatabaseInconsistencyException("Detected non-empty blocksets with no associated blocks!"); - - if (cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""FileLookup"" WHERE ""BlocksetID"" != @FolderBlocksetId AND ""BlocksetID"" != @SymlinkBlocksetId AND NOT ""BlocksetID"" IN (SELECT ""ID"" FROM ""Blockset"")") - .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) - .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) - .ExecuteScalarInt64(0) != 0) - throw new DatabaseInconsistencyException("Detected files associated with non-existing blocksets!"); - - if (!laxVerifyForRepair) + if (filesetsMissingVolumes != 0) { - var filesetsMissingVolumes = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State)") - .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()) - .ExecuteScalarInt64(0); - - if (filesetsMissingVolumes != 0) + if (filesetsMissingVolumes == 1) { - if (filesetsMissingVolumes == 1) - using (var reader = cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Timestamp"", ""VolumeID"" FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State)") - .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()) - .ExecuteReader()) - if (reader.Read()) - throw new DatabaseInconsistencyException($"Detected 1 fileset with missing volume: FilesetId = {reader.ConvertValueToInt64(0)}, Time = ({ParseFromEpochSeconds(reader.ConvertValueToInt64(1))}), unmatched VolumeID {reader.ConvertValueToInt64(2)}"); - - throw new DatabaseInconsistencyException($"Detected {filesetsMissingVolumes} filesets with missing volumes"); + cmd.SetCommandAndParameters(@" + SELECT + ""ID"", + ""Timestamp"", + ""VolumeID"" + FROM ""Fileset"" + WHERE ""VolumeID"" NOT IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE + ""Type"" = @Type + AND ""State"" != @State + ) + "); + cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); + cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); + await using var reader = await cmd.ExecuteReaderAsync(token) + .ConfigureAwait(false); + if (await reader.ReadAsync(token).ConfigureAwait(false)) + throw new DatabaseInconsistencyException($"Detected 1 fileset with missing volume: FilesetId = {reader.ConvertValueToInt64(0)}, Time = ({ParseFromEpochSeconds(reader.ConvertValueToInt64(1))}), unmatched VolumeID {reader.ConvertValueToInt64(2)}"); } - var volumesMissingFilests = cmd.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")") - .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()) - .ExecuteScalarInt64(0); - if (volumesMissingFilests != 0) - { - if (volumesMissingFilests == 1) - using (var reader = cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Name"", ""State"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" != @State AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")") - .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()) - .ExecuteReader()) - if (reader.Read()) - throw new DatabaseInconsistencyException($"Detected 1 volume with missing filesets: VolumeId = {reader.ConvertValueToInt64(0)}, Name = {reader.ConvertValueToString(1)}, State = {reader.ConvertValueToString(2)}"); + throw new DatabaseInconsistencyException($"Detected {filesetsMissingVolumes} filesets with missing volumes"); + } - throw new DatabaseInconsistencyException($"Detected {volumesMissingFilests} volumes with missing filesets"); + cmd.SetCommandAndParameters(@" + SELECT COUNT(*) + FROM ""RemoteVolume"" + WHERE + ""Type"" = @Type + AND ""State"" != @State + AND ""ID"" NOT IN ( + SELECT ""VolumeID"" + FROM ""Fileset"" + ) + "); + cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); + cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); + var volumesMissingFilests = await cmd.ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false); + + if (volumesMissingFilests != 0) + { + if (volumesMissingFilests == 1) + { + cmd.SetCommandAndParameters(@" + SELECT + ""ID"", + ""Name"", + ""State"" + FROM ""RemoteVolume"" + WHERE + ""Type"" = @Type + AND ""State"" != @State + AND ""ID"" NOT IN ( + SELECT ""VolumeID"" + FROM ""Fileset"" + ) + "); + cmd.SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); + cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()); + await using var reader = await cmd.ExecuteReaderAsync(token) + .ConfigureAwait(false); + if (await reader.ReadAsync(token).ConfigureAwait(false)) + throw new DatabaseInconsistencyException($"Detected 1 volume with missing filesets: VolumeId = {reader.ConvertValueToInt64(0)}, Name = {reader.ConvertValueToString(1)}, State = {reader.ConvertValueToString(2)}"); + } + + throw new DatabaseInconsistencyException($"Detected {volumesMissingFilests} volumes with missing filesets"); + } + } + + var nonAttachedFiles = await cmd.ExecuteScalarInt64Async(@" + SELECT COUNT(*) + FROM ""FilesetEntry"" + WHERE ""FileID"" NOT IN ( + SELECT ""ID"" + FROM ""FileLookup"" + ) + ", token) + .ConfigureAwait(false); + + if (nonAttachedFiles != 0) + { + // Attempt to create a better error message by finding the first 10 fileset ids with the issue + await using var filesetIdReader = await cmd.ExecuteReaderAsync(@" + SELECT DISTINCT(FilesetID) + FROM ""FilesetEntry"" + WHERE ""FileID"" NOT IN ( + SELECT ""ID"" + FROM ""FileLookup"" + ) + LIMIT 11 + ", token) + .ConfigureAwait(false); + + var filesetIds = new HashSet(); + var overflow = false; + while (await filesetIdReader.ReadAsync(token).ConfigureAwait(false)) + { + if (filesetIds.Count >= 10) + { + overflow = true; + break; + } + filesetIds.Add(filesetIdReader.ConvertValueToInt64(0)); + } + + var pairs = FilesetTimes(token) + .Select((x, i) => new { FilesetId = x.Key, Version = i, Time = x.Value }) + .Where(x => filesetIds.Contains(x.FilesetId)) + .Select(x => $"Fileset {x.Version}: {x.Time} (id = {x.FilesetId})"); + + // Fall back to a generic error message if we can't find the fileset ids + if (!await pairs.AnyAsync(cancellationToken: token).ConfigureAwait(false)) + throw new DatabaseInconsistencyException($"Detected {nonAttachedFiles} file(s) in FilesetEntry without corresponding FileLookup entry"); + + if (overflow) + pairs = pairs.Append("... and more"); + + throw new DatabaseInconsistencyException($"Detected {nonAttachedFiles} file(s) in FilesetEntry without corresponding FileLookup entry in the following filesets:{Environment.NewLine}{string.Join(Environment.NewLine, pairs)}"); + } + + if (verifyfilelists) + { + var anyError = new List(); + await using (var cmd2 = m_connection.CreateCommand()) + { + cmd2.SetTransaction(m_rtr); + cmd.SetCommandAndParameters(@" + SELECT ""ID"" + FROM ""Fileset"" + "); + await foreach (var filesetid in cmd.ExecuteReaderEnumerableAsync(token).Select(x => x.ConvertValueToInt64(0, -1)).ConfigureAwait(false)) + { + var expandedlist = await cmd2.SetCommandAndParameters($@" + SELECT COUNT(*) + FROM ( + SELECT DISTINCT ""Path"" + FROM ({LIST_FILESETS}) + UNION + SELECT DISTINCT ""Path"" + FROM ({LIST_FOLDERS_AND_SYMLINKS}) + ) + ") + .SetParameterValue("@FilesetId", filesetid) + .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) + .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) + .ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false); + + //var storedfilelist = cmd2.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""FileLookup"" WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FileLookup"".""BlocksetID"" != @FolderBlocksetId AND ""FileLookup"".""BlocksetID"" != @SymlinkBlocksetId"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); + + var storedlist = await cmd2.SetCommandAndParameters(@" + SELECT COUNT(*) + FROM ""FilesetEntry"" + WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId + ") + .SetParameterValue("@FilesetId", filesetid) + .ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false); + + if (expandedlist != storedlist) + { + var filesetname = filesetid.ToString(); + var fileset = await FilesetTimes(token) + .Zip( + AsyncEnumerable.Range(0, await FilesetTimes(token) + .CountAsync(cancellationToken: token) + .ConfigureAwait(false) + ), (a, b) => new Tuple(b, a.Key, a.Value) + ) + .FirstOrDefaultAsync(x => x.Item2 == filesetid, cancellationToken: token) + .ConfigureAwait(false); + + if (fileset != null) + filesetname = $"version {fileset.Item1}: {fileset.Item3} (database id: {fileset.Item2})"; + + anyError.Add($"Unexpected difference in fileset {filesetname}, found {expandedlist} entries, but expected {storedlist}"); + } } } - var nonAttachedFiles = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FileID"" NOT IN (SELECT ""ID"" FROM ""FileLookup"")"); - if (nonAttachedFiles != 0) + if (anyError.Count != 0) { - // Attempt to create a better error message by finding the first 10 fileset ids with the issue - using var filesetIdReader = cmd.ExecuteReader(@"SELECT DISTINCT(FilesetID) FROM ""FilesetEntry"" WHERE ""FileID"" NOT IN (SELECT ""ID"" FROM ""FileLookup"") LIMIT 11"); - var filesetIds = new HashSet(); - var overflow = false; - while (filesetIdReader.Read()) - { - if (filesetIds.Count >= 10) - { - overflow = true; - break; - } - filesetIds.Add(filesetIdReader.ConvertValueToInt64(0)); - } - - var pairs = FilesetTimes - .Select((x, i) => new { FilesetId = x.Key, Version = i, Time = x.Value }) - .Where(x => filesetIds.Contains(x.FilesetId)) - .Select(x => $"Fileset {x.Version}: {x.Time} (id = {x.FilesetId})"); - - // Fall back to a generic error message if we can't find the fileset ids - if (!pairs.Any()) - throw new DatabaseInconsistencyException($"Detected {nonAttachedFiles} file(s) in FilesetEntry without corresponding FileLookup entry"); - - if (overflow) - pairs = pairs.Append("... and more"); - - throw new DatabaseInconsistencyException($"Detected {nonAttachedFiles} file(s) in FilesetEntry without corresponding FileLookup entry in the following filesets:{Environment.NewLine}{string.Join(Environment.NewLine, pairs)}"); - } - - if (verifyfilelists) - { - var anyError = new List(); - using (var cmd2 = m_connection.CreateCommand(transaction)) - { - foreach (var filesetid in cmd.ExecuteReaderEnumerable(@"SELECT ""ID"" FROM ""Fileset"" ").Select(x => x.ConvertValueToInt64(0, -1))) - { - var expandedCmd = FormatInvariant($@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({LocalDatabase.LIST_FILESETS}) UNION SELECT DISTINCT ""Path"" FROM ({LocalDatabase.LIST_FOLDERS_AND_SYMLINKS}))"); - var expandedlist = cmd2 - .SetCommandAndParameters(expandedCmd) - .SetParameterValue("@FilesetId", filesetid) - .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) - .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) - .ExecuteScalarInt64(0); - //var storedfilelist = cmd2.ExecuteScalarInt64(FormatInvariant(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""FileLookup"" WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FileLookup"".""BlocksetID"" != @FolderBlocksetId AND ""FileLookup"".""BlocksetID"" != @SymlinkBlocksetId"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); - var storedlist = cmd2.SetCommandAndParameters(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetEntry"".""FilesetID"" = @FilesetId") - .SetParameterValue("@FilesetId", filesetid) - .ExecuteScalarInt64(0); - - if (expandedlist != storedlist) - { - var filesetname = filesetid.ToString(); - var fileset = FilesetTimes.Zip(Enumerable.Range(0, FilesetTimes.Count()), (a, b) => new Tuple(b, a.Key, a.Value)).FirstOrDefault(x => x.Item2 == filesetid); - if (fileset != null) - filesetname = $"version {fileset.Item1}: {fileset.Item3} (database id: {fileset.Item2})"; - anyError.Add($"Unexpected difference in fileset {filesetname}, found {expandedlist} entries, but expected {storedlist}"); - } - } - } - if (anyError.Any()) - { - throw new DatabaseInconsistencyException(string.Join("\n\r", anyError), "FilesetDifferences"); - } + throw new DatabaseInconsistencyException(string.Join("\n\r", anyError), "FilesetDifferences"); } } } + /// + /// Represents a block in the database. + /// public interface IBlock { + /// + /// Gets the hash of the block. + /// string Hash { get; } + /// + /// Gets the size of the block in bytes. + /// long Size { get; } } - internal class Block : IBlock + /// + /// Represents a block in the database with its hash and size. + /// + /// The hash of the block. + /// The size of the block in bytes. + internal class Block(string hash, long size) : IBlock { - public string Hash { get; private set; } - public long Size { get; private set; } - - public Block(string hash, long size) - { - Hash = hash; - Size = size; - } + public string Hash { get; private set; } = hash; + public long Size { get; private set; } = size; } - public IEnumerable GetBlocks(long volumeid, IDbTransaction? transaction = null) + /// + /// Retrieves all blocks associated with a specific volume ID. + /// + /// The ID of the volume to retrieve blocks for. + /// Cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of blocks associated with the specified volume ID. + public async IAsyncEnumerable GetBlocks(long volumeid, [EnumeratorCancellation] CancellationToken token) { - using var cmd = m_connection.CreateCommand(transaction) - .SetCommandAndParameters(@"SELECT DISTINCT ""Hash"", ""Size"" FROM ""Block"" WHERE ""VolumeID"" = @VolumeId") + await using var cmd = await m_connection.CreateCommandAsync(@" + SELECT DISTINCT + ""Hash"", + ""Size"" + FROM ""Block"" + WHERE ""VolumeID"" = @VolumeId + ", token) + .ConfigureAwait(false); + + cmd.SetTransaction(m_rtr) .SetParameterValue("@VolumeId", volumeid); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - yield return new Block(rd.ConvertValueToString(0) ?? throw new Exception("Hash is null"), rd.ConvertValueToInt64(1)); + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + yield return new Block( + rd.ConvertValueToString(0) ?? throw new Exception("Hash is null"), + rd.ConvertValueToInt64(1) + ); } - // TODO: Replace this with an enumerable method - private class BlocklistHashEnumerable : IEnumerable + /// + /// An asynchronous enumerable that retrieves blocklist hashes for files in a specific fileset. + /// + private class BlocklistHashEnumerable : IAsyncEnumerable { - private class BlocklistHashEnumerator : IEnumerator + /// + /// An asynchronous enumerator for the blocklist hashes. + /// + private class BlocklistHashEnumerator : IDisposable, IAsyncEnumerator { - private readonly IDataReader m_reader; + /// + /// The data reader used to read blocklist hashes. + /// + private readonly SqliteDataReader m_reader; + /// + /// The parent enumerable that this enumerator belongs to. + /// private readonly BlocklistHashEnumerable m_parent; + /// + /// The path of the current file being processed. + /// private string? m_path = null; + /// + /// Indicates if this is the first entry being processed. + /// private bool m_first = true; + /// + /// The current blocklist hash being processed. + /// private string? m_current = null; + /// + /// The cancellation token for this enumerator. + /// + private readonly CancellationToken m_token; - public BlocklistHashEnumerator(BlocklistHashEnumerable parent, IDataReader reader) + /// + /// Initializes a new instance of the class. + /// + /// The parent enumerable that this enumerator belongs to. + /// The data reader used to read blocklist hashes. + /// Cancellation token to monitor for cancellation requests. + public BlocklistHashEnumerator(BlocklistHashEnumerable parent, SqliteDataReader reader, CancellationToken token) { m_reader = reader; m_parent = parent; + m_token = token; } + /// + /// Gets the current blocklist hash. + /// public string Current { get { return m_current!; } } - public void Dispose() - { - } + public void Dispose() { } + // The warning is suppressed because the interface requires the method, but there's nothing to dispose. +#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously + public async ValueTask DisposeAsync() { } +#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously - object System.Collections.IEnumerator.Current { get { return Current; } } - - public bool MoveNext() + public async ValueTask MoveNextAsync() { m_first = false; @@ -1084,7 +2139,7 @@ ON if (m_current == null) return false; - if (!m_reader.Read()) + if (!await m_reader.ReadAsync(m_token).ConfigureAwait(false)) { m_current = null; m_parent.MoreData = false; @@ -1103,6 +2158,9 @@ ON } } + /// + /// Resets the enumerator to its initial state. + /// public void Reset() { if (!m_first) @@ -1112,269 +2170,368 @@ ON } } - private readonly IDataReader m_reader; + /// + /// The data reader used to read blocklist hashes. + /// + private readonly SqliteDataReader m_reader; - public BlocklistHashEnumerable(IDataReader reader) + /// + /// Initializes a new instance of the class. + /// + public BlocklistHashEnumerable(SqliteDataReader reader) { m_reader = reader; MoreData = true; } + /// + /// Indicates if there is more data to read. + /// public bool MoreData { get; protected set; } - public IEnumerator GetEnumerator() + /// + /// Gets an enumerator that iterates through the blocklist hashes. + /// + /// Cancellation token to monitor for cancellation requests. + public IAsyncEnumerator GetEnumerator(CancellationToken token) { - return new BlocklistHashEnumerator(this, m_reader); + return new BlocklistHashEnumerator(this, m_reader, token); } - System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() + IAsyncEnumerator IAsyncEnumerable.GetAsyncEnumerator(CancellationToken token) { - return GetEnumerator(); + return GetEnumerator(token); } } - public const string LIST_FILESETS = @" -SELECT - ""L"".""Path"", - ""L"".""Lastmodified"", - ""L"".""Filelength"", - ""L"".""Filehash"", - ""L"".""Metahash"", - ""L"".""Metalength"", - ""L"".""BlocklistHash"", - ""L"".""FirstBlockHash"", - ""L"".""FirstBlockSize"", - ""L"".""FirstMetaBlockHash"", - ""L"".""FirstMetaBlockSize"", - ""M"".""Hash"" AS ""MetaBlocklistHash"" -FROM - ( - SELECT - ""J"".""Path"", - ""J"".""Lastmodified"", - ""J"".""Filelength"", - ""J"".""Filehash"", - ""J"".""Metahash"", - ""J"".""Metalength"", - ""K"".""Hash"" AS ""BlocklistHash"", - ""J"".""FirstBlockHash"", - ""J"".""FirstBlockSize"", - ""J"".""FirstMetaBlockHash"", - ""J"".""FirstMetaBlockSize"", - ""J"".""MetablocksetID"" - FROM - ( - SELECT - ""A"".""Path"" AS ""Path"", - ""D"".""Lastmodified"" AS ""Lastmodified"", - ""B"".""Length"" AS ""Filelength"", - ""B"".""FullHash"" AS ""Filehash"", - ""E"".""FullHash"" AS ""Metahash"", - ""E"".""Length"" AS ""Metalength"", - ""A"".""BlocksetID"" AS ""BlocksetID"", - ""F"".""Hash"" AS ""FirstBlockHash"", - ""F"".""Size"" AS ""FirstBlockSize"", - ""H"".""Hash"" AS ""FirstMetaBlockHash"", - ""H"".""Size"" AS ""FirstMetaBlockSize"", - ""C"".""BlocksetID"" AS ""MetablocksetID"" - FROM - ""File"" A - LEFT JOIN ""Blockset"" B - ON ""A"".""BlocksetID"" = ""B"".""ID"" - LEFT JOIN ""Metadataset"" C - ON ""A"".""MetadataID"" = ""C"".""ID"" - LEFT JOIN ""FilesetEntry"" D - ON ""A"".""ID"" = ""D"".""FileID"" - LEFT JOIN ""Blockset"" E - ON ""E"".""ID"" = ""C"".""BlocksetID"" - LEFT JOIN ""BlocksetEntry"" G - ON ""B"".""ID"" = ""G"".""BlocksetID"" - LEFT JOIN ""Block"" F - ON ""G"".""BlockID"" = ""F"".""ID"" - LEFT JOIN ""BlocksetEntry"" I - ON ""E"".""ID"" = ""I"".""BlocksetID"" - LEFT JOIN ""Block"" H - ON ""I"".""BlockID"" = ""H"".""ID"" - WHERE - ""A"".""BlocksetId"" >= 0 AND - ""D"".""FilesetID"" = @FilesetId AND - (""I"".""Index"" = 0 OR ""I"".""Index"" IS NULL) AND - (""G"".""Index"" = 0 OR ""G"".""Index"" IS NULL) - ) J - LEFT OUTER JOIN - ""BlocklistHash"" K - ON - ""K"".""BlocksetID"" = ""J"".""BlocksetID"" - ORDER BY ""J"".""Path"", ""K"".""Index"" - ) L - -LEFT OUTER JOIN - ""BlocklistHash"" M -ON - ""M"".""BlocksetID"" = ""L"".""MetablocksetID"" -"; - - public const string LIST_FOLDERS_AND_SYMLINKS = @" -SELECT - ""G"".""BlocksetID"", - ""G"".""ID"", - ""G"".""Path"", - ""G"".""Length"", - ""G"".""FullHash"", - ""G"".""Lastmodified"", - ""G"".""FirstMetaBlockHash"", - ""H"".""Hash"" AS ""MetablocklistHash"" -FROM - ( - SELECT - ""B"".""BlocksetID"", - ""B"".""ID"", - ""B"".""Path"", - ""D"".""Length"", - ""D"".""FullHash"", - ""A"".""Lastmodified"", - ""F"".""Hash"" AS ""FirstMetaBlockHash"", - ""C"".""BlocksetID"" AS ""MetaBlocksetID"" - FROM - ""FilesetEntry"" A, - ""File"" B, - ""Metadataset"" C, - ""Blockset"" D, - ""BlocksetEntry"" E, - ""Block"" F - WHERE - ""A"".""FileID"" = ""B"".""ID"" - AND ""B"".""MetadataID"" = ""C"".""ID"" - AND ""C"".""BlocksetID"" = ""D"".""ID"" - AND ""E"".""BlocksetID"" = ""C"".""BlocksetID"" - AND ""E"".""BlockID"" = ""F"".""ID"" - AND ""E"".""Index"" = 0 - AND (""B"".""BlocksetID"" = @FolderBlocksetId OR ""B"".""BlocksetID"" = @SymlinkBlocksetId) - AND ""A"".""FilesetID"" = @FilesetId - ) G -LEFT OUTER JOIN - ""BlocklistHash"" H -ON - ""H"".""BlocksetID"" = ""G"".""MetaBlocksetID"" -ORDER BY - ""G"".""Path"", ""H"".""Index"" - -"; - - public void WriteFileset(Volumes.FilesetVolumeWriter filesetvolume, long filesetId, IDbTransaction transaction) - { - using (var cmd = m_connection.CreateCommand(transaction)) - { - cmd.SetCommandAndParameters(LIST_FOLDERS_AND_SYMLINKS) - .SetParameterValue("@FilesetId", filesetId) - .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) - .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID); - - string? lastpath = null; - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - { - var blocksetID = rd.ConvertValueToInt64(0, -1); - var path = rd.ConvertValueToString(2); - var metalength = rd.ConvertValueToInt64(3, -1); - var metahash = rd.ConvertValueToString(4); - var metablockhash = rd.ConvertValueToString(6); - var metablocklisthash = rd.ConvertValueToString(7); - - if (path == lastpath) - Logging.Log.WriteWarningMessage(LOGTAG, "DuplicatePathFound", null, "Duplicate path detected: {0}", path); - - lastpath = path; - - if (blocksetID == FOLDER_BLOCKSET_ID) - filesetvolume.AddDirectory(path, metahash, metalength, metablockhash, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash }); - else if (blocksetID == SYMLINK_BLOCKSET_ID) - filesetvolume.AddSymlink(path, metahash, metalength, metablockhash, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash }); - } - - // TODO: Perhaps run the above query after recreate and compare count(*) with count(*) from filesetentry where id = x - - cmd.SetCommandAndParameters(LIST_FILESETS) - .SetParameterValue("@FilesetId", filesetId); - - using (var rd = cmd.ExecuteReader()) - if (rd.Read()) - { - var more = false; - do - { - var path = rd.ConvertValueToString(0); - var filehash = rd.ConvertValueToString(3); - var size = rd.ConvertValueToInt64(2); - var lastmodified = new DateTime(rd.ConvertValueToInt64(1, 0), DateTimeKind.Utc); - var metahash = rd.ConvertValueToString(4); - var metasize = rd.ConvertValueToInt64(5, -1); - var p = rd.GetValue(6); - var blrd = (p == null || p == DBNull.Value) ? null : new BlocklistHashEnumerable(rd); - var blockhash = rd.ConvertValueToString(7); - var blocksize = rd.ConvertValueToInt64(8, -1); - var metablockhash = rd.ConvertValueToString(9); - //var metablocksize = rd.ConvertValueToInt64(10, -1); - var metablocklisthash = rd.ConvertValueToString(11); - - if (blockhash == filehash) - blockhash = null; - - if (metablockhash == metahash) - metablockhash = null; - - filesetvolume.AddFile(path, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blrd, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash }); - if (blrd == null) - more = rd.Read(); - else - more = blrd.MoreData; - - } while (more); - } - } - } - - public void LinkFilesetToVolume(long filesetid, long volumeid, IDbTransaction transaction) - { - using (var cmd = m_connection.CreateCommand(transaction)) - { - var c = cmd.SetCommandAndParameters(@"UPDATE ""Fileset"" SET ""VolumeID"" = @VolumeId WHERE ""ID"" = @FilesetId") - .SetParameterValue("@VolumeId", volumeid) - .SetParameterValue("@FilesetId", filesetid) - .ExecuteNonQuery(); - - if (c != 1) - throw new Exception($"Failed to link filesetid {filesetid} to volumeid {volumeid}"); - } - } - - public void PushTimestampChangesToPreviousVersion(long filesetId, IDbTransaction transaction) - { - var query = @" -UPDATE FilesetEntry AS oldVersion -SET Lastmodified = tempVersion.Lastmodified -FROM FilesetEntry AS tempVersion -WHERE oldVersion.FileID = tempVersion.FileID -AND tempVersion.FilesetID = @FilesetId -AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER BY Timestamp DESC LIMIT 1)"; - - using (var cmd = m_connection.CreateCommand(transaction, query)) - cmd.SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQuery(); - } - /// - /// Keeps a list of filenames in a temporary table with a single column Path - /// - public class FilteredFilenameTable : IDisposable - { - public string Tablename { get; private set; } - private readonly IDbConnection m_connection; + /// SQL query for listing all filesets in the database, including their metadata and blocklist hashes. + /// + public const string LIST_FILESETS = @" + SELECT + ""L"".""Path"", + ""L"".""Lastmodified"", + ""L"".""Filelength"", + ""L"".""Filehash"", + ""L"".""Metahash"", + ""L"".""Metalength"", + ""L"".""BlocklistHash"", + ""L"".""FirstBlockHash"", + ""L"".""FirstBlockSize"", + ""L"".""FirstMetaBlockHash"", + ""L"".""FirstMetaBlockSize"", + ""M"".""Hash"" AS ""MetaBlocklistHash"" + FROM ( + SELECT + ""J"".""Path"", + ""J"".""Lastmodified"", + ""J"".""Filelength"", + ""J"".""Filehash"", + ""J"".""Metahash"", + ""J"".""Metalength"", + ""K"".""Hash"" AS ""BlocklistHash"", + ""J"".""FirstBlockHash"", + ""J"".""FirstBlockSize"", + ""J"".""FirstMetaBlockHash"", + ""J"".""FirstMetaBlockSize"", + ""J"".""MetablocksetID"" + FROM ( + SELECT + ""A"".""Path"" AS ""Path"", + ""D"".""Lastmodified"" AS ""Lastmodified"", + ""B"".""Length"" AS ""Filelength"", + ""B"".""FullHash"" AS ""Filehash"", + ""E"".""FullHash"" AS ""Metahash"", + ""E"".""Length"" AS ""Metalength"", + ""A"".""BlocksetID"" AS ""BlocksetID"", + ""F"".""Hash"" AS ""FirstBlockHash"", + ""F"".""Size"" AS ""FirstBlockSize"", + ""H"".""Hash"" AS ""FirstMetaBlockHash"", + ""H"".""Size"" AS ""FirstMetaBlockSize"", + ""C"".""BlocksetID"" AS ""MetablocksetID"" + FROM ""File"" ""A"" + LEFT JOIN ""Blockset"" ""B"" + ON ""A"".""BlocksetID"" = ""B"".""ID"" + LEFT JOIN ""Metadataset"" ""C"" + ON ""A"".""MetadataID"" = ""C"".""ID"" + LEFT JOIN ""FilesetEntry"" ""D"" + ON ""A"".""ID"" = ""D"".""FileID"" + LEFT JOIN ""Blockset"" ""E"" + ON ""E"".""ID"" = ""C"".""BlocksetID"" + LEFT JOIN ""BlocksetEntry"" ""G"" + ON ""B"".""ID"" = ""G"".""BlocksetID"" + LEFT JOIN ""Block"" ""F"" + ON ""G"".""BlockID"" = ""F"".""ID"" + LEFT JOIN ""BlocksetEntry"" ""I"" + ON ""E"".""ID"" = ""I"".""BlocksetID"" + LEFT JOIN ""Block"" ""H"" + ON ""I"".""BlockID"" = ""H"".""ID"" + WHERE + ""A"".""BlocksetId"" >= 0 + AND ""D"".""FilesetID"" = @FilesetId + AND ( + ""I"".""Index"" = 0 + OR ""I"".""Index"" IS NULL + ) + AND ( + ""G"".""Index"" = 0 + OR ""G"".""Index"" IS NULL + ) + ) ""J"" + LEFT OUTER JOIN ""BlocklistHash"" ""K"" + ON ""K"".""BlocksetID"" = ""J"".""BlocksetID"" + ORDER BY + ""J"".""Path"", + ""K"".""Index"" + ) ""L"" + LEFT OUTER JOIN ""BlocklistHash"" ""M"" + ON ""M"".""BlocksetID"" = ""L"".""MetablocksetID"" + "; - public FilteredFilenameTable(IDbConnection connection, IFilter filter, IDbTransaction? transaction) + /// + /// SQL query for listing folders and symlinks in a specific fileset, including their metadata and blocklist hashes. + /// + public const string LIST_FOLDERS_AND_SYMLINKS = @" + SELECT + ""G"".""BlocksetID"", + ""G"".""ID"", + ""G"".""Path"", + ""G"".""Length"", + ""G"".""FullHash"", + ""G"".""Lastmodified"", + ""G"".""FirstMetaBlockHash"", + ""H"".""Hash"" AS ""MetablocklistHash"" + FROM ( + SELECT + ""B"".""BlocksetID"", + ""B"".""ID"", + ""B"".""Path"", + ""D"".""Length"", + ""D"".""FullHash"", + ""A"".""Lastmodified"", + ""F"".""Hash"" AS ""FirstMetaBlockHash"", + ""C"".""BlocksetID"" AS ""MetaBlocksetID"" + FROM + ""FilesetEntry"" ""A"", + ""File"" ""B"", + ""Metadataset"" ""C"", + ""Blockset"" ""D"", + ""BlocksetEntry"" ""E"", + ""Block"" ""F"" + WHERE + ""A"".""FileID"" = ""B"".""ID"" + AND ""B"".""MetadataID"" = ""C"".""ID"" + AND ""C"".""BlocksetID"" = ""D"".""ID"" + AND ""E"".""BlocksetID"" = ""C"".""BlocksetID"" + AND ""E"".""BlockID"" = ""F"".""ID"" + AND ""E"".""Index"" = 0 + AND ( + ""B"".""BlocksetID"" = @FolderBlocksetId + OR ""B"".""BlocksetID"" = @SymlinkBlocksetId + ) + AND ""A"".""FilesetID"" = @FilesetId + ) ""G"" + LEFT OUTER JOIN ""BlocklistHash"" ""H"" + ON ""H"".""BlocksetID"" = ""G"".""MetaBlocksetID"" + ORDER BY + ""G"".""Path"", + ""H"".""Index"" + "; + + /// + /// Writes the contents of a fileset to a specified volume writer. + /// + /// The volume writer to which the fileset will be written. + /// The ID of the fileset to write. + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the fileset has been written. + public async Task WriteFileset(Volumes.FilesetVolumeWriter filesetvolume, long filesetId, CancellationToken token) + { + await using var cmd = m_connection.CreateCommand() + .SetCommandAndParameters(LIST_FOLDERS_AND_SYMLINKS) + .SetTransaction(m_rtr) + .SetParameterValue("@FilesetId", filesetId) + .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) + .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID); + + string? lastpath = null; + await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) + { + var blocksetID = rd.ConvertValueToInt64(0, -1); + var path = rd.ConvertValueToString(2); + var metalength = rd.ConvertValueToInt64(3, -1); + var metahash = rd.ConvertValueToString(4); + var metablockhash = rd.ConvertValueToString(6); + var metablocklisthash = rd.ConvertValueToString(7); + + if (path == lastpath) + Logging.Log.WriteWarningMessage(LOGTAG, "DuplicatePathFound", null, "Duplicate path detected: {0}", path); + + lastpath = path; + + if (blocksetID == FOLDER_BLOCKSET_ID) + filesetvolume.AddDirectory(path, metahash, metalength, metablockhash, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash }); + else if (blocksetID == SYMLINK_BLOCKSET_ID) + filesetvolume.AddSymlink(path, metahash, metalength, metablockhash, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash }); + } + + // TODO: Perhaps run the above query after recreate and compare count(*) with count(*) from filesetentry where id = x + + cmd.SetCommandAndParameters(LIST_FILESETS); + cmd.SetParameterValue("@FilesetId", filesetId); + + await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + if (await rd.ReadAsync(token).ConfigureAwait(false)) + { + var more = false; + do + { + var path = rd.ConvertValueToString(0); + var filehash = rd.ConvertValueToString(3); + var size = rd.ConvertValueToInt64(2); + var lastmodified = new DateTime(rd.ConvertValueToInt64(1, 0), DateTimeKind.Utc); + var metahash = rd.ConvertValueToString(4); + var metasize = rd.ConvertValueToInt64(5, -1); + var p = rd.GetValue(6); + var blrd = (p == null || p == DBNull.Value) ? null : new BlocklistHashEnumerable(rd); + var blockhash = rd.ConvertValueToString(7); + var blocksize = rd.ConvertValueToInt64(8, -1); + var metablockhash = rd.ConvertValueToString(9); + //var metablocksize = rd.ConvertValueToInt64(10, -1); + var metablocklisthash = rd.ConvertValueToString(11); + + if (blockhash == filehash) + blockhash = null; + + if (metablockhash == metahash) + metablockhash = null; + + await filesetvolume.AddFile( + path, + filehash, + size, + lastmodified, + metahash, + metasize, + metablockhash, + blockhash, + blocksize, + blrd, + string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash } + ) + .ConfigureAwait(false); + + if (blrd == null) + more = await rd.ReadAsync(token).ConfigureAwait(false); + else + more = blrd.MoreData; + + } while (more); + } + } + + /// + /// Links a fileset to a specific volume by updating the VolumeID in the Fileset table. + /// + /// The ID of the fileset to link. + /// The ID of the volume to link the fileset to. + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the link operation is finished. + public async Task LinkFilesetToVolume(long filesetid, long volumeid, CancellationToken token) + { + await using var cmd = await m_connection.CreateCommandAsync(@" + UPDATE ""Fileset"" + SET ""VolumeID"" = @VolumeId + WHERE ""ID"" = @FilesetId + ", token) + .ConfigureAwait(false); + + var c = await cmd.SetTransaction(m_rtr) + .SetParameterValue("@VolumeId", volumeid) + .SetParameterValue("@FilesetId", filesetid) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + if (c != 1) + throw new Exception($"Failed to link filesetid {filesetid} to volumeid {volumeid}"); + } + + /// + /// Pushes timestamp changes from the latest version of a fileset to the previous version. + /// + /// The ID of the fileset whose timestamp changes will be pushed. + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the timestamp changes have been pushed. + public async Task PushTimestampChangesToPreviousVersion(long filesetId, CancellationToken token) + { + var query = @" + UPDATE ""FilesetEntry"" AS ""oldVersion"" + SET ""Lastmodified"" = ""tempVersion"".""Lastmodified"" + FROM ""FilesetEntry"" AS ""tempVersion"" + WHERE + ""oldVersion"".""FileID"" = ""tempVersion"".""FileID"" + AND ""tempVersion"".""FilesetID"" = @FilesetId + AND ""oldVersion"".""FilesetID"" = ( + SELECT ""ID"" + FROM ""Fileset"" + WHERE ""ID"" != @FilesetId + ORDER BY ""Timestamp"" DESC + LIMIT 1 + ) + "; + + await using var cmd = await m_connection + .CreateCommandAsync(query, token) + .ConfigureAwait(false); + + await cmd.SetTransaction(m_rtr) + .SetParameterValue("@FilesetId", filesetId) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } + + /// + /// Keeps a list of filenames in a temporary table with a single column Path. + /// + public class FilteredFilenameTable : IDisposable, IAsyncDisposable + { + /// + /// The name of the temporary table that holds the filtered filenames. + /// + public string Tablename { get; private set; } + /// + /// The database used to create and manage the temporary table. + /// + private readonly LocalDatabase m_db; + + /// + /// Initializes a new instance of the class. + /// + /// The database to use for creating the temporary table. + private FilteredFilenameTable(LocalDatabase db) { - m_connection = connection; Tablename = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + m_db = db; + } + + [Obsolete("Calling this constructor will throw an exception. Use the Create method instead.")] + public FilteredFilenameTable(SqliteConnection connection, IFilter filter) + { + throw new NotImplementedException("Use the Create method instead."); + } + + /// + /// Creates a new instance of the class asynchronously. + /// + /// The database to use for creating the temporary table. + /// The filter to apply to the filenames. + /// Cancellation token to monitor for cancellation requests. + /// A task that represents the asynchronous operation, with a as the result. + public static async Task CreateFilteredFilenameTableAsync(LocalDatabase db, IFilter filter, CancellationToken token) + { + var ftt = new FilteredFilenameTable(db); + var type = FilterType.Regexp; if (filter is FilterExpression expression) type = expression.Type; @@ -1385,36 +2542,60 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER if (filter.Empty) { - using (var cmd = m_connection.CreateCommand(transaction)) - { - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{Tablename}"" AS SELECT DISTINCT ""Path"" FROM ""File"" ")); - return; - } + await using var cmd = await db.Connection.CreateCommandAsync($@" + CREATE TEMPORARY TABLE ""{ftt.Tablename}"" AS + SELECT DISTINCT ""Path"" + FROM ""File"" + ", token) + .ConfigureAwait(false); + + await cmd.SetTransaction(db.Transaction) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + return ftt; } if (type == FilterType.Regexp || type == FilterType.Group) { - using (var cmd = m_connection.CreateCommand(transaction)) - { - // TODO: Optimize this to not rely on the "File" view, and not instantiate the paths in full - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{Tablename}"" (""Path"" TEXT NOT NULL)")); - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) + // TODO: Optimize this to not rely on the "File" view, and not instantiate the paths in full + await using var cmd = await db.Connection.CreateCommandAsync($@" + CREATE TEMPORARY TABLE ""{ftt.Tablename}"" ( + ""Path"" TEXT NOT NULL + ) + ", token) + .ConfigureAwait(false); + + await cmd.SetTransaction(db.Transaction) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + cmd.SetCommandAndParameters($@" + INSERT INTO ""{ftt.Tablename}"" (""Path"") + VALUES (@Path) + "); + + await using var c2 = await db.Connection.CreateCommandAsync(@" + SELECT DISTINCT ""Path"" + FROM ""File"" + ", token) + .ConfigureAwait(false); + + c2.SetTransaction(db.Transaction); + await using (var rd = await c2.ExecuteReaderAsync(token).ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) { - cmd.SetCommandAndParameters(tr.Parent, FormatInvariant($@"INSERT INTO ""{Tablename}"" (""Path"") VALUES (@Path)")); - using (var c2 = m_connection.CreateCommand()) - using (var rd = c2.ExecuteReader(@"SELECT DISTINCT ""Path"" FROM ""File"" ")) - while (rd.Read()) - { - var p = rd.ConvertValueToString(0) ?? ""; - if (FilterExpression.Matches(filter, p)) - cmd.SetParameterValue("@Path", p) - .ExecuteNonQuery(); - } - - - tr.Commit(); + var p = rd.ConvertValueToString(0) ?? ""; + if (FilterExpression.Matches(filter, p)) + { + await cmd + .SetParameterValue("@Path", p) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } } - } + + await db.Transaction.CommitAsync(token: token).ConfigureAwait(false); } else { @@ -1428,94 +2609,154 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER var argName = $"@Arg{args.Count}"; if (type == FilterType.Wildcard) { - sb.Append(FormatInvariant(@$"""Path"" LIKE {argName}")); + sb.Append(@$"""Path"" LIKE {argName}"); args.Add(argName, f.Replace('*', '%').Replace('?', '_')); } else { - sb.Append(FormatInvariant(@$"""Path"" = {argName}")); + sb.Append(@$"""Path"" = {argName}"); args.Add(argName, f); } } - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - using (var cmd = m_connection.CreateCommand(tr.Parent)) - { - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{Tablename}"" (""Path"" TEXT NOT NULL)")); - cmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""{Tablename}"" SELECT DISTINCT ""Path"" FROM ""File"" WHERE {sb}"), args); - tr.Commit(); - } + await using var cmd = db.Connection.CreateCommand(); + cmd.SetTransaction(db.Transaction); + + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{ftt.Tablename}"" ( + ""Path"" TEXT NOT NULL + ) + ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@" + INSERT INTO ""{ftt.Tablename}"" + SELECT DISTINCT ""Path"" + FROM ""File"" + WHERE {sb} + ", args, token) + .ConfigureAwait(false); + + await db.Transaction.CommitAsync(token: token).ConfigureAwait(false); } + + return ftt; } public void Dispose() + { + DisposeAsync().AsTask().Await(); + } + + public async ValueTask DisposeAsync() { if (Tablename != null) try { - using (var cmd = m_connection.CreateCommand()) - cmd.ExecuteNonQuery(FormatInvariant(@$"DROP TABLE IF EXISTS ""{Tablename}"" ")); + await using var cmd = m_db.Connection.CreateCommand(); + await cmd.ExecuteNonQueryAsync(@$" + DROP TABLE IF EXISTS ""{Tablename}"" + ", default) + .ConfigureAwait(false); } catch { } finally { Tablename = null!; } } } - public void RenameRemoteFile(string oldname, string newname, IDbTransaction? transaction) + /// + /// Renames a remote file in the database, preserving its ID links. + /// + /// The current name of the remote file. + /// The new name for the remote file. + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the renaming operation is finished. + /// Thrown if the renaming operation does not affect exactly one row. + public async Task RenameRemoteFile(string oldname, string newname, CancellationToken token) { - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - using (var cmd = m_connection.CreateCommand(tr.Parent)) + //Rename the old entry, to preserve ID links + await using var cmd = await m_connection.CreateCommandAsync(@" + UPDATE ""Remotevolume"" + SET ""Name"" = @Newname + WHERE ""Name"" = @Oldname + ", token) + .ConfigureAwait(false); + + var c = await cmd.SetTransaction(m_rtr) + .SetParameterValue("@Newname", newname) + .SetParameterValue("@Oldname", oldname) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + if (c != 1) + throw new Exception($"Unexpected result from renaming \"{oldname}\" to \"{newname}\", expected {1} got {c}"); + + // Grab the type of entry + var type = (RemoteVolumeType)Enum.Parse( + typeof(RemoteVolumeType), + (await cmd.SetCommandAndParameters(@" + SELECT ""Type"" + FROM ""Remotevolume"" + WHERE ""Name"" = @Name + ") + .SetParameterValue("@Name", newname) + .ExecuteScalarAsync(token) + .ConfigureAwait(false) + )?.ToString() ?? "", + true + ); + + //Create a fake new entry with the old name and mark as deleting + // as this ensures we will remove it, if it shows up in some later listing + var newvolId = + await RegisterRemoteVolume(oldname, type, RemoteVolumeState.Deleting, token) + .ConfigureAwait(false); + + // IF needed, also create an empty fileset, so the validation works + if (type == RemoteVolumeType.Files) { - //Rename the old entry, to preserve ID links - var c = cmd.SetCommandAndParameters(@"UPDATE ""Remotevolume"" SET ""Name"" = @Newname WHERE ""Name"" = @Oldname") - .SetParameterValue("@Newname", newname) - .SetParameterValue("@Oldname", oldname) - .ExecuteNonQuery(); - - if (c != 1) - throw new Exception($"Unexpected result from renaming \"{oldname}\" to \"{newname}\", expected {1} got {c}"); - - // Grab the type of entry - var type = (RemoteVolumeType)Enum.Parse( - typeof(RemoteVolumeType), - cmd.SetCommandAndParameters(@"SELECT ""Type"" FROM ""Remotevolume"" WHERE ""Name"" = @Name") - .SetParameterValue("@Name", newname) - .ExecuteScalar() - ?.ToString() ?? "", - true); - - //Create a fake new entry with the old name and mark as deleting - // as this ensures we will remove it, if it shows up in some later listing - var newvolId = RegisterRemoteVolume(oldname, type, RemoteVolumeState.Deleting, tr.Parent); - - // IF needed, also create an empty fileset, so the validation works - if (type == RemoteVolumeType.Files) - CreateFileset(newvolId, DateTime.UnixEpoch, tr.Parent); - - tr.Commit(); + await CreateFileset(newvolId, DateTime.UnixEpoch, token) + .ConfigureAwait(false); + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } } /// /// Creates a timestamped backup operation to correctly associate the fileset with the time it was created. /// - /// The ID of the fileset volume to update - /// The timestamp of the operation to create - /// An optional external transaction - public virtual long CreateFileset(long volumeid, DateTime timestamp, IDbTransaction? transaction = null) + /// The ID of the fileset volume to update. + /// The timestamp of the operation to create. + /// Cancellation token to monitor for cancellation requests. + /// A task that when awaited contains the ID of the newly created fileset. + public virtual async Task CreateFileset(long volumeid, DateTime timestamp, CancellationToken token) { - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - using (var cmd = m_connection.CreateCommand(tr.Parent)) - { - var id = cmd.SetCommandAndParameters(@"INSERT INTO ""Fileset"" (""OperationID"", ""Timestamp"", ""VolumeID"", ""IsFullBackup"") VALUES (@OperationId, @Timestamp, @VolumeId, @IsFullBackup); SELECT last_insert_rowid();") - .SetParameterValue("@OperationId", m_operationid) - .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(timestamp)) - .SetParameterValue("@VolumeId", volumeid) - .SetParameterValue("@IsFullBackup", BackupType.PARTIAL_BACKUP) - .ExecuteScalarInt64(-1); - tr.Commit(); - return id; - } + await using var cmd = await m_connection.CreateCommandAsync(@" + INSERT INTO ""Fileset"" ( + ""OperationID"", + ""Timestamp"", + ""VolumeID"", + ""IsFullBackup"" + ) + VALUES ( + @OperationId, + @Timestamp, + @VolumeId, + @IsFullBackup + ); + SELECT last_insert_rowid(); + ", token) + .ConfigureAwait(false); + + var id = await cmd + .SetTransaction(m_rtr) + .SetParameterValue("@OperationId", m_operationid) + .SetParameterValue("@Timestamp", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(timestamp)) + .SetParameterValue("@VolumeId", volumeid) + .SetParameterValue("@IsFullBackup", BackupType.PARTIAL_BACKUP) + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); + + return id; } /// @@ -1523,270 +2764,430 @@ AND oldVersion.FilesetID = (SELECT ID FROM Fileset WHERE ID != @FilesetId ORDER /// /// The ID of the index volume. /// The ID of the block volume. - /// An optional transaction. - public void AddIndexBlockLink(long indexVolumeID, long blockVolumeID, IDbTransaction transaction) + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the link has been added. + /// Thrown if either volume ID is less than or equal to 0. + public async Task AddIndexBlockLink(long indexVolumeID, long blockVolumeID, CancellationToken token) { if (indexVolumeID <= 0) throw new ArgumentOutOfRangeException(nameof(indexVolumeID), "Index volume ID must be greater than 0."); if (blockVolumeID <= 0) throw new ArgumentOutOfRangeException(nameof(blockVolumeID), "Block volume ID must be greater than 0."); - m_insertIndexBlockLink.SetParameterValue("@IndexVolumeId", indexVolumeID) + await m_insertIndexBlockLink + .SetTransaction(m_rtr) + .SetParameterValue("@IndexVolumeId", indexVolumeID) .SetParameterValue("@BlockVolumeId", blockVolumeID) - .ExecuteNonQuery(transaction); + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); } /// - /// Returns all unique blocklists for a given volume + /// Returns all unique blocklists for a given volume. /// - /// The volume ID to get blocklists for - /// The blocksize - /// The size of the hash - /// An optional external transaction + /// The volume ID to get blocklists for. + /// The blocksize. + /// The size of the hash. + /// Cancellation token to monitor for cancellation requests. /// An enumerable of tuples containing the blocklist hash, the blocklist data and the length of the data - public IEnumerable<(string Hash, byte[] Buffer, int Size)> GetBlocklists(long volumeid, long blocksize, int hashsize, IDbTransaction? transaction = null) + public async IAsyncEnumerable<(string Hash, byte[] Buffer, int Size)> GetBlocklists(long volumeid, long blocksize, int hashsize, [EnumeratorCancellation] CancellationToken token) { - using (var cmd = m_connection.CreateCommand(transaction)) - { - // Group subquery by hash to ensure that each blocklist hash appears only once in the result - // The AllBlocks CTE is used to map both active and duplicate blocks from the volume, - // because the new volume is initially registered with only duplicate blocks. - var sql = @" -WITH ""AllBlocksInVolume"" AS ( - SELECT DISTINCT ""ID"", ""Hash"", ""Size"" FROM ( - SELECT ""ID"", ""Hash"", ""Size"" - FROM ""Block"" - WHERE ""VolumeID"" = @VolumeId - - UNION - - SELECT ""DuplicateBlock"".""BlockID"" AS ""ID"", ""Block"".""Hash"" AS ""Hash"", ""Block"".""Size"" AS ""Size"" - FROM ""DuplicateBlock"" INNER JOIN ""Block"" - ON ""DuplicateBlock"".""BlockID"" = ""Block"".""ID"" - WHERE ""DuplicateBlock"".""VolumeID"" = @VolumeId - ) -) + await using var cmd = m_connection.CreateCommand(m_rtr); + // Group subquery by hash to ensure that each blocklist hash appears only once in the result + // The AllBlocks CTE is used to map both active and duplicate blocks from the volume, + // because the new volume is initially registered with only duplicate blocks. + var sql = @" + WITH ""AllBlocksInVolume"" AS ( + SELECT DISTINCT + ""ID"", + ""Hash"", + ""Size"" + FROM ( + SELECT + ""ID"", + ""Hash"", + ""Size"" + FROM ""Block"" + WHERE ""VolumeID"" = @VolumeId + UNION + SELECT + ""DuplicateBlock"".""BlockID"" AS ""ID"", + ""Block"".""Hash"" AS ""Hash"", + ""Block"".""Size"" AS ""Size"" + FROM ""DuplicateBlock"" + INNER JOIN ""Block"" + ON ""DuplicateBlock"".""BlockID"" = ""Block"".""ID"" + WHERE ""DuplicateBlock"".""VolumeID"" = @VolumeId + ) + ) -SELECT ""A"".""Hash"" AS ""BlockHash"", ""C"".""Hash"" AS ""ItemHash"" -FROM ( - SELECT ""BlocklistHash"".""BlocksetID"", ""AllBlocksInVolume"".""Hash"", ""BlocklistHash"".""Index"" - FROM ""BlocklistHash"",""AllBlocksInVolume"" - WHERE ""BlocklistHash"".""Hash"" = ""AllBlocksInVolume"".""Hash"" GROUP BY ""AllBlocksInVolume"".""Hash"", ""AllBlocksInVolume"".""Size"" -) A, -""BlocksetEntry"" B, -""Block"" C + SELECT + ""A"".""Hash"" AS ""BlockHash"", + ""C"".""Hash"" AS ""ItemHash"" + FROM + ( + SELECT + ""BlocklistHash"".""BlocksetID"", + ""AllBlocksInVolume"".""Hash"", + ""BlocklistHash"".""Index"" + FROM + ""BlocklistHash"", + ""AllBlocksInVolume"" + WHERE ""BlocklistHash"".""Hash"" = ""AllBlocksInVolume"".""Hash"" + GROUP BY + ""AllBlocksInVolume"".""Hash"", + ""AllBlocksInVolume"".""Size"" + ) ""A"", + ""BlocksetEntry"" ""B"", + ""Block"" ""C"" -WHERE - ""B"".""BlocksetID"" = ""A"".""BlocksetID"" - AND ""B"".""Index"" >= (""A"".""Index"" * @HashesPerBlock) - AND ""B"".""Index"" < ((""A"".""Index"" + 1) * @HashesPerBlock) - AND ""C"".""ID"" = ""B"".""BlockID"" -ORDER BY ""A"".""BlocksetID"", ""B"".""Index"" -"; + WHERE + ""B"".""BlocksetID"" = ""A"".""BlocksetID"" + AND ""B"".""Index"" >= (""A"".""Index"" * @HashesPerBlock) + AND ""B"".""Index"" < ((""A"".""Index"" + 1) * @HashesPerBlock) + AND ""C"".""ID"" = ""B"".""BlockID"" + ORDER BY ""A"".""BlocksetID"", ""B"".""Index"" + "; - string? curHash = null; - var count = 0; - var buffer = new byte[blocksize]; + string? curHash = null; + var count = 0; + var buffer = new byte[blocksize]; - cmd.SetCommandAndParameters(sql) - .SetParameterValue("@VolumeId", volumeid) - .SetParameterValue("@HashesPerBlock", blocksize / hashsize); + cmd.SetCommandAndParameters(sql) + .SetParameterValue("@VolumeId", volumeid) + .SetParameterValue("@HashesPerBlock", blocksize / hashsize); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) + await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) + { + var blockhash = rd.ConvertValueToString(0); + if (curHash != null && (blockhash != curHash || count + hashsize > buffer.Length)) { - var blockhash = rd.ConvertValueToString(0); - if (curHash != null && (blockhash != curHash || count + hashsize > buffer.Length)) - { - yield return (curHash, buffer, count); - buffer = new byte[blocksize]; - count = 0; - } - - var hash = Convert.FromBase64String(rd.ConvertValueToString(1) ?? throw new Exception("Hash is null")); - Array.Copy(hash, 0, buffer, count, hashsize); - curHash = blockhash; - count += hashsize; + yield return (curHash, buffer, count); + buffer = new byte[blocksize]; + count = 0; } - if (curHash != null) - yield return (curHash, buffer, count); - } + var hash = Convert.FromBase64String(rd.ConvertValueToString(1) ?? throw new Exception("Hash is null")); + Array.Copy(hash, 0, buffer, count, hashsize); + curHash = blockhash; + count += hashsize; + } + + if (curHash != null) + yield return (curHash, buffer, count); } /// - /// Update fileset with full backup state + /// Update fileset with full backup state. /// - /// Existing file set to update - /// Full backup state - /// An optional external transaction - public void UpdateFullBackupStateInFileset(long fileSetId, bool isFullBackup, IDbTransaction? transaction = null) + /// Existing file set to update. + /// Full backup state. + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the update is finished. + public async Task UpdateFullBackupStateInFileset(long fileSetId, bool isFullBackup, CancellationToken token) { - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - using (var cmd = m_connection.CreateCommand(tr.Parent)) - { - cmd.SetCommandAndParameters(@"UPDATE ""Fileset"" SET ""IsFullBackup"" = @IsFullBackup WHERE ""ID"" = @FilesetId;") - .SetParameterValue("@FilesetId", fileSetId) - .SetParameterValue("@IsFullBackup", isFullBackup ? BackupType.FULL_BACKUP : BackupType.PARTIAL_BACKUP) - .ExecuteNonQuery(); - tr.Commit(); - } + await using var cmd = await m_connection.CreateCommandAsync(@" + UPDATE ""Fileset"" + SET ""IsFullBackup"" = @IsFullBackup + WHERE ""ID"" = @FilesetId + ", token) + .ConfigureAwait(false); + + await cmd.SetTransaction(m_rtr) + .SetParameterValue("@FilesetId", fileSetId) + .SetParameterValue("@IsFullBackup", isFullBackup ? BackupType.FULL_BACKUP : BackupType.PARTIAL_BACKUP) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); } /// - /// Removes all entries in the fileset entry table for a given fileset ID + /// Removes all entries in the fileset entry table for a given fileset ID. /// - /// The fileset ID to clear - /// The transaction to use - public void ClearFilesetEntries(long filesetId, IDbTransaction transaction) + /// The fileset ID to clear. + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the entries have been cleared. + public async Task ClearFilesetEntries(long filesetId, CancellationToken token) { - using (var cmd = m_connection.CreateCommand(transaction)) - cmd.SetCommandAndParameters(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId") - .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQuery(); + await using var cmd = await m_connection.CreateCommandAsync(@" + DELETE FROM ""FilesetEntry"" + WHERE ""FilesetID"" = @FilesetId + ", token) + .ConfigureAwait(false); + + await cmd.SetTransaction(m_rtr) + .SetParameterValue("@FilesetId", filesetId) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); } /// - /// Gets the last previous fileset that was incomplete + /// Gets the last previous fileset that was incomplete. /// - /// The transaction to use - /// The last incomplete fileset or default - public RemoteVolumeEntry GetLastIncompleteFilesetVolume(IDbTransaction transaction) + /// The transaction to use. + /// Cancellation token to monitor for cancellation requests. + /// A task that when awaited returns the last incomplete fileset or default + public async Task GetLastIncompleteFilesetVolume(CancellationToken token) { - var candidates = GetIncompleteFilesets(transaction).OrderBy(x => x.Value).ToArray(); - if (candidates.Any()) - return GetRemoteVolumeFromFilesetID(candidates.Last().Key, transaction); + var candidates = GetIncompleteFilesets(token) + .OrderBy(x => x.Value); + + if (await candidates.AnyAsync(cancellationToken: token).ConfigureAwait(false)) + return await GetRemoteVolumeFromFilesetID( + ( + await candidates.LastAsync(cancellationToken: token).ConfigureAwait(false) + ).Key + , token) + .ConfigureAwait(false); return default; } /// - /// Gets a list of incomplete filesets + /// Gets a list of incomplete filesets. /// - /// An optional transaction - /// A list of fileset IDs and timestamps - public IEnumerable> GetIncompleteFilesets(IDbTransaction? transaction) + /// Cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of key-value pairs where the key is the fileset ID and the value is the timestamp of the fileset. + public async IAsyncEnumerable> GetIncompleteFilesets([EnumeratorCancellation] CancellationToken token) { - using (var cmd = m_connection.CreateCommand(transaction)) - using (var rd = cmd.ExecuteReader(FormatInvariant(@$"SELECT DISTINCT ""Fileset"".""ID"", ""Fileset"".""Timestamp"" FROM ""Fileset"", ""RemoteVolume"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" IN (SELECT ""FilesetID"" FROM ""FilesetEntry"") AND (""RemoteVolume"".""State"" = '{RemoteVolumeState.Uploading}' OR ""RemoteVolume"".""State"" = '{RemoteVolumeState.Temporary}')"))) - while (rd.Read()) - { - yield return new KeyValuePair( - rd.ConvertValueToInt64(0), - ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime() - ); - } + await using var cmd = await m_connection.CreateCommandAsync(@$" + SELECT DISTINCT + ""Fileset"".""ID"", + ""Fileset"".""Timestamp"" + FROM + ""Fileset"", + ""RemoteVolume"" + WHERE + ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" + AND ""Fileset"".""ID"" IN ( + SELECT ""FilesetID"" + FROM ""FilesetEntry"" + ) + AND ( + ""RemoteVolume"".""State"" = '{RemoteVolumeState.Uploading}' + OR ""RemoteVolume"".""State"" = '{RemoteVolumeState.Temporary}' + ) + ", token) + .ConfigureAwait(false); + + await using var rd = await cmd.SetTransaction(m_rtr) + .ExecuteReaderAsync(token) + .ConfigureAwait(false); + + while (await rd.ReadAsync(token).ConfigureAwait(false)) + { + yield return new KeyValuePair( + rd.ConvertValueToInt64(0), + ParseFromEpochSeconds(rd.ConvertValueToInt64(1)) + .ToLocalTime() + ); + } } /// - /// Gets the remote volume entry from the fileset ID + /// Gets the remote volume entry from the fileset ID. /// - /// The fileset ID - /// An optional transaction - /// The remote volume entry or default - public RemoteVolumeEntry GetRemoteVolumeFromFilesetID(long filesetID, IDbTransaction? transaction = null) + /// The fileset ID. + /// Cancellation token to monitor for cancellation requests. + /// A task that when awaited returns the remote volume entry associated with the fileset ID, or default if not found. + public async Task GetRemoteVolumeFromFilesetID(long filesetID, CancellationToken token) { - using (var cmd = m_connection.CreateCommand(transaction)) - using (var rd = cmd.SetCommandAndParameters(@"SELECT ""RemoteVolume"".""ID"", ""Name"", ""Type"", ""Size"", ""Hash"", ""State"", ""DeleteGraceTime"", ""ArchiveTime"" FROM ""RemoteVolume"", ""Fileset"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Fileset"".""ID"" = @FilesetId") + await using var cmd = await m_connection.CreateCommandAsync(@" + SELECT + ""RemoteVolume"".""ID"", + ""Name"", + ""Type"", + ""Size"", + ""Hash"", + ""State"", + ""DeleteGraceTime"", + ""ArchiveTime"" + FROM + ""RemoteVolume"", + ""Fileset"" + WHERE + ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" + AND ""Fileset"".""ID"" = @FilesetId + ", token) + .ConfigureAwait(false); + + await using var rd = await cmd.SetTransaction(m_rtr) .SetParameterValue("@FilesetId", filesetID) - .ExecuteReader()) - if (rd.Read()) - return new RemoteVolumeEntry( - rd.ConvertValueToInt64(0, -1), - rd.ConvertValueToString(1), - rd.ConvertValueToString(4), - rd.ConvertValueToInt64(3, -1), - (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), rd.ConvertValueToString(2) ?? ""), - (RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.ConvertValueToString(5) ?? ""), - ParseFromEpochSeconds(rd.ConvertValueToInt64(6)).ToLocalTime(), - ParseFromEpochSeconds(rd.ConvertValueToInt64(7)).ToLocalTime() - ); - else - return default(RemoteVolumeEntry); + .ExecuteReaderAsync(token) + .ConfigureAwait(false); + + if (await rd.ReadAsync(token).ConfigureAwait(false)) + return new RemoteVolumeEntry( + rd.ConvertValueToInt64(0, -1), + rd.ConvertValueToString(1), + rd.ConvertValueToString(4), + rd.ConvertValueToInt64(3, -1), + (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), rd.ConvertValueToString(2) ?? ""), + (RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.ConvertValueToString(5) ?? ""), + ParseFromEpochSeconds(rd.ConvertValueToInt64(6)).ToLocalTime(), + ParseFromEpochSeconds(rd.ConvertValueToInt64(7)).ToLocalTime() + ); + else + return default(RemoteVolumeEntry); } - public void PurgeLogData(DateTime threshold) + /// + /// Purges log data and remote operations older than the specified threshold. + /// + /// The threshold date and time; all log data and remote operations older than this will be purged. + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the purge operation is finished. + public async Task PurgeLogData(DateTime threshold, CancellationToken token) { - using (var tr = m_connection.BeginTransactionSafe()) - using (var cmd = m_connection.CreateCommand(tr)) - { - var t = Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(threshold); - cmd.SetCommandAndParameters(@"DELETE FROM ""LogData"" WHERE ""Timestamp"" < @Timestamp") - .SetParameterValue("@Timestamp", t) - .ExecuteNonQuery(); - cmd.SetCommandAndParameters(@"DELETE FROM ""RemoteOperation"" WHERE ""Timestamp"" < @Timestamp") - .SetParameterValue("@Timestamp", t) - .ExecuteNonQuery(); + await using var cmd = m_connection.CreateCommand(m_rtr); + var t = Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(threshold); - tr.Commit(); - } + await cmd.SetCommandAndParameters(@" + DELETE FROM ""LogData"" + WHERE ""Timestamp"" < @Timestamp + ") + .SetParameterValue("@Timestamp", t) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + await cmd.SetCommandAndParameters(@" + DELETE FROM ""RemoteOperation"" + WHERE ""Timestamp"" < @Timestamp + ") + .SetParameterValue("@Timestamp", t) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } - public void PurgeDeletedVolumes(DateTime threshold) + /// + /// Purges deleted remote volumes that have not been modified since the specified threshold. + /// + /// The threshold date and time; all deleted remote volumes older than this will be purged. + /// A task that completes when the purge operation is finished. + public async Task PurgeDeletedVolumes(DateTime threshold, CancellationToken token) { - using (var tr = m_connection.BeginTransactionSafe()) - using (var cmd = m_connection.CreateCommand(tr)) - { - m_removedeletedremotevolumeCommand.SetParameterValue("@Now", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(threshold)) - .ExecuteNonQuery(tr); - tr.Commit(); - } + await m_removedeletedremotevolumeCommand + .SetTransaction(m_rtr) + .SetParameterValue("@Now", Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(threshold)) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } /// /// Returns the ID of an empty metadata blockset. If no empty blockset is found, it returns the ID of the smallest blockset that is not in the given block volume IDs. /// If no such blockset is found, it returns -1. /// - /// The volume ids to ignore when searching for a suitable metadata block - /// The hash of the empty blockset - /// The size of the empty blockset - /// The transaction to use for the query - /// The ID of the empty metadata blockset, or -1 if no suitable blockset is found - public long GetEmptyMetadataBlocksetId(IEnumerable blockVolumeIds, string emptyHash, long emptyHashSize, IDbTransaction? transaction) + /// The volume ids to ignore when searching for a suitable metadata block. + /// The hash of the empty blockset. + /// The size of the empty blockset. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains the ID of the empty metadata blockset, or -1 if no suitable blockset is found + public async Task GetEmptyMetadataBlocksetId(IEnumerable blockVolumeIds, string emptyHash, long emptyHashSize, CancellationToken token) { - using var cmd = Connection.CreateCommand(transaction, @"SELECT ""ID"" FROM ""Blockset"" WHERE ""FullHash"" = @EmptyHash AND ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))") - .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) - .SetParameterValue("@EmptyHash", emptyHash) - .SetParameterValue("@EmptyHashSize", emptyHashSize); + await using var cmd = Connection.CreateCommand(@" + SELECT ""ID"" + FROM ""Blockset"" + WHERE + ""FullHash"" = @EmptyHash + AND ""Length"" == @EmptyHashSize + AND ""ID"" NOT IN ( + SELECT ""BlocksetID"" + FROM + ""BlocksetEntry"", + ""Block"" + WHERE + ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) + ) + ") + .SetTransaction(m_rtr) + .ExpandInClauseParameterMssqlite("@BlockVolumeIds", blockVolumeIds) + .SetParameterValue("@EmptyHash", emptyHash) + .SetParameterValue("@EmptyHashSize", emptyHashSize); - var res = cmd.ExecuteScalarInt64(-1); + var res = await cmd.ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); // No empty block found, try to find a zero-length block instead if (res < 0 && emptyHashSize != 0) - res = cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" == @EmptyHashSize AND ""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds))") - .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) + res = await cmd.SetCommandAndParameters(@" + SELECT ""ID"" + FROM ""Blockset"" + WHERE + ""Length"" == @EmptyHashSize + AND ""ID"" NOT IN ( + SELECT ""BlocksetID"" + FROM + ""BlocksetEntry"", + ""Block"" + WHERE + ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) + ) + ") + .ExpandInClauseParameterMssqlite("@BlockVolumeIds", blockVolumeIds) .SetParameterValue("@EmptyHashSize", 0) - .ExecuteScalarInt64(-1); + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); // No empty block found, pick the smallest one if (res < 0) - res = cmd.SetCommandAndParameters(@"SELECT ""Blockset"".""ID"" FROM ""BlocksetEntry"", ""Blockset"", ""Metadataset"", ""Block"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) AND ""Blockset"".""Length"" > 0 ORDER BY ""Blockset"".""Length"" ASC LIMIT 1") - .ExpandInClauseParameter("@BlockVolumeIds", blockVolumeIds) - .ExecuteScalarInt64(-1); + res = await cmd.SetCommandAndParameters(@" + SELECT ""Blockset"".""ID"" + FROM + ""BlocksetEntry"", + ""Blockset"", + ""Metadataset"", + ""Block"" + WHERE + ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" + AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" + AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" + AND ""Block"".""VolumeID"" NOT IN (@BlockVolumeIds) + AND ""Blockset"".""Length"" > 0 + ORDER BY ""Blockset"".""Length"" ASC + LIMIT 1 + ") + .ExpandInClauseParameterMssqlite("@BlockVolumeIds", blockVolumeIds) + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); return res; } public virtual void Dispose() + { + this.DisposeAsync().AsTask().Await(); + } + + public virtual async ValueTask DisposeAsync() { if (IsDisposed) return; - DisposeAllFields(this, false); + DisposeAllFields(this, false); if (ShouldCloseConnection && m_connection != null) { + await m_rtr.DisposeAsync().ConfigureAwait(false); if (m_connection.State == ConnectionState.Open && !m_hasExecutedVacuum) { - using (var transaction = m_connection.BeginTransactionSafe()) - using (var command = m_connection.CreateCommand(transaction)) + await using (var command = m_connection.CreateCommand()) + await using (var tr = m_connection.BeginTransaction()) { // SQLite recommends that PRAGMA optimize is run just before closing each database connection. - command.ExecuteNonQuery("PRAGMA optimize"); + await command + .SetTransaction(tr) + .ExecuteNonQueryAsync("PRAGMA optimize", default) + .ConfigureAwait(false); try { - transaction.Commit(); + await tr.CommitAsync().ConfigureAwait(false); } catch (Exception ex) { @@ -1797,18 +3198,18 @@ ORDER BY ""A"".""BlocksetID"", ""B"".""Index"" m_connection.Close(); } - m_connection.Dispose(); + await m_connection.DisposeAsync().ConfigureAwait(false); } IsDisposed = true; } /// - /// Disposes all fields of a certain type, in the instance and its bases + /// Disposes all fields of a certain type, in the instance and its bases. /// - /// The type of fields to find - /// The item to dispose - /// True if an aggregate exception should be thrown, or false if exceptions are silently captured + /// The type of fields to find. + /// The item to dispose. + /// True if an aggregate exception should be thrown, or false if exceptions are silently captured. public static void DisposeAllFields(object item, bool throwExceptions) where T : IDisposable { @@ -1824,8 +3225,14 @@ ORDER BY ""A"".""BlocksetID"", ""B"".""Index"" var fields = typechain.SelectMany(x => - x.GetFields(System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.FlattenHierarchy) - ).Distinct().Where(x => x.FieldType.IsAssignableFrom(typeof(T))); + x.GetFields( + System.Reflection.BindingFlags.NonPublic | + System.Reflection.BindingFlags.Public | + System.Reflection.BindingFlags.Instance | + System.Reflection.BindingFlags.FlattenHierarchy + ) + ).Distinct() + .Where(x => x.FieldType.IsAssignableFrom(typeof(T))); foreach (var p in fields) try @@ -1844,7 +3251,13 @@ ORDER BY ""A"".""BlocksetID"", ""B"".""Index"" throw new AggregateException(exceptions); } - public void WriteResults(IBasicResults result) + /// + /// Writes the results of a basic operation to the log. + /// + /// The results to write. + /// Cancellation token to monitor for cancellation requests. + /// A task that completes when the results have been written. + public async Task WriteResults(IBasicResults result, CancellationToken token) { if (IsDisposed) return; @@ -1853,36 +3266,37 @@ ORDER BY ""A"".""BlocksetID"", ""B"".""Index"" { if (result is BasicResults basicResults) { - basicResults.FlushLog(this); + await basicResults.FlushLog(this, token).ConfigureAwait(false); if (basicResults.EndTime.Ticks == 0) basicResults.EndTime = DateTime.UtcNow; } var serializer = new JsonFormatSerializer(); - LogMessage("Result", + await LogMessage("Result", serializer.SerializeResults(result), null, - null - ); + token + ) + .ConfigureAwait(false); } } /// - /// The current index into the path prefix buffer + /// The current index into the path prefix buffer. /// private int m_pathPrefixIndex = 0; /// - /// The path prefix lookup list + /// The path prefix lookup list. /// private readonly KeyValuePair[] m_pathPrefixLookup = new KeyValuePair[5]; /// - /// Gets the path prefix ID, optionally creating it in the process. + /// Gets the path prefix ID, optionally creating it in the process.. /// - /// The path prefix ID. /// The path to get the prefix for. - /// The transaction to use for insertion, or null for no transaction - public long GetOrCreatePathPrefix(string prefix, IDbTransaction? transaction) + /// Cancellation token to monitor for cancellation requests. + /// A task that when awaited returns the path prefix ID. + public async Task GetOrCreatePathPrefix(string prefix, CancellationToken token) { // Ring-buffer style lookup for (var i = 0; i < m_pathPrefixLookup.Length; i++) @@ -1892,13 +3306,20 @@ ORDER BY ""A"".""BlocksetID"", ""B"".""Index"" return m_pathPrefixLookup[ix].Value; } - m_findpathprefixCommand.Transaction = transaction; - var id = m_findpathprefixCommand.SetParameterValue("@Prefix", prefix) - .ExecuteScalarInt64(transaction); + var id = await m_findpathprefixCommand + .SetTransaction(m_rtr) + .SetParameterValue("@Prefix", prefix) + .ExecuteScalarInt64Async(token) + .ConfigureAwait(false); if (id < 0) - id = m_insertpathprefixCommand.SetParameterValue("@Prefix", prefix) - .ExecuteScalarInt64(transaction); + { + id = await m_insertpathprefixCommand + .SetTransaction(m_rtr) + .SetParameterValue("@Prefix", prefix) + .ExecuteScalarInt64Async(token) + .ConfigureAwait(false); + } m_pathPrefixIndex = (m_pathPrefixIndex + 1) % m_pathPrefixLookup.Length; m_pathPrefixLookup[m_pathPrefixIndex] = new KeyValuePair(prefix, id); @@ -1907,18 +3328,18 @@ ORDER BY ""A"".""BlocksetID"", ""B"".""Index"" } /// - /// The path separators on this system + /// The path separators on this system. /// - private static readonly char[] _pathseparators = new char[] { + private static readonly char[] _pathseparators = [ Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar, - }; + ]; /// - /// Helper method that splits a path on the last path separator + /// Helper method that splits a path on the last path separator. /// - /// The prefix and name. /// The path to split. + /// The prefix and name. public static KeyValuePair SplitIntoPrefixAndName(string path) { if (string.IsNullOrEmpty(path)) @@ -1933,11 +3354,18 @@ ORDER BY ""A"".""BlocksetID"", ""B"".""Index"" } /// - /// Defines the backups types + /// Defines the backups types. /// public static class BackupType { + /// + /// Indicates a partial backup. + /// public const int PARTIAL_BACKUP = 0; + /// + /// Indicates a full backup. + /// public const int FULL_BACKUP = 1; } + } diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index 20a2cd912..9737f71ab 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -1,113 +1,296 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable using System; using System.Collections.Generic; -using System.Data; using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; using Duplicati.Library.Interface; +using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database { + /// + /// A local database for deleting filesets and blocks. + /// internal class LocalDeleteDatabase : LocalDatabase { /// - /// The tag used for logging + /// The tag used for logging. /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(); /// - /// Flag for toggling temporary tables; set to empty string if debugging + /// Flag for toggling temporary tables; set to empty string if debugging. /// private const string TEMPORARY = "TEMPORARY"; - private const string REGISTER_COMMAND = @"INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") SELECT ""ID"", @VolumeId FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size "; + /// + /// SQL command to register a duplicate block. + /// + private const string REGISTER_COMMAND = @" + INSERT OR IGNORE INTO ""DuplicateBlock"" ( + ""BlockID"", + ""VolumeID"" + ) + SELECT + ""ID"", + @VolumeId + FROM ""Block"" + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size + "; - private IDbCommand m_registerDuplicateBlockCommand; + /// + /// The command to register a duplicate block. + /// + private SqliteCommand m_registerDuplicateBlockCommand = null!; - public LocalDeleteDatabase(string path, string operation, long pagecachesize) - : base(path, operation, true, pagecachesize) + /// + /// Creates a new instance of the class. + /// + /// The path to the database file. + /// The operation name. + /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains a new instance of . + public static async Task CreateAsync(string path, string operation, LocalDeleteDatabase? dbnew, CancellationToken token) { - m_registerDuplicateBlockCommand = m_connection.CreateCommand(REGISTER_COMMAND); + dbnew ??= new LocalDeleteDatabase(); + + dbnew = (LocalDeleteDatabase) + await CreateLocalDatabaseAsync(path, operation, true, dbnew, token) + .ConfigureAwait(false); + + dbnew.m_registerDuplicateBlockCommand = + await dbnew.Connection.CreateCommandAsync(REGISTER_COMMAND, token) + .ConfigureAwait(false); + + return dbnew; } - public LocalDeleteDatabase(LocalDatabase db) - : base(db) + /// + /// Creates a new instance of the class with a parent database. + /// + /// The parent database to use. + /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains a new instance of . + public static async Task CreateAsync(LocalDatabase dbparent, LocalDeleteDatabase? dbnew, CancellationToken token) { - m_registerDuplicateBlockCommand = m_connection.CreateCommand(REGISTER_COMMAND); + dbnew ??= new LocalDeleteDatabase(); + + dbnew = (LocalDeleteDatabase) + await CreateLocalDatabaseAsync(dbparent, dbnew, token) + .ConfigureAwait(false); + + dbnew.m_registerDuplicateBlockCommand = + await dbnew.Connection.CreateCommandAsync(REGISTER_COMMAND, token) + .ConfigureAwait(false); + + return dbnew; } /// /// Drops all entries related to operations listed in the table. /// - /// The fileset entries to delete - /// The transaction to execute the commands in - /// A list of filesets to delete - public IEnumerable> DropFilesetsFromTable(DateTime[] toDelete, IDbTransaction transaction) + /// The fileset entries to delete. + /// A cancellation token to cancel the operation. + /// An async enumerable of key-value pairs, where the key is the fileset name and the value is the size of the fileset. + public async IAsyncEnumerable> DropFilesetsFromTable(DateTime[] toDelete, [EnumeratorCancellation] CancellationToken token) { - using (var cmd = m_connection.CreateCommand(transaction)) - { - var deleted = 0; + await using var cmd = m_connection.CreateCommand(m_rtr); + var deleted = 0; - using (var tempTable = new TemporaryDbValueList(m_connection, transaction, toDelete.Select(Library.Utility.Utility.NormalizeDateTimeToEpochSeconds))) - deleted += cmd.SetCommandAndParameters(@"DELETE FROM ""Fileset"" WHERE ""Timestamp"" IN (@Timestamps) ") - .ExpandInClauseParameter("@Timestamps", tempTable) - .ExecuteNonQuery(); + await using (var tempTable = await TemporaryDbValueList.CreateAsync(this, toDelete.Select(Library.Utility.Utility.NormalizeDateTimeToEpochSeconds), token).ConfigureAwait(false)) + deleted += await ( + await cmd.SetCommandAndParameters(@" + DELETE FROM ""Fileset"" + WHERE ""Timestamp"" IN (@Timestamps) + ") + .ExpandInClauseParameterMssqliteAsync("@Timestamps", tempTable, token) + .ConfigureAwait(false) + ) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); - if (deleted != toDelete.Length) - throw new Exception($"Unexpected number of deleted filesets {deleted} vs {toDelete.Length}"); + if (deleted != toDelete.Length) + throw new Exception($"Unexpected number of deleted filesets {deleted} vs {toDelete.Length}"); - //Then we delete anything that is no longer being referenced - cmd.ExecuteNonQuery(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" NOT IN (SELECT DISTINCT ""ID"" FROM ""Fileset"")"); - cmd.ExecuteNonQuery(@"DELETE FROM ""ChangeJournalData"" WHERE ""FilesetID"" NOT IN (SELECT DISTINCT ""ID"" FROM ""Fileset"")"); - cmd.ExecuteNonQuery(@"DELETE FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FileID"" FROM ""FilesetEntry"") "); - cmd.ExecuteNonQuery(@"DELETE FROM ""Metadataset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""MetadataID"" FROM ""FileLookup"") "); - cmd.ExecuteNonQuery(@"DELETE FROM ""Blockset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""BlocksetID"" FROM ""FileLookup"" UNION SELECT DISTINCT ""BlocksetID"" FROM ""Metadataset"") "); - cmd.ExecuteNonQuery(@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" NOT IN (SELECT DISTINCT ""ID"" FROM ""Blockset"") "); - cmd.ExecuteNonQuery(@"DELETE FROM ""BlocklistHash"" WHERE ""BlocksetID"" NOT IN (SELECT DISTINCT ""ID"" FROM ""Blockset"") "); + //Then we delete anything that is no longer being referenced + await cmd.ExecuteNonQueryAsync(@" + DELETE FROM ""FilesetEntry"" + WHERE ""FilesetID"" NOT IN ( + SELECT DISTINCT ""ID"" + FROM ""Fileset"" + ) + ", token) + .ConfigureAwait(false); - //We save the block info for the remote files, before we delete it - cmd.ExecuteNonQuery(@"INSERT INTO ""DeletedBlock"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""Block"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""BlockID"" AS ""BlockID"" FROM ""BlocksetEntry"" UNION SELECT DISTINCT ""ID"" FROM ""Block"", ""BlocklistHash"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"") "); - cmd.ExecuteNonQuery(@"DELETE FROM ""Block"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""BlockID"" FROM ""BlocksetEntry"" UNION SELECT DISTINCT ""ID"" FROM ""Block"", ""BlocklistHash"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"") "); + await cmd.ExecuteNonQueryAsync(@" + DELETE FROM ""ChangeJournalData"" + WHERE ""FilesetID"" NOT IN ( + SELECT DISTINCT ""ID"" + FROM ""Fileset"" + ) + ", token) + .ConfigureAwait(false); - //Find all remote filesets that are no longer required, and mark them as deleting - var updated = cmd.SetCommandAndParameters(@"UPDATE ""RemoteVolume"" SET ""State"" = @NewState WHERE ""Type"" = @CurrentType AND ""State"" IN (@AllowedStates) AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"") ") - .SetParameterValue("@NewState", RemoteVolumeState.Deleting.ToString()) - .SetParameterValue("@CurrentType", RemoteVolumeType.Files.ToString()) - .ExpandInClauseParameter("@AllowedStates", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString(), RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Deleting.ToString()]) - .ExecuteNonQuery(); + await cmd.ExecuteNonQueryAsync(@" + DELETE FROM ""FileLookup"" + WHERE ""ID"" NOT IN ( + SELECT DISTINCT ""FileID"" + FROM ""FilesetEntry"" + ) + ", token) + .ConfigureAwait(false); - if (deleted != updated) - throw new Exception($"Unexpected number of remote volumes marked as deleted. Found {deleted} filesets, but {updated} volumes"); + await cmd.ExecuteNonQueryAsync(@" + DELETE FROM ""Metadataset"" + WHERE ""ID"" NOT IN ( + SELECT DISTINCT ""MetadataID"" + FROM ""FileLookup"" + ) + ", token) + .ConfigureAwait(false); - cmd.SetCommandAndParameters(@"SELECT ""Name"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" = @State ") - .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .SetParameterValue("@State", RemoteVolumeState.Deleting.ToString()); + await cmd.ExecuteNonQueryAsync(@" + DELETE FROM ""Blockset"" + WHERE ""ID"" NOT IN ( + SELECT DISTINCT ""BlocksetID"" + FROM ""FileLookup"" + UNION + SELECT DISTINCT ""BlocksetID"" + FROM ""Metadataset"" + ) + ", token) + .ConfigureAwait(false); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); - } + await cmd.ExecuteNonQueryAsync(@" + DELETE FROM ""BlocksetEntry"" + WHERE ""BlocksetID"" NOT IN ( + SELECT DISTINCT ""ID"" + FROM ""Blockset"" + ) + ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync(@" + DELETE FROM ""BlocklistHash"" + WHERE ""BlocksetID"" NOT IN ( + SELECT DISTINCT ""ID"" + FROM ""Blockset"" + ) + ", token) + .ConfigureAwait(false); + + //We save the block info for the remote files, before we delete it + await cmd.ExecuteNonQueryAsync(@" + INSERT INTO ""DeletedBlock"" ( + ""Hash"", + ""Size"", + ""VolumeID"" + ) + SELECT + ""Hash"", + ""Size"", + ""VolumeID"" + FROM ""Block"" + WHERE ""ID"" NOT IN ( + SELECT DISTINCT ""BlockID"" AS ""BlockID"" + FROM ""BlocksetEntry"" + UNION + SELECT DISTINCT ""ID"" + FROM + ""Block"", + ""BlocklistHash"" + WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" + ) + ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync(@" + DELETE FROM ""Block"" + WHERE ""ID"" NOT IN ( + SELECT DISTINCT ""BlockID"" + FROM ""BlocksetEntry"" + UNION + SELECT DISTINCT ""ID"" + FROM + ""Block"", + ""BlocklistHash"" + WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" + ) + ", token) + .ConfigureAwait(false); + + //Find all remote filesets that are no longer required, and mark them as deleting + var updated = await cmd.SetCommandAndParameters(@" + UPDATE ""RemoteVolume"" + SET ""State"" = @NewState + WHERE + ""Type"" = @CurrentType + AND ""State"" IN (@AllowedStates) + AND ""ID"" NOT IN ( + SELECT ""VolumeID"" + FROM ""Fileset"" + ) + ") + .SetParameterValue("@NewState", RemoteVolumeState.Deleting.ToString()) + .SetParameterValue("@CurrentType", RemoteVolumeType.Files.ToString()) + .ExpandInClauseParameterMssqlite("@AllowedStates", [ + RemoteVolumeState.Uploaded.ToString(), + RemoteVolumeState.Verified.ToString(), + RemoteVolumeState.Temporary.ToString(), + RemoteVolumeState.Deleting.ToString() + ]) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + if (deleted != updated) + throw new Exception($"Unexpected number of remote volumes marked as deleted. Found {deleted} filesets, but {updated} volumes"); + + cmd.SetCommandAndParameters(@" + SELECT + ""Name"", + ""Size"" + FROM ""RemoteVolume"" + WHERE + ""Type"" = @Type + AND ""State"" = @State + ") + .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) + .SetParameterValue("@State", RemoteVolumeState.Deleting.ToString()); + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + yield return new KeyValuePair( + rd.ConvertValueToString(0) ?? "", + rd.ConvertValueToInt64(1) + ); } /// @@ -115,37 +298,66 @@ namespace Duplicati.Library.Main.Database /// exposed to the user. This is in contrast to other cases where the Version is the ID in the /// Fileset table. /// - internal IEnumerable FilesetsWithBackupVersion + /// A cancellation token to cancel the operation. + /// An async enumerable of IListResultFileset. + internal async IAsyncEnumerable FilesetsWithBackupVersion([EnumeratorCancellation] CancellationToken token) { - get - { - var filesets = new List(); - using (var cmd = m_connection.CreateCommand()) - { - // We can also use the ROW_NUMBER() window function to generate the backup versions, - // but this requires at least SQLite 3.25, which is not available in some common - // distributions (e.g., Debian) currently. - using (IDataReader reader = cmd.ExecuteReader(@"SELECT ""IsFullBackup"", ""Timestamp"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC")) - { - int version = 0; - while (reader.Read()) - { - filesets.Add(new ListResultFileset(version++, reader.GetInt32(0), ParseFromEpochSeconds(reader.ConvertValueToInt64(1)).ToLocalTime(), -1L, -1L)); - } - } - } + await using var cmd = m_connection.CreateCommand(m_rtr); + // TODO check if this is still the case? (shouldn't be with new sqlite driver): + // We can also use the ROW_NUMBER() window function to generate the backup versions, + // but this requires at least SQLite 3.25, which is not available in some common + // distributions (e.g., Debian) currently. + cmd.SetCommandAndParameters(@" + SELECT + ""IsFullBackup"", + ""Timestamp"" + FROM ""Fileset"" + ORDER BY ""Timestamp"" DESC + "); - return filesets; + await using var reader = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + int version = 0; + while (await reader.ReadAsync(token).ConfigureAwait(false)) + { + yield return new ListResultFileset( + version++, + reader.GetInt32(0), + ParseFromEpochSeconds(reader.ConvertValueToInt64(1)).ToLocalTime(), + -1L, + -1L + ); } } + /// + /// Represents the usage of a volume in the database. + /// private struct VolumeUsage { + /// + /// The name of the volume. + /// public readonly string Name; + /// + /// The size of the data stored in the volume. + /// public readonly long DataSize; + /// + /// The size of the data that is no longer needed in the volume. + /// public readonly long WastedSize; + /// + /// The size of the data that is compressed in the volume. + /// public readonly long CompressedSize; + /// + /// Initializes a new instance of the struct. + /// + /// The name of the volume. + /// The size of the data stored in the volume. + /// The size of the data that is no longer needed in the volume. + /// The size of the data that is compressed in the volume. public VolumeUsage(string name, long datasize, long wastedsize, long compressedsize) { Name = name; @@ -160,71 +372,275 @@ namespace Duplicati.Library.Main.Database /// and the number of bytes no longer needed in each volume. /// The sizes are the uncompressed values. /// - /// A list of tuples with name, datasize, wastedbytes. - private IEnumerable GetWastedSpaceReport(IDbTransaction transaction) + /// A cancellation token to cancel the operation. + /// + /// An asynchronous enumerable of , holding the name of the volume, + /// the size of the data stored in the volume, the size of the data that is no longer needed in the volume, + /// and the size of the data that is compressed in the volume. + /// + private async IAsyncEnumerable GetWastedSpaceReport([EnumeratorCancellation] CancellationToken token) { var tmptablename = "UsageReport-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var usedBlocks = @"SELECT SUM(""Block"".""Size"") AS ""ActiveSize"", ""Block"".""VolumeID"" AS ""VolumeID"" FROM ""Block"", ""Remotevolume"" WHERE ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Block"".""ID"" NOT IN (SELECT ""Block"".""ID"" FROM ""Block"",""DeletedBlock"" WHERE ""Block"".""Hash"" = ""DeletedBlock"".""Hash"" AND ""Block"".""Size"" = ""DeletedBlock"".""Size"" AND ""Block"".""VolumeID"" = ""DeletedBlock"".""VolumeID"") GROUP BY ""Block"".""VolumeID"" "; - var lastmodifiedFile = @"SELECT ""Block"".""VolumeID"" AS ""VolumeID"", ""Fileset"".""Timestamp"" AS ""Sorttime"" FROM ""Fileset"", ""FilesetEntry"", ""FileLookup"", ""BlocksetEntry"", ""Block"" WHERE ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" "; - var lastmodifiedMetadata = @"SELECT ""Block"".""VolumeID"" AS ""VolumeID"", ""Fileset"".""Timestamp"" AS ""Sorttime"" FROM ""Fileset"", ""FilesetEntry"", ""FileLookup"", ""BlocksetEntry"", ""Block"", ""Metadataset"" WHERE ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" AND ""FileLookup"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" "; - var scantime = @"SELECT ""VolumeID"" AS ""VolumeID"", MIN(""Sorttime"") AS ""Sorttime"" FROM (" + lastmodifiedFile + @" UNION " + lastmodifiedMetadata + @") GROUP BY ""VolumeID"" "; - var active = @"SELECT ""A"".""ActiveSize"" AS ""ActiveSize"", 0 AS ""InactiveSize"", ""A"".""VolumeID"" AS ""VolumeID"", CASE WHEN ""B"".""Sorttime"" IS NULL THEN 0 ELSE ""B"".""Sorttime"" END AS ""Sorttime"" FROM (" + usedBlocks + @") A LEFT OUTER JOIN (" + scantime + @") B ON ""B"".""VolumeID"" = ""A"".""VolumeID"" "; + var usedBlocks = @" + SELECT + SUM(""Block"".""Size"") AS ""ActiveSize"", + ""Block"".""VolumeID"" AS ""VolumeID"" FROM ""Block"", + ""Remotevolume"" + WHERE + ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" + AND ""Block"".""ID"" NOT IN ( + SELECT ""Block"".""ID"" + FROM + ""Block"", + ""DeletedBlock"" + WHERE + ""Block"".""Hash"" = ""DeletedBlock"".""Hash"" + AND ""Block"".""Size"" = ""DeletedBlock"".""Size"" + AND ""Block"".""VolumeID"" = ""DeletedBlock"".""VolumeID"" + ) + GROUP BY ""Block"".""VolumeID"" + "; - var inactive = @"SELECT 0 AS ""ActiveSize"", SUM(""Size"") AS ""InactiveSize"", ""VolumeID"" AS ""VolumeID"", 0 AS ""SortScantime"" FROM ""DeletedBlock"" GROUP BY ""VolumeID"" "; - var empty = @"SELECT 0 AS ""ActiveSize"", 0 AS ""InactiveSize"", ""Remotevolume"".""ID"" AS ""VolumeID"", 0 AS ""SortScantime"" FROM ""Remotevolume"" WHERE ""Remotevolume"".""Type"" = @Type AND ""Remotevolume"".""State"" IN (@AllowedStates) AND ""Remotevolume"".""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Block"") "; + var lastmodifiedFile = @" + SELECT + ""Block"".""VolumeID"" AS ""VolumeID"", + ""Fileset"".""Timestamp"" AS ""Sorttime"" + FROM + ""Fileset"", + ""FilesetEntry"", + ""FileLookup"", + ""BlocksetEntry"", + ""Block"" + WHERE + ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" + AND ""FileLookup"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" + "; - var combined = active + " UNION " + inactive + " UNION " + empty; - var collected = @"SELECT ""VolumeID"" AS ""VolumeID"", SUM(""ActiveSize"") AS ""ActiveSize"", SUM(""InactiveSize"") AS ""InactiveSize"", MAX(""Sorttime"") AS ""Sorttime"" FROM (" + combined + @") GROUP BY ""VolumeID"" "; - var createtable = FormatInvariant($"{@$"CREATE {TEMPORARY} TABLE ""{tmptablename}"" AS "}{collected}"); + var lastmodifiedMetadata = @" + SELECT + ""Block"".""VolumeID"" AS ""VolumeID"", + ""Fileset"".""Timestamp"" AS ""Sorttime"" + FROM + ""Fileset"", + ""FilesetEntry"", + ""FileLookup"", + ""BlocksetEntry"", + ""Block"", + ""Metadataset"" + WHERE + ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" + AND ""FileLookup"".""MetadataID"" = ""Metadataset"".""ID"" + AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" + "; - using (var cmd = m_connection.CreateCommand(transaction)) + var scantime = @$" + SELECT + ""VolumeID"" AS ""VolumeID"", + MIN(""Sorttime"") AS ""Sorttime"" + FROM ( + {lastmodifiedFile} + UNION {lastmodifiedMetadata} + ) + GROUP BY ""VolumeID"" + "; + + var active = @$" + SELECT + ""A"".""ActiveSize"" AS ""ActiveSize"", + 0 AS ""InactiveSize"", + ""A"".""VolumeID"" AS ""VolumeID"", + CASE + WHEN ""B"".""Sorttime"" IS NULL + THEN 0 + ELSE ""B"".""Sorttime"" + END AS ""Sorttime"" + FROM ({usedBlocks}) ""A"" + LEFT OUTER JOIN ({scantime}) ""B"" + ON ""B"".""VolumeID"" = ""A"".""VolumeID"" + "; + + var inactive = @" + SELECT + 0 AS ""ActiveSize"", + SUM(""Size"") AS ""InactiveSize"", + ""VolumeID"" AS ""VolumeID"", + 0 AS ""SortScantime"" + FROM ""DeletedBlock"" + GROUP BY ""VolumeID"" + "; + + var empty = @" + SELECT + 0 AS ""ActiveSize"", + 0 AS ""InactiveSize"", + ""Remotevolume"".""ID"" AS ""VolumeID"", + 0 AS ""SortScantime"" + FROM ""Remotevolume"" + WHERE + ""Remotevolume"".""Type"" = @Type + AND ""Remotevolume"".""State"" IN (@AllowedStates) + AND ""Remotevolume"".""ID"" NOT IN ( + SELECT ""VolumeID"" + FROM ""Block"" + ) + "; + + var combined = $"{active} UNION {inactive} UNION {empty}"; + var collected = @$" + SELECT + ""VolumeID"" AS ""VolumeID"", + SUM(""ActiveSize"") AS ""ActiveSize"", + SUM(""InactiveSize"") AS ""InactiveSize"", + MAX(""Sorttime"") AS ""Sorttime"" + FROM ({combined}) + GROUP BY ""VolumeID"" + "; + + var createtable = $"{@$"CREATE {TEMPORARY} TABLE ""{tmptablename}"" AS "}{collected}"; + + await using var cmd = m_connection.CreateCommand(m_rtr); + try + { + await cmd + .SetCommandAndParameters(createtable) + .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) + .ExpandInClauseParameterMssqlite("@AllowedStates", [ + RemoteVolumeState.Uploaded.ToString(), + RemoteVolumeState.Verified.ToString() + ]) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + cmd.SetCommandAndParameters($@" + SELECT + ""A"".""Name"", + ""B"".""ActiveSize"", + ""B"".""InactiveSize"", + ""A"".""Size"" + FROM + ""Remotevolume"" ""A"", + ""{tmptablename}"" ""B"" + WHERE ""A"".""ID"" = ""B"".""VolumeID"" + ORDER BY ""B"".""Sorttime"" ASC + "); + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + yield return new VolumeUsage( + rd.ConvertValueToString(0) ?? "", + rd.ConvertValueToInt64(1, 0) + rd.ConvertValueToInt64(2, 0), + rd.ConvertValueToInt64(2, 0), + rd.ConvertValueToInt64(3, 0) + ); + } + finally { try { - cmd.SetCommandAndParameters(createtable) - .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) - .ExpandInClauseParameter("@AllowedStates", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]) - .ExecuteNonQuery(); - - using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT ""A"".""Name"", ""B"".""ActiveSize"", ""B"".""InactiveSize"", ""A"".""Size"" FROM ""Remotevolume"" A, ""{tmptablename}"" B WHERE ""A"".""ID"" = ""B"".""VolumeID"" ORDER BY ""B"".""Sorttime"" ASC "))) - while (rd.Read()) - yield return new VolumeUsage(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1, 0) + rd.ConvertValueToInt64(2, 0), rd.ConvertValueToInt64(2, 0), rd.ConvertValueToInt64(3, 0)); - } - finally - { - try { cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tmptablename}"" ")); } - catch { } + await cmd + .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmptablename}"" ", token) + .ConfigureAwait(false); } + catch { } } } + /// + /// Generates a report on the volumes that can be reclaimed or compacted. + /// public interface ICompactReport { + /// + /// Gets the volumes that can be deleted. + /// IEnumerable DeleteableVolumes { get; } + /// + /// Gets the volumes that can be compacted. + /// IEnumerable CompactableVolumes { get; } + /// + /// Gets the report on whether reclamation or compaction should be performed. + /// bool ShouldReclaim { get; } + /// + /// Gets the report on whether compaction should be performed. + /// bool ShouldCompact { get; } + /// + /// Emits the report data to the log. + /// void ReportCompactData(); } + /// + /// A report on the volumes that can be reclaimed or compacted. + /// private class CompactReport : ICompactReport { + /// + /// The report to emit to the log. + /// private readonly IEnumerable m_report; + /// + /// The volumes that can be fully deleted. + /// private readonly IEnumerable m_cleandelete; + /// + /// The volumes that have wasted space. + /// private readonly IEnumerable m_wastevolumes; + /// + /// The volumes that are smaller than the specified size. + /// private readonly IEnumerable m_smallvolumes; + /// + /// The count of volumes that can be fully deleted. + /// private readonly long m_deletablevolumes; + /// + /// The total size of wasted space across all volumes. + /// private readonly long m_wastedspace; + /// + /// The total size of small volumes. + /// private readonly long m_smallspace; + /// + /// The total size of all data across all volumes. + /// private readonly long m_fullsize; + /// + /// The count of small volumes. + /// private readonly long m_smallvolumecount; + /// + /// The threshold for wasted space percentage to trigger compaction. + /// private readonly long m_wastethreshold; + /// + /// The size of the volume to trigger compaction. + /// private readonly long m_volsize; + /// + /// The maximum number of small files to trigger compaction. + /// private readonly long m_maxsmallfilecount; + /// + /// Initializes a new instance of the class. + /// + /// The size of the volume to trigger compaction. + /// The threshold for wasted space percentage to trigger compaction. + /// The size of small files to trigger compaction. + /// The maximum number of small files to trigger compaction. + /// The report data containing volume usage information. public CompactReport(long volsize, long wastethreshold, long smallfilesize, long maxsmallfilecount, IEnumerable report) { m_report = report; @@ -245,6 +661,9 @@ namespace Duplicati.Library.Main.Database m_smallvolumecount = m_smallvolumes.Count(); } + /// + /// Emits the report data to the log. + /// public void ReportCompactData() { var wastepercentage = ((m_wastedspace / (float)m_fullsize) * 100); @@ -264,6 +683,9 @@ namespace Duplicati.Library.Main.Database Logging.Log.WriteInformationMessage(LOGTAG, "CompactReason", "Compacting not required"); } + /// + /// Gets a value indicating whether reclamation should be performed. + /// public bool ShouldReclaim { get @@ -272,6 +694,9 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Gets a value indicating whether compaction should be performed. + /// public bool ShouldCompact { get @@ -280,11 +705,17 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Gets the volumes that can be deleted. + /// public IEnumerable DeleteableVolumes { get { return from n in m_cleandelete select n.Name; } } + /// + /// Gets the volumes that can be compacted. + /// public IEnumerable CompactableVolumes { get @@ -297,41 +728,104 @@ namespace Duplicati.Library.Main.Database } } - public ICompactReport GetCompactReport(long volsize, long wastethreshold, long smallfilesize, long maxsmallfilecount, IDbTransaction transaction) + /// + /// Gets a compact report on the volumes that can be reclaimed or compacted. + /// + /// The size of the volume to trigger compaction. + /// The threshold for wasted space percentage to trigger compaction. + /// The size of small files to trigger compaction. + /// The maximum number of small files to trigger compaction. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains an instance of . + public async Task GetCompactReport(long volsize, long wastethreshold, long smallfilesize, long maxsmallfilecount, CancellationToken token) { - return new CompactReport(volsize, wastethreshold, smallfilesize, maxsmallfilecount, GetWastedSpaceReport(transaction).ToList()); + return new CompactReport( + volsize, + wastethreshold, + smallfilesize, + maxsmallfilecount, + await GetWastedSpaceReport(token) + .ToListAsync(cancellationToken: token) + .ConfigureAwait(false) + ); } - + /// + /// Registers a block as duplicated to a new volume. + /// This is used when a block is moved to a new volume, and we want to keep track of it. + /// public interface IBlockQuery : IDisposable { /// - /// Checks if a block is in use. If volumeId is not -1, check specific volume + /// Checks if a block is in use. If volumeId is not -1, check specific volume. /// - /// The hash of the block - /// The size of the block - /// The volume ID to check, or -1 to check all volumes - /// The transaction to execute the command in - /// True if the block is in use - bool UseBlock(string hash, long size, long volumeId, IDbTransaction? transaction); + /// The hash of the block. + /// The size of the block. + /// The volume ID to check, or -1 to check all volumes. + /// A cancellation token to cancel the operation. + /// A task that when awaited returns true if the block is in use, false otherwise. + Task UseBlock(string hash, long size, long volumeId, CancellationToken token); } + /// + /// A helper class to query blocks in the database. + /// private class BlockQuery : IBlockQuery { - private IDbCommand m_command; + /// + /// The database instance to use for the queries. + /// + private LocalDatabase m_db = null!; - public BlockQuery(IDbConnection con, IDbTransaction transaction) + /// + /// The command used to query blocks in the database. + /// + private SqliteCommand m_command = null!; + + [Obsolete("Calling this constructor will throw an exception. Use CreateAsync() instead.")] + public BlockQuery(SqliteConnection connection, SqliteTransaction? transaction) { - m_command = con.CreateCommand(transaction, @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size "); + throw new NotImplementedException("Use CreateAsync() instead"); + } + + /// + /// Constructs a new instance of the class. + /// It is private to prevent instantiation without CreateAsync. + /// + private BlockQuery() { } + + /// + /// Creates a new instance of the class. + /// + /// The local database to use for the queries. + /// A cancellation token to cancel the operation. + /// A task that when awaited returns a new instance of . + public static async Task CreateAsync(LocalDatabase db, CancellationToken token) + { + return new BlockQuery + { + m_db = db, + m_command = await db.Connection.CreateCommandAsync(@" + SELECT ""VolumeID"" + FROM ""Block"" + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size + ", token) + .ConfigureAwait(false) + }; } /// - public bool UseBlock(string hash, long size, long volumeId, IDbTransaction? transaction) + public async Task UseBlock(string hash, long size, long volumeId, CancellationToken token) { - var r = m_command.SetTransaction(transaction) + var r = await m_command + .SetTransaction(m_db.Transaction) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .ExecuteScalarInt64(-1); + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); + if (r == -1) { return false; @@ -356,81 +850,162 @@ namespace Duplicati.Library.Main.Database } /// - /// Builds a lookup table to enable faster response to block queries + /// Builds a lookup table to enable faster response to block queries. /// - public IBlockQuery CreateBlockQueryHelper(IDbTransaction transaction) + /// A cancellation token to cancel the operation. + public async Task CreateBlockQueryHelper(CancellationToken token) { - return new BlockQuery(m_connection, transaction); + return await BlockQuery.CreateAsync(this, token).ConfigureAwait(false); } /// - /// Registers a block as moved to a new volume + /// Registers a block as moved to a new volume. /// - /// The hash of the block - /// The size of the block - /// The new volume ID - /// The transaction to execute the command in - public void RegisterDuplicatedBlock(string hash, long size, long volumeID, IDbTransaction tr) + /// The hash of the block. + /// The size of the block. + /// The new volume ID. + /// A cancellation token to cancel the operation. + /// A task that when completed indicates the block has been registered. + public async Task RegisterDuplicatedBlock(string hash, long size, long volumeID, CancellationToken token) { - m_registerDuplicateBlockCommand.Transaction = tr; - m_registerDuplicateBlockCommand.SetParameterValue("@VolumeId", volumeID); - m_registerDuplicateBlockCommand.SetParameterValue("@Hash", hash); - m_registerDuplicateBlockCommand.SetParameterValue("@Size", size); // Using INSERT OR IGNORE to avoid duplicate entries, result may be 1 or 0 - m_registerDuplicateBlockCommand.ExecuteNonQuery(); + await m_registerDuplicateBlockCommand + .SetTransaction(m_rtr) + .SetParameterValue("@VolumeId", volumeID) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); } /// - /// After new volumes are uploaded, this method will update the blocks from the old volumes to point to the new volumes + /// After new volumes are uploaded, this method will update the blocks from the old volumes to point to the new volumes. /// - /// The file to remove - /// The volume IDs that will be removed - /// The transaction to execute the command in - public void PrepareForDelete(string filename, IEnumerable volumeIdsToBeRemoved, IDbTransaction transaction) + /// The file to remove. + /// The volume IDs that will be removed. + /// A cancellation token to cancel the operation. + /// A task that when completed indicates the operation has finished. + public async Task PrepareForDelete(string filename, IEnumerable volumeIdsToBeRemoved, CancellationToken token) { - var deletedVolume = GetRemoteVolume(filename, transaction); + var deletedVolume = await GetRemoteVolume(filename, token) + .ConfigureAwait(false); if (deletedVolume.Type != RemoteVolumeType.Blocks) return; - using (var cmd = m_connection.CreateCommand(transaction)) + await using var cmd = m_connection.CreateCommand(m_rtr); + var updatedBlocks = "BlocksToUpdate-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + var replacementBlocks = "ReplacementBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + try + { + await cmd.SetCommandAndParameters($@" + CREATE {TEMPORARY} TABLE ""{updatedBlocks}"" AS + SELECT ""ID"" + FROM ""Block"" + WHERE ""VolumeID"" = @VolumeId + ") + .SetParameterValue("@VolumeId", deletedVolume.ID) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + await using (var tempTable = await TemporaryDbValueList.CreateAsync(this, volumeIdsToBeRemoved, token).ConfigureAwait(false)) + await ( + await cmd.SetCommandAndParameters($@" + CREATE {TEMPORARY} TABLE ""{replacementBlocks}"" AS + SELECT + ""BlockID"", + MAX(""VolumeID"") AS ""VolumeID"" + FROM ""DuplicateBlock"" + WHERE + ""VolumeID"" NOT IN (@VolumeIds) + AND ""BlockID"" IN ( + SELECT ""ID"" + FROM ""{updatedBlocks}"" + ) + GROUP BY ""BlockID"" + ") + .ExpandInClauseParameterMssqliteAsync("@VolumeIds", tempTable, token) + .ConfigureAwait(false) + ) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + var targetCount = await cmd.ExecuteScalarInt64Async($@" + SELECT COUNT(*) + FROM ""{updatedBlocks}"" + ", token) + .ConfigureAwait(false); + + if (targetCount == 0) + return; + + var replacementCount = await cmd.ExecuteScalarInt64Async($@" + SELECT COUNT(*) + FROM ""{replacementBlocks}"" + ", token) + .ConfigureAwait(false); + + var updateCount = await cmd.SetCommandAndParameters(@$" + UPDATE ""Block"" + SET ""VolumeID"" = ( + SELECT ""VolumeID"" + FROM ""{replacementBlocks}"" + WHERE + ""{replacementBlocks}"".""BlockID"" = ""Block"".""ID"" + AND ""Block"".""VolumeID"" = @VolumeId + ) + WHERE ""Block"".""VolumeID"" = @VolumeId + ") + .SetParameterValue("@VolumeId", deletedVolume.ID) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + var deleteCount = await cmd.ExecuteNonQueryAsync(@$" + DELETE FROM ""DuplicateBlock"" + WHERE + ( + ""DuplicateBlock"".""BlockID"" + || ':' + || ""DuplicateBlock"".""VolumeID"" + ) IN ( + SELECT + ""RB"".""BlockID"" + || ':' + || ""RB"".""VolumeID"" + FROM ""{replacementBlocks}"" ""RB"" + ) + ", token) + .ConfigureAwait(false); + + if (targetCount != updateCount + || replacementCount != deleteCount + || updateCount != deleteCount) + { + throw new Exception($"Unexpected number of rows updated. Expected {targetCount} but got updated {updateCount}, deleted {deleteCount}, and replaced {replacementCount}"); + } + + // Remove knowledge of any old blocks + await cmd.SetCommandAndParameters(@$" + DELETE FROM ""DuplicateBlock"" + WHERE ""VolumeID"" = @VolumeId + ") + .SetParameterValue("@VolumeId", deletedVolume.ID) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } + finally { - var updatedBlocks = "BlocksToUpdate-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var replacementBlocks = "ReplacementBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); try { - cmd.SetCommandAndParameters(FormatInvariant($@"CREATE {TEMPORARY} TABLE ""{updatedBlocks}"" AS SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" = @VolumeId")) - .SetParameterValue("@VolumeId", deletedVolume.ID) - .ExecuteNonQuery(); - - using (var tempTable = new TemporaryDbValueList(m_connection, transaction, volumeIdsToBeRemoved)) - cmd.SetCommandAndParameters(FormatInvariant($@"CREATE {TEMPORARY} TABLE ""{replacementBlocks}"" AS SELECT ""BlockID"", MAX(""VolumeID"") AS ""VolumeID"" FROM ""DuplicateBlock"" WHERE ""VolumeID"" NOT IN (@VolumeIds) AND ""BlockID"" IN (SELECT ""ID"" FROM ""{updatedBlocks}"") GROUP BY ""BlockID"" ")) - .ExpandInClauseParameter("@VolumeIds", tempTable) - .ExecuteNonQuery(); - - var targetCount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""{updatedBlocks}"" ")); - if (targetCount == 0) - return; - - var replacementCount = cmd.ExecuteScalarInt64($@"SELECT COUNT(*) FROM ""{replacementBlocks}"" "); - var updateCount = cmd.SetCommandAndParameters(FormatInvariant(@$"UPDATE ""Block"" SET ""VolumeID"" = (SELECT ""VolumeID"" FROM ""{replacementBlocks}"" WHERE ""{replacementBlocks}"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" = @VolumeId) WHERE ""Block"".""VolumeID"" = @VolumeId ")) - .SetParameterValue("@VolumeId", deletedVolume.ID) - .ExecuteNonQuery(); - var deleteCount = cmd.ExecuteNonQuery(FormatInvariant(@$"DELETE FROM ""DuplicateBlock"" WHERE (""DuplicateBlock"".""BlockID"" || ':' || ""DuplicateBlock"".""VolumeID"") IN (SELECT ""RB"".""BlockID"" || ':' || ""RB"".""VolumeID"" FROM ""{replacementBlocks}"" RB)")); - if (targetCount != updateCount || replacementCount != deleteCount || updateCount != deleteCount) - throw new Exception($"Unexpected number of rows updated. Expected {targetCount} but got updated {updateCount}, deleted {deleteCount}, and replaced {replacementCount}"); - - // Remove knowledge of any old blocks - cmd.SetCommandAndParameters(FormatInvariant(@$"DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" = @VolumeId")) - .SetParameterValue("@VolumeId", deletedVolume.ID) - .ExecuteNonQuery(); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{updatedBlocks}"" ", token) + .ConfigureAwait(false); } - finally + catch { } + try { - try { cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{updatedBlocks}"" ")); } - catch { } - try { cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{replacementBlocks}"" ")); } - catch { } + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{replacementBlocks}"" ", token) + .ConfigureAwait(false); } + catch { } } } @@ -438,55 +1013,74 @@ namespace Duplicati.Library.Main.Database /// Calculates the sequence in which files should be deleted based on their relations. /// /// Block volumes slated for deletion. - /// The transaction to execute the command in - /// The deletable volumes. - public IEnumerable ReOrderDeleteableVolumes(IEnumerable deleteableVolumes, IDbTransaction transaction) + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of that represents the order in which volumes should be deleted. + public async IAsyncEnumerable ReOrderDeleteableVolumes(IEnumerable deleteableVolumes, [EnumeratorCancellation] CancellationToken token) { - using (var cmd = m_connection.CreateCommand(transaction)) - { - // Although the generated index volumes are always in pairs, - // this code handles many-to-many relations between - // index files and block volumes, should this be added later - var lookupBlock = new Dictionary>(); - var lookupIndexfiles = new Dictionary>(); + await using var cmd = m_connection.CreateCommand(m_rtr); + // Although the generated index volumes are always in pairs, + // this code handles many-to-many relations between + // index files and block volumes, should this be added later + var lookupBlock = new Dictionary>(); + var lookupIndexfiles = new Dictionary>(); - using (var rd = cmd.ExecuteReader(@"SELECT ""C"".""Name"", ""B"".""Name"", ""B"".""Hash"", ""B"".""Size"" FROM ""IndexBlockLink"" A, ""RemoteVolume"" B, ""RemoteVolume"" C WHERE ""A"".""IndexVolumeID"" = ""B"".""ID"" AND ""A"".""BlockVolumeID"" = ""C"".""ID"" AND ""B"".""Hash"" IS NOT NULL AND ""B"".""Size"" IS NOT NULL ")) - while (rd.Read()) + cmd.SetCommandAndParameters(@" + SELECT + ""C"".""Name"", + ""B"".""Name"", + ""B"".""Hash"", + ""B"".""Size"" + FROM + ""IndexBlockLink"" A, + ""RemoteVolume"" B, + ""RemoteVolume"" C + WHERE + ""A"".""IndexVolumeID"" = ""B"".""ID"" + AND ""A"".""BlockVolumeID"" = ""C"".""ID"" + AND ""B"".""Hash"" IS NOT NULL + AND ""B"".""Size"" IS NOT NULL + "); + + await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) + { + var name = rd.ConvertValueToString(0) ?? ""; + if (!lookupBlock.TryGetValue(name, out var indexfileList)) { - var name = rd.ConvertValueToString(0) ?? ""; - if (!lookupBlock.TryGetValue(name, out var indexfileList)) - { - indexfileList = new List(); - lookupBlock.Add(name, indexfileList); - } - - var v = new RemoteVolume(rd.ConvertValueToString(1), rd.ConvertValueToString(2), rd.ConvertValueToInt64(3)); - indexfileList.Add(v); - - if (!lookupIndexfiles.TryGetValue(v.Name, out var blockList)) - { - blockList = new List(); - lookupIndexfiles.Add(v.Name, blockList); - } - blockList.Add(name); + indexfileList = new List(); + lookupBlock.Add(name, indexfileList); } - foreach (var r in deleteableVolumes.Distinct()) - { - // Return the input - yield return r; - if (lookupBlock.TryGetValue(r.Name, out var indexfileList)) - foreach (var sh in indexfileList) - { - if (lookupIndexfiles.TryGetValue(sh.Name, out var backref)) - { - //If this is the last reference, - // remove the index file as well - if (backref.Remove(r.Name) && backref.Count == 0) - yield return sh; - } - } + var v = new RemoteVolume( + rd.ConvertValueToString(1), + rd.ConvertValueToString(2), + rd.ConvertValueToInt64(3) + ); + indexfileList.Add(v); + + if (!lookupIndexfiles.TryGetValue(v.Name, out var blockList)) + { + blockList = new List(); + lookupIndexfiles.Add(v.Name, blockList); + } + blockList.Add(name); } + + foreach (var r in deleteableVolumes.Distinct()) + { + // Return the input + yield return r; + if (lookupBlock.TryGetValue(r.Name, out var indexfileList)) + foreach (var sh in indexfileList) + { + if (lookupIndexfiles.TryGetValue(sh.Name, out var backref)) + { + //If this is the last reference, + // remove the index file as well + if (backref.Remove(r.Name) && backref.Count == 0) + yield return sh; + } + } } } diff --git a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs index c60f26e5d..02f10ce58 100644 --- a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable @@ -24,131 +24,396 @@ using System; using System.Linq; using System.Collections.Generic; +using System.Threading.Tasks; +using System.Threading; +using System.Runtime.CompilerServices; namespace Duplicati.Library.Main.Database { + /// + /// LocalListAffectedDatabase is a specialized database class for listing + /// affected filesets, files, remote logs, and remote volumes. + /// internal class LocalListAffectedDatabase : LocalDatabase { - public LocalListAffectedDatabase(string path, long pagecachesize) - : base(path, "ListAffected", false, pagecachesize) + /// + /// Creates a new instance of . + /// + /// The path to the database file. + /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A cancellation token to cancel the operation. + /// A task that when awaited returns a new instance of . + public static async Task CreateAsync(string path, LocalListAffectedDatabase? dbnew, CancellationToken token) { - ShouldCloseConnection = true; + dbnew ??= new LocalListAffectedDatabase(); + + dbnew = (LocalListAffectedDatabase) + await CreateLocalDatabaseAsync(path, "ListAffected", false, dbnew, token) + .ConfigureAwait(false); + dbnew.ShouldCloseConnection = true; + + return dbnew; } + /// + /// Represents a fileset result for listing affected filesets. + /// private class ListResultFileset : Interface.IListResultFileset { + /// + /// Gets or sets the version of the fileset. + /// public long Version { get; set; } + /// + /// Gets or sets whether the backup related to this fileset is a full backup. + /// public int IsFullBackup { get; set; } + /// + /// Gets or sets the time when the fileset was created. + /// public DateTime Time { get; set; } + /// + /// Gets or sets the number of files in the fileset. + /// public long FileCount { get; set; } + /// + /// Gets or sets the total size of files in the fileset. + /// public long FileSizes { get; set; } } + /// + /// Represents a file result for listing affected files. + /// private class ListResultFile : Interface.IListResultFile { + /// + /// Gets or sets the path of the file reperesented by this result entry. + /// public required string Path { get; set; } + /// + /// Gets or sets the sizes of the file blocks, if available. + /// This can be null if the sizes are not available or not applicable. + /// public required IEnumerable? Sizes { get; set; } } + /// + /// Represents a remote log entry for listing affected remote logs. + /// private class ListResultRemoteLog : Interface.IListResultRemoteLog { + /// + /// Gets or sets the timestamp of the log entry. + /// public required DateTime Timestamp { get; set; } + /// + /// Gets or sets the message of the log entry. + /// public required string Message { get; set; } } + /// + /// Represents a remote volume entry for listing affected remote volumes. + /// private class ListResultRemoteVolume : Interface.IListResultRemoteVolume { + /// + /// Gets or sets the name of the remote volume. + /// public required string Name { get; set; } } - public IEnumerable GetFilesets(IEnumerable items) + /// + /// Retrieves the fileset times from the database. + /// + /// The items to filter the filesets by. + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of fileset times. + public async IAsyncEnumerable GetFilesets(IEnumerable items, [EnumeratorCancellation] CancellationToken token) { - var filesets = FilesetTimes.ToArray(); + var filesets = await FilesetTimes(token) + .ToArrayAsync(cancellationToken: token) + .ConfigureAwait(false); + var dict = new Dictionary(); for (var i = 0; i < filesets.Length; i++) dict[filesets[i].Key] = i; - var sql = FormatInvariant($@"SELECT DISTINCT ""FilesetID"" FROM ( -SELECT ""FilesetID"" FROM ""FilesetEntry"" WHERE ""FileID"" IN ( SELECT ""ID"" FROM ""FileLookup"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN ( SELECT ""ID"" From ""Block"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names))))) -UNION -SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)) -)"); - using (var tmptable = new TemporaryDbValueList(m_connection, null, items)) - using (var cmd = m_connection.CreateCommand(sql).ExpandInClauseParameter("@Names", tmptable)) - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - { - var v = dict[rd.ConvertValueToInt64(0)]; - yield return new ListResultFileset() - { - Version = v, - Time = filesets[v].Value - }; - } - } + var sql = $@" + SELECT DISTINCT ""FilesetID"" + FROM ( + SELECT ""FilesetID"" + FROM ""FilesetEntry"" + WHERE ""FileID"" IN ( + SELECT ""ID"" + FROM ""FileLookup"" + WHERE ""BlocksetID"" IN ( + SELECT ""BlocksetID"" + FROM ""BlocksetEntry"" + WHERE ""BlockID"" IN ( + SELECT ""ID"" + FROM ""Block"" + WHERE ""VolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" IN (@Names) + ) + ) + ) + ) + UNION + SELECT ""ID"" + FROM ""Fileset"" + WHERE ""VolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" IN (@Names) + ) + ) + "; - public IEnumerable GetFiles(IEnumerable items) - { - var sql = FormatInvariant($@"SELECT DISTINCT ""Path"" FROM ( -SELECT ""Path"" FROM ""File"" WHERE ""BlocksetID"" IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT ""ID"" from ""RemoteVolume"" WHERE ""Name"" IN (@Names)))) - UNION -SELECT ""Path"" FROM ""File"" WHERE ""MetadataID"" IN (SELECT ""ID"" FROM ""Metadataset"" WHERE ""BlocksetID"" IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT ""ID"" from ""RemoteVolume"" WHERE ""Name"" IN (@Names))))) - UNION -SELECT ""Path"" FROM ""File"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)))) -) ORDER BY ""Path"""); - - using (var tmptable = new TemporaryDbValueList(m_connection, null, items)) - using (var cmd = m_connection.CreateCommand(sql).ExpandInClauseParameter("@Names", tmptable)) - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - yield return new ListResultFile() - { - Path = rd.ConvertValueToString(0) ?? throw new InvalidOperationException("Path is null"), - Sizes = null - }; - } - - public IEnumerable GetLogLines(IEnumerable items) - { - using var cmd = m_connection.CreateCommand(); - - foreach (var slice in items.Chunk(CHUNK_SIZE / 2)) + await using var tmptable = await TemporaryDbValueList + .CreateAsync(this, items, token) + .ConfigureAwait(false); + await using var cmd = await m_connection + .CreateCommand(sql) + .SetTransaction(m_rtr) + .ExpandInClauseParameterMssqliteAsync("@Names", tmptable, token) + .ConfigureAwait(false); + await using var rd = await cmd + .ExecuteReaderAsync(token) + .ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) { - var sql = FormatInvariant($@" -SELECT ""TimeStamp"", ""Message"" || ' ' || CASE WHEN ""Exception"" IS NULL THEN '' ELSE ""Exception"" END FROM ""LogData"" WHERE {string.Join(" OR ", slice.Select((x, i) => @$"""Message"" LIKE @Message{i}"))} -UNION -SELECT ""Timestamp"", ""Data"" FROM ""RemoteOperation"" WHERE ""Path"" IN (@Paths)"); - - cmd.SetCommandAndParameters(sql); - cmd.ExpandInClauseParameter("@Paths", items.ToArray()); - foreach ((var x, var i) in items.Select((x, i) => (x, i))) - cmd.SetParameterValue($"@Message{i}", "%" + x + "%"); - - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - yield return new ListResultRemoteLog() - { - Timestamp = ParseFromEpochSeconds(rd.ConvertValueToInt64(0)), - Message = rd.ConvertValueToString(1) ?? "" - }; + var v = dict[rd.ConvertValueToInt64(0)]; + yield return new ListResultFileset() + { + Version = v, + Time = filesets[v].Value + }; } } - public IEnumerable GetVolumes(IEnumerable items) + /// + /// Retrieves the list of files from the database. + /// + /// The items to filter the files by. + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of file results. + public async IAsyncEnumerable GetFiles(IEnumerable items, [EnumeratorCancellation] CancellationToken token) { - var sql = FormatInvariant($@"SELECT DISTINCT ""Name"" FROM ( -SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""FileLookup"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names))))))) -UNION -SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""Metadataset"" WHERE ""ID"" IN ( SELECT ""MetadataID"" FROM ""FileLookup"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)))))))) -)"); + var sql = $@" + SELECT DISTINCT ""Path"" + FROM ( + SELECT ""Path"" + FROM ""File"" + WHERE ""BlocksetID"" IN ( + SELECT ""BlocksetID"" + FROM ""BlocksetEntry"" + WHERE ""BlockID"" IN ( + SELECT ""ID"" + FROM ""Block"" + WHERE ""VolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" IN (@Names) + ) + ) + ) + UNION + SELECT ""Path"" + FROM ""File"" + WHERE ""MetadataID"" IN ( + SELECT ""ID"" + FROM ""Metadataset"" + WHERE ""BlocksetID"" IN ( + SELECT ""BlocksetID"" + FROM ""BlocksetEntry"" + WHERE ""BlockID"" IN ( + SELECT ""ID"" + FROM ""Block"" + WHERE ""VolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" IN (@Names) + ) + ) + ) + ) + UNION + SELECT ""Path"" + FROM ""File"" + WHERE ""ID"" IN ( + SELECT ""FileID"" + FROM ""FilesetEntry"" + WHERE ""FilesetID"" IN ( + SELECT ""ID"" + FROM ""Fileset"" + WHERE ""VolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" IN (@Names) + ) + ) + ) + ) + ORDER BY ""Path"" + "; - using (var cmd = m_connection.CreateCommand(sql).ExpandInClauseParameter("@Names", items.ToArray())) - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - yield return new ListResultRemoteVolume() + await using var tmptable = await TemporaryDbValueList + .CreateAsync(this, items, token) + .ConfigureAwait(false); + await using var cmd = await m_connection + .CreateCommand(sql) + .ExpandInClauseParameterMssqliteAsync("@Names", tmptable, token) + .ConfigureAwait(false); + await using var rd = await cmd + .ExecuteReaderAsync(token) + .ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + yield return new ListResultFile() + { + Path = rd.ConvertValueToString(0) ?? throw new InvalidOperationException("Path is null"), + Sizes = null + }; + } + + /// + /// Retrieves the log lines from the database that match the specified items. + /// The log lines are retrieved from both the LogData and RemoteOperation tables. + /// + /// The items to filter the log lines by. + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of log line results. + public async IAsyncEnumerable GetLogLines(IEnumerable items, [EnumeratorCancellation] CancellationToken token) + { + await using var cmd = m_connection.CreateCommand(); + + foreach (var slice in items.Chunk(CHUNK_SIZE / 2)) + { + var sql = $@" + SELECT + ""TimeStamp"", + ""Message"" + || ' ' + || + CASE + WHEN ""Exception"" IS NULL + THEN '' + ELSE ""Exception"" + END + FROM ""LogData"" + WHERE + {string.Join(" OR ", slice.Select((x, i) => @$"""Message"" LIKE @Message{i}"))} + UNION + SELECT + ""Timestamp"", + ""Data"" + FROM ""RemoteOperation"" + WHERE ""Path"" IN (@Paths) + "; + + cmd.SetCommandAndParameters(sql) + .ExpandInClauseParameterMssqlite("@Paths", items.ToArray()); + foreach ((var x, var i) in items.Select((x, i) => (x, i))) + cmd.SetParameterValue($"@Message{i}", "%" + x + "%"); + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + yield return new ListResultRemoteLog() { - Name = rd.ConvertValueToString(0) ?? "" + Timestamp = ParseFromEpochSeconds(rd.ConvertValueToInt64(0)), + Message = rd.ConvertValueToString(1) ?? "" }; + } + } + + /// + /// Retrieves the remote volumes that match the specified items. + /// This method queries the database for remote volumes associated with filesets and metadata datasets. + /// + /// The names of the remote volumes to filter by. + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of remote volume results. + public async IAsyncEnumerable GetVolumes(IEnumerable items, [EnumeratorCancellation] CancellationToken token) + { + var sql = $@" + SELECT DISTINCT ""Name"" + FROM ( + SELECT ""Name"" + FROM ""Remotevolume"" + WHERE ""ID"" IN ( + SELECT ""VolumeID"" + FROM ""Block"" + WHERE ""ID"" IN ( + SELECT ""BlockID"" + FROM ""BlocksetEntry"" + WHERE ""BlocksetID"" IN ( + SELECT ""BlocksetID"" + FROM ""FileLookup"" + WHERE ""ID"" IN ( + SELECT ""FileID"" + FROM ""FilesetEntry"" + WHERE ""FilesetID"" IN ( + SELECT ""ID"" + FROM ""Fileset"" + WHERE ""VolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" IN (@Names) + ) + ) + ) + ) + ) + ) + UNION + SELECT ""Name"" + FROM ""Remotevolume"" + WHERE ""ID"" IN ( + SELECT ""VolumeID"" + FROM ""Block"" + WHERE ""ID"" IN ( + SELECT ""BlockID"" + FROM ""BlocksetEntry"" + WHERE ""BlocksetID"" IN ( + SELECT ""BlocksetID"" + FROM ""Metadataset"" + WHERE ""ID"" IN ( + SELECT ""MetadataID"" + FROM ""FileLookup"" + WHERE ""ID"" IN ( + SELECT ""FileID"" + FROM ""FilesetEntry"" + WHERE ""FilesetID"" IN ( + SELECT ""ID"" + FROM ""Fileset"" + WHERE ""VolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" IN (@Names) + ) + ) + ) + ) + ) + ) + ) + ) + "; + + await using var cmd = m_connection + .CreateCommand(sql) + .ExpandInClauseParameterMssqlite("@Names", items.ToArray()); + await using var rd = await cmd + .ExecuteReaderAsync(token) + .ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + yield return new ListResultRemoteVolume() + { + Name = rd.ConvertValueToString(0) ?? "" + }; } } } diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index 9db3f367e..4a868d91a 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -1,214 +1,450 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable using System; using System.Collections.Generic; -using System.Data; using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; namespace Duplicati.Library.Main.Database { - internal class LocalListBrokenFilesDatabase : LocalDatabase - { - private static readonly string BLOCK_VOLUME_IDS = FormatInvariant($@"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks.ToString()}'"); - - // Invalid blocksets include those that: - // - Have BlocksetEntries with unknown/invalid blocks (meaning the data to rebuild the blockset isn't available) - // - Invalid blocks include those that appear to be in non-Blocks volumes (e.g., are listed as being in an Index or Files volume) or that appear in an unknown volume (-1) - // - Have BlocklistHash entries with unknown/invalid blocks (meaning the data which defines the list of hashes that makes up the blockset isn't available) - // - Are defined in the Blockset table but have no entries in the BlocksetEntries table (this can happen during recreate if Files volumes reference blocksets that are not found in any Index files) - // However, blocksets with a length of 0 are excluded from this check, as the corresponding blocks for these are not needed. - private static readonly string BROKEN_FILE_IDS = FormatInvariant($@" -SELECT DISTINCT ""ID"" FROM ( - SELECT ""ID"" AS ""ID"", ""BlocksetID"" AS ""BlocksetID"" FROM ""FileLookup"" WHERE ""BlocksetID"" != {FOLDER_BLOCKSET_ID} AND ""BlocksetID"" != {SYMLINK_BLOCKSET_ID} -UNION - SELECT ""A"".""ID"" AS ""ID"", ""B"".""BlocksetID"" AS ""BlocksetID"" FROM ""FileLookup"" A LEFT JOIN ""Metadataset"" B ON ""A"".""MetadataID"" = ""B"".""ID"" -) -WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN - ( - SELECT DISTINCT ""BlocksetID"" FROM - ( - SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" NOT IN - (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN - ({BLOCK_VOLUME_IDS})) - UNION - SELECT ""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Hash"" NOT IN - (SELECT ""Hash"" FROM ""Block"" WHERE ""VolumeID"" IN - ({BLOCK_VOLUME_IDS})) - UNION - SELECT ""A"".""ID"" AS ""BlocksetID"" FROM ""Blockset"" A LEFT JOIN ""BlocksetEntry"" B ON ""A"".""ID"" = ""B"".""BlocksetID"" WHERE ""A"".""Length"" > 0 AND ""B"".""BlocksetID"" IS NULL - ) - WHERE ""BlocksetID"" NOT IN (SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" == 0) - ) -"); - private static readonly string BROKEN_FILE_SETS = FormatInvariant($@"SELECT DISTINCT ""B"".""Timestamp"", ""A"".""FilesetID"", COUNT(""A"".""FileID"") AS ""FileCount"" FROM ""FilesetEntry"" A, ""Fileset"" B WHERE ""A"".""FilesetID"" = ""B"".""ID"" AND ""A"".""FileID"" IN ({BROKEN_FILE_IDS})"); - - private static readonly string BROKEN_FILE_NAMES = FormatInvariant($@"SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A LEFT JOIN ""Blockset"" B ON (""A"".""BlocksetID"" = ""B"".""ID"") WHERE ""A"".""ID"" IN ({BROKEN_FILE_IDS}) AND ""A"".""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId)"); - - private static string INSERT_BROKEN_IDS(string tablename, string IDfieldname) => FormatInvariant($@"INSERT INTO ""{tablename}"" (""{IDfieldname}"") {BROKEN_FILE_IDS} AND ""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId)"); - - public LocalListBrokenFilesDatabase(string path, long pagecachesize) - : base(path, "ListBrokenFiles", false, pagecachesize) - { - ShouldCloseConnection = true; - } - - public LocalListBrokenFilesDatabase(LocalDatabase parent) - : base(parent) - { - ShouldCloseConnection = false; - } - - public IEnumerable<(DateTime FilesetTime, long FilesetID, long RemoveFileCount)> GetBrokenFilesets(DateTime time, long[]? versions, IDbTransaction transaction) - { - var query = BROKEN_FILE_SETS; - var clause = GetFilelistWhereClause(time, versions); - if (!string.IsNullOrWhiteSpace(clause.Item1)) - query += FormatInvariant($@" AND ""A"".""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" {clause.Item1})"); - - query += @" GROUP BY ""A"".""FilesetID"""; - - using (var cmd = Connection.CreateCommand(transaction)) - foreach (var rd in cmd.SetCommandAndParameters(query).SetParameterValues(clause.Values).ExecuteReaderEnumerable()) - if (!rd.IsDBNull(0)) - yield return (ParseFromEpochSeconds(rd.ConvertValueToInt64(0, 0)), rd.ConvertValueToInt64(1, -1), rd.ConvertValueToInt64(2, 0)); - } - - public IEnumerable> GetBrokenFilenames(long filesetid, IDbTransaction transaction) - { - using (var cmd = Connection.CreateCommand(transaction, BROKEN_FILE_NAMES).SetParameterValue("@FilesetId", filesetid)) - foreach (var rd in cmd.ExecuteReaderEnumerable()) - if (!rd.IsDBNull(0)) - yield return new Tuple(rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), rd.ConvertValueToInt64(1)); - } - /// - /// Returns all index files that are orphaned, i.e., not referenced by any block files. + /// A database for listing broken files, i.e., files that reference blocksets or blocks that are not available. /// - /// Transaction to use for the query. - /// >All index files that are orphaned. - public IEnumerable GetOrphanedIndexFiles(IDbTransaction transaction) + internal class LocalListBrokenFilesDatabase : LocalDatabase { - using var cmd = Connection.CreateCommand(transaction, FormatInvariant($@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Index.ToString()}' AND ""ID"" NOT IN (SELECT ""IndexVolumeID"" FROM ""IndexBlockLink"")")); + /// + /// SQL query to get the IDs of all block volumes. + /// + private static readonly string BLOCK_VOLUME_IDS = $@" + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Type"" = '{RemoteVolumeType.Blocks}' + "; - foreach (var rd in cmd.ExecuteReaderEnumerable()) - yield return new RemoteVolume( - rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), - rd.ConvertValueToString(1) ?? throw new Exception("Hash was null"), - rd.ConvertValueToInt64(2, -1) - ); - } + // Invalid blocksets include those that: + // - Have BlocksetEntries with unknown/invalid blocks (meaning the data to rebuild the blockset isn't available) + // - Invalid blocks include those that appear to be in non-Blocks volumes (e.g., are listed as being in an Index or Files volume) or that appear in an unknown volume (-1) + // - Have BlocklistHash entries with unknown/invalid blocks (meaning the data which defines the list of hashes that makes up the blockset isn't available) + // - Are defined in the Blockset table but have no entries in the BlocksetEntries table (this can happen during recreate if Files volumes reference blocksets that are not found in any Index files) + // However, blocksets with a length of 0 are excluded from this check, as the corresponding blocks for these are not needed. - /// - /// Inserts the broken file IDs into the given table. The table must have a single column with the same name as the ID field name. - /// - /// The filset id for the current operation - /// The name of the table to insert into - /// The name of the ID field in the table - /// The transaction to use for the query - public void InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname, IDbTransaction? transaction) - { - using var cmd = Connection.CreateCommand(transaction, INSERT_BROKEN_IDS(tablename, IDfieldname)) - .SetParameterValue("@FilesetId", filesetid); - cmd.ExecuteNonQuery(); - } + /// + /// SQL query to get the IDs of broken files. + /// A broken file is one that has a BlocksetID that is not in the list of valid blocksets, or that has a MetadataID that is not in the list of valid metadata blocksets. + /// + private static readonly string BROKEN_FILE_IDS = $@" + SELECT DISTINCT ""ID"" + FROM ( + SELECT + ""ID"" AS ""ID"", + ""BlocksetID"" AS ""BlocksetID"" + FROM ""FileLookup"" + WHERE + ""BlocksetID"" != {FOLDER_BLOCKSET_ID} + AND ""BlocksetID"" != {SYMLINK_BLOCKSET_ID} + UNION + SELECT + ""A"".""ID"" AS ""ID"", + ""B"".""BlocksetID"" AS ""BlocksetID"" + FROM ""FileLookup"" ""A"" + LEFT JOIN ""Metadataset"" ""B"" + ON ""A"".""MetadataID"" = ""B"".""ID"" + ) + WHERE + ""BlocksetID"" IS NULL + OR ""BlocksetID"" IN ( + SELECT DISTINCT ""BlocksetID"" + FROM ( + SELECT ""BlocksetID"" + FROM ""BlocksetEntry"" + WHERE ""BlockID"" NOT IN ( + SELECT ""ID"" + FROM ""Block"" + WHERE ""VolumeID"" IN ({BLOCK_VOLUME_IDS}) + ) + UNION + SELECT ""BlocksetID"" + FROM ""BlocklistHash"" + WHERE ""Hash"" NOT IN ( + SELECT ""Hash"" + FROM ""Block"" + WHERE ""VolumeID"" IN ({BLOCK_VOLUME_IDS}) + ) + UNION + SELECT ""A"".""ID"" AS ""BlocksetID"" + FROM ""Blockset"" ""A"" + LEFT JOIN ""BlocksetEntry"" ""B"" + ON ""A"".""ID"" = ""B"".""BlocksetID"" + WHERE + ""A"".""Length"" > 0 + AND ""B"".""BlocksetID"" IS NULL + ) + WHERE ""BlocksetID"" NOT IN ( + SELECT ""ID"" + FROM ""Blockset"" + WHERE ""Length"" == 0 + ) + ) + "; - /// - /// Replaces the metadata blockset ID in the Metadataset table with the empty blockset ID for all fileset entries that are not in any block volume. - /// This is used to clean up the metadata blocksets that are now missing. - /// - /// The filesetId to target - /// The empty blockset ID to replace with - /// The transaction to use for the query - /// The number of rows affected - public int ReplaceMetadata(long filesetId, long emptyBlocksetId, IDbTransaction? transaction) - { - using var cmd = m_connection.CreateCommand(transaction, @" -UPDATE ""Metadataset"" -SET ""BlocksetID"" = @EmptyBlocksetID -WHERE - ""ID"" IN ( - SELECT ""FileLookup"".""MetadataID"" - FROM ""FileLookup"", ""FilesetEntry"" - WHERE - ""FilesetEntry"".""FilesetId"" = @FilesetId - AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" - ) - AND ""BlocksetID"" NOT IN ( - SELECT ""BlocksetID"" - FROM ""BlocksetEntry"", ""Block"" - WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - )") - .SetParameterValue("@EmptyBlocksetId", emptyBlocksetId) - .SetParameterValue("@FilesetId", filesetId); - return cmd.ExecuteNonQuery(); - } + /// + /// SQL query to get the broken filesets, i.e., filesets that contain broken files. + /// + private static readonly string BROKEN_FILE_SETS = $@" + SELECT DISTINCT + ""B"".""Timestamp"", + ""A"".""FilesetID"", + COUNT(""A"".""FileID"") AS ""FileCount"" + FROM + ""FilesetEntry"" ""A"", + ""Fileset"" ""B"" + WHERE + ""A"".""FilesetID"" = ""B"".""ID"" + AND ""A"".""FileID"" IN ({BROKEN_FILE_IDS}) + "; - public void RemoveMissingBlocks(IEnumerable names, IDbTransaction transaction) - { - if (names == null || !names.Any()) return; - if (transaction == null) - throw new Exception("This function cannot be called when not in a transaction, as it leaves the database in an inconsistent state"); + /// + /// SQL query to get the names and lengths of broken files. + /// + private static readonly string BROKEN_FILE_NAMES = $@" + SELECT + ""A"".""Path"", + ""B"".""Length"" + FROM ""File"" ""A"" + LEFT JOIN ""Blockset"" ""B"" + ON (""A"".""BlocksetID"" = ""B"".""ID"") + WHERE + ""A"".""ID"" IN ({BROKEN_FILE_IDS}) + AND ""A"".""ID"" IN ( + SELECT ""FileID"" + FROM ""FilesetEntry"" + WHERE ""FilesetID"" = @FilesetId + ) + "; - using (var deletecmd = m_connection.CreateCommand(transaction)) - { - var temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var volidstable = "DelVolSetIds-" + temptransguid; + /// + /// SQL query to insert broken file IDs into a specified table. + /// The table must have a single column with the same name as the ID field name. + /// + private static string INSERT_BROKEN_IDS(string tablename, string IDfieldname) => $@" + INSERT INTO ""{tablename}"" ( + ""{IDfieldname}"" + ) + {BROKEN_FILE_IDS} + AND ""ID"" IN ( + SELECT ""FileID"" + FROM ""FilesetEntry"" + WHERE ""FilesetID"" = @FilesetId + ) + "; - // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. - deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{volidstable}"" (""ID"" INTEGER PRIMARY KEY)")); - - using (var tmptable = new TemporaryDbValueList(m_connection, transaction, names)) - deletecmd.SetCommandAndParameters(FormatInvariant($@"INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)")) - .ExpandInClauseParameter("@Names", tmptable) - .ExecuteNonQuery(); - - var volIdsSubQuery = FormatInvariant($@"SELECT ""ID"" FROM ""{volidstable}"" "); - deletecmd.Parameters.Clear(); - - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})")); - - // Clean up temp tables for subqueries. We truncate content and then try to delete. - // Drop in try-block, as it fails in nested transactions (SQLite problem) - // SQLite.SQLiteException (0x80004005): database table is locked - deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{volidstable}"" ")); - try + /// + /// Creates a new instance of the class. + /// + /// The path to the database file. + /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains a new instance of . + public static async Task CreateAsync(string path, LocalListBrokenFilesDatabase? dbnew, CancellationToken token) { - deletecmd.CommandTimeout = 2; - deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{volidstable}"" ")); - } - catch { /* Ignore, will be deleted on close anyway. */ } - } - } + dbnew ??= new LocalListBrokenFilesDatabase(); - public long GetFilesetFileCount(long filesetid, IDbTransaction transaction) - { - using var cmd = m_connection.CreateCommand(transaction, @"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId") - .SetParameterValue("@FilesetId", filesetid); - return cmd.ExecuteScalarInt64(0); + dbnew = (LocalListBrokenFilesDatabase) + await CreateLocalDatabaseAsync(path, "ListBrokenFiles", false, dbnew, token) + .ConfigureAwait(false); + dbnew.ShouldCloseConnection = true; + + return dbnew; + } + + /// + /// Creates a new instance of the class. + /// + /// The parent database to use for the new database. + /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains a new instance of . + public static async Task CreateAsync(LocalDatabase dbparent, LocalListBrokenFilesDatabase? dbnew, CancellationToken token) + { + dbnew ??= new LocalListBrokenFilesDatabase(); + + dbnew = (LocalListBrokenFilesDatabase) + await CreateLocalDatabaseAsync(dbparent, dbnew, token) + .ConfigureAwait(false); + dbnew.ShouldCloseConnection = false; + + return dbnew; + } + + /// + /// Returns all broken file IDs, i.e., files that reference blocksets or blocks that are not available. + /// + /// The time to filter filesets by. + /// Optional array of versions to filter filesets by. If null, all versions are considered. + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of broken file IDs. + public async IAsyncEnumerable<(DateTime FilesetTime, long FilesetID, long RemoveFileCount)> GetBrokenFilesets(DateTime time, long[]? versions, [EnumeratorCancellation] CancellationToken token) + { + var query = BROKEN_FILE_SETS; + var clause = await GetFilelistWhereClause(time, versions, null, false, token) + .ConfigureAwait(false); + + if (!string.IsNullOrWhiteSpace(clause.Query)) + query += $@" + AND ""A"".""FilesetID"" IN ( + SELECT ""ID"" + FROM ""Fileset"" + {clause.Query} + ) + "; + + query += @" GROUP BY ""A"".""FilesetID"""; + + await using var cmd = Connection.CreateCommand(m_rtr) + .SetCommandAndParameters(query) + .SetParameterValues(clause.Values); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) + if (!rd.IsDBNull(0)) + yield return ( + ParseFromEpochSeconds(rd.ConvertValueToInt64(0, 0)), + rd.ConvertValueToInt64(1, -1), + rd.ConvertValueToInt64(2, 0) + ); + } + + /// + /// Returns all broken file IDs, i.e., files that reference blocksets or blocks that are not available. + /// + /// The fileset ID to filter by. + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of broken file IDs. + public async IAsyncEnumerable> GetBrokenFilenames(long filesetid, [EnumeratorCancellation] CancellationToken token) + { + await using var cmd = Connection.CreateCommand(m_rtr) + .SetCommandAndParameters(BROKEN_FILE_NAMES) + .SetParameterValue("@FilesetId", filesetid); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) + if (!rd.IsDBNull(0)) + yield return new Tuple( + rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), + rd.ConvertValueToInt64(1) + ); + } + + /// + /// Returns all index files that are orphaned, i.e., not referenced by any block files. + /// + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of representing the orphaned index files. + public async IAsyncEnumerable GetOrphanedIndexFiles([EnumeratorCancellation] CancellationToken token) + { + await using var cmd = Connection.CreateCommand($@" + SELECT + ""Name"", + ""Hash"", + ""Size"" + FROM ""RemoteVolume"" + WHERE + ""Type"" = '{RemoteVolumeType.Index}' + AND ""ID"" NOT IN ( + SELECT ""IndexVolumeID"" + FROM ""IndexBlockLink"" + ) + ") + .SetTransaction(m_rtr); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) + yield return new RemoteVolume( + rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), + rd.ConvertValueToString(1) ?? throw new Exception("Hash was null"), + rd.ConvertValueToInt64(2, -1) + ); + } + + /// + /// Inserts the broken file IDs into the given table. The table must have a single column with the same name as the ID field name. + /// + /// The filset id for the current operation. + /// The name of the table to insert into. + /// The name of the ID field in the table. + /// A cancellation token to cancel the operation. + /// A task that completes when the insertion is finished. + public async Task InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname, CancellationToken token) + { + await using var cmd = Connection.CreateCommand(m_rtr) + .SetCommandAndParameters(INSERT_BROKEN_IDS(tablename, IDfieldname)) + .SetParameterValue("@FilesetId", filesetid); + + await cmd.ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } + + /// + /// Replaces the metadata blockset ID in the Metadataset table with the empty blockset ID for all fileset entries that are not in any block volume. + /// This is used to clean up the metadata blocksets that are now missing. + /// + /// The filesetId to target. + /// The empty blockset ID to replace with. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains the number of rows affected + public async Task ReplaceMetadata(long filesetId, long emptyBlocksetId, CancellationToken token) + { + await using var cmd = m_connection.CreateCommand(@" + UPDATE ""Metadataset"" + SET ""BlocksetID"" = @EmptyBlocksetID + WHERE + ""ID"" IN ( + SELECT ""FileLookup"".""MetadataID"" + FROM + ""FileLookup"", + ""FilesetEntry"" + WHERE + ""FilesetEntry"".""FilesetId"" = @FilesetID + AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" + ) + AND ""BlocksetID"" NOT IN ( + SELECT ""BlocksetID"" + FROM + ""BlocksetEntry"", + ""Block"" + WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + ) + ") + .SetTransaction(m_rtr) + .SetParameterValue("@EmptyBlocksetID", emptyBlocksetId) + .SetParameterValue("@FilesetID", filesetId); + + return await cmd.ExecuteNonQueryAsync(token).ConfigureAwait(false); + } + + /// + /// Removes all blocks, blocksets, and index links that are missing from the specified volumes. + /// + /// The names of the volumes to check for missing blocks. + /// A cancellation token to cancel the operation. + /// A task that completes when the removal is finished. + public async Task RemoveMissingBlocks(IEnumerable names, CancellationToken token) + { + if (names == null || !names.Any()) return; + + await using var deletecmd = m_connection.CreateCommand(m_rtr); + var temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + var volidstable = "DelVolSetIds-" + temptransguid; + + // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way. + await deletecmd.ExecuteNonQueryAsync($@" + CREATE TEMP TABLE ""{volidstable}"" ( + ""ID"" INTEGER PRIMARY KEY + ) + ", token) + .ConfigureAwait(false); + + await using (var tmptable = await TemporaryDbValueList.CreateAsync(this, names, token).ConfigureAwait(false)) + await ( + await deletecmd.SetCommandAndParameters($@" + INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" IN (@Names) + ") + .ExpandInClauseParameterMssqliteAsync("@Names", tmptable, token) + .ConfigureAwait(false) + ) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + var volIdsSubQuery = $@" + SELECT ""ID"" + FROM ""{volidstable}"" + "; + + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""IndexBlockLink"" + WHERE + ""BlockVolumeID"" IN ({volIdsSubQuery}) + OR ""IndexVolumeID"" IN ({volIdsSubQuery}) + ", token) + .ConfigureAwait(false); + + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""Block"" + WHERE ""VolumeID"" IN ({volIdsSubQuery}) + ", token) + .ConfigureAwait(false); + + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""DeletedBlock"" + WHERE ""VolumeID"" IN ({volIdsSubQuery}) + ", token) + .ConfigureAwait(false); + + await deletecmd.ExecuteNonQueryAsync($@" + DELETE FROM ""DuplicateBlock"" + WHERE ""VolumeID"" IN ({volIdsSubQuery}) + ", token) + .ConfigureAwait(false); + + // Clean up temp tables for subqueries. We truncate content and then try to delete. + // Drop in try-block, as it fails in nested transactions (SQLite problem) + // SQLite.SQLiteException (0x80004005): database table is locked + await deletecmd + .ExecuteNonQueryAsync($@"DELETE FROM ""{volidstable}"" ", token) + .ConfigureAwait(false); + + try + { + deletecmd.CommandTimeout = 2; + await deletecmd + .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{volidstable}"" ", token) + .ConfigureAwait(false); + } + catch { /* Ignore, will be deleted on close anyway. */ } + } + + /// + /// Gets the count of files in a specific fileset. + /// + /// The ID of the fileset to count files in. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains the count of files in the specified fileset. + public async Task GetFilesetFileCount(long filesetid, CancellationToken token) + { + await using var cmd = m_connection.CreateCommand(@" + SELECT COUNT(*) + FROM ""FilesetEntry"" + WHERE ""FilesetID"" = @FilesetId + ") + .SetTransaction(m_rtr) + .SetParameterValue("@FilesetId", filesetid); + + return await cmd + .ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false); + } } - } } diff --git a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs index 78dff1c3c..83bad5cf5 100644 --- a/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs @@ -1,76 +1,183 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable using System; -using System.Data; using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; using Duplicati.Library.Utility; +using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database { + /// + /// A local database for tracking changes in file lists, such as added, deleted, or modified files. + /// internal class LocalListChangesDatabase : LocalDatabase { - public LocalListChangesDatabase(string path, long pagecachesize) - : base(path, "ListChanges", false, pagecachesize) + + /// + /// Creates a new instance of the class. + /// + /// The path to the database file. + /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains a new instance of . + public static async Task CreateAsync(string path, LocalListChangesDatabase? dbnew, CancellationToken token) { - ShouldCloseConnection = true; + dbnew ??= new LocalListChangesDatabase(); + + dbnew = (LocalListChangesDatabase) + await CreateLocalDatabaseAsync(path, "ListChanges", false, dbnew, token) + .ConfigureAwait(false); + dbnew.ShouldCloseConnection = true; + + return dbnew; } - public interface IStorageHelper : IDisposable + /// + /// Interface for storage helper that manages temporary storage of file changes. + /// + public interface IStorageHelper : IDisposable, IAsyncDisposable { - void AddElement(string path, string filehash, string metahash, long size, Interface.ListChangesElementType type, bool asNew); + /// + /// Adds an element to the temporary storage. + /// + /// The path of the file or folder. + /// The file hash, if applicable. + /// The metadata hash. + /// The size of the file or folder. + /// The type of the element (file, folder, symlink). + /// If true, adds to the current table; otherwise, adds to the previous table. + /// A cancellation token to cancel the operation. + /// A task that completes when the element is added. + Task AddElement(string path, string filehash, string metahash, long size, Interface.ListChangesElementType type, bool asNew, CancellationToken token); - void AddFromDb(long filesetId, bool asNew, IFilter filter); + /// + /// Adds elements from the database to the temporary storage. + /// + /// The ID of the fileset to add. + /// If true, adds to the current table; otherwise, adds to the previous table. + /// An optional filter to apply when adding elements. + /// A cancellation token to cancel the operation. + /// A task that completes when the elements are added. + Task AddFromDb(long filesetId, bool asNew, IFilter filter, CancellationToken token); - IChangeCountReport CreateChangeCountReport(); - IChangeSizeReport CreateChangeSizeReport(); - IEnumerable> CreateChangedFileReport(); + /// + /// Creates a report containing the count of added, deleted, and modified elements. + /// + /// A cancellation token to cancel the operation. + /// A task that, when awaited, returns an with the change counts. + Task CreateChangeCountReport(CancellationToken token); + + /// + /// Creates a report containing the size information for added, deleted, previous, and current elements. + /// + /// A cancellation token to cancel the operation. + /// A task that, when awaited, returns an with the size details. + Task CreateChangeSizeReport(CancellationToken token); + + /// + /// Asynchronously generates a report of changed files, yielding tuples that describe the change type, element type, and file path. + /// + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of tuples containing the change type, element type, and file path. + IAsyncEnumerable> CreateChangedFileReport(CancellationToken token); } + /// + /// Interface for reporting changes in file counts and sizes. + /// public interface IChangeCountReport { + /// + /// Gets the count of added folders. + /// long AddedFolders { get; } + /// + /// Gets the count of added symlinks. + /// long AddedSymlinks { get; } + /// + /// Gets the count of added files. + /// long AddedFiles { get; } + /// + /// Gets the count of deleted folders. + /// long DeletedFolders { get; } + /// + /// Gets the count of deleted symlinks. + /// long DeletedSymlinks { get; } + /// + /// Gets the count of deleted files. + /// long DeletedFiles { get; } + /// + /// Gets the count of modified folders. + /// long ModifiedFolders { get; } + /// + /// Gets the count of modified symlinks. + /// long ModifiedSymlinks { get; } + /// + /// Gets the count of modified files. + /// long ModifiedFiles { get; } } + /// + /// Interface for a report describing changes in file sizes. + /// public interface IChangeSizeReport { + /// + /// Gets the total size of added files. + /// long AddedSize { get; } + /// + /// Gets the total size of deleted files. + /// long DeletedSize { get; } + /// + /// Gets the size of files in the previous state. + /// long PreviousSize { get; } + /// + /// Gets the size of files in the current state. + /// long CurrentSize { get; } } + /// + /// Internal class that implements the interface to report changes in file counts. + /// internal class ChangeCountReport : IChangeCountReport { public long AddedFolders { get; internal set; } @@ -86,6 +193,9 @@ namespace Duplicati.Library.Main.Database public long ModifiedFiles { get; internal set; } } + /// + /// Internal class that implements the interface to report changes in file sizes. + /// internal class ChangeSizeReport : IChangeSizeReport { public long AddedSize { get; internal set; } @@ -95,126 +205,382 @@ namespace Duplicati.Library.Main.Database public long CurrentSize { get; internal set; } } + /// + /// Helper class for managing temporary storage of file changes. + /// Implements the interface. + /// private class StorageHelper : IStorageHelper { - private readonly IDbConnection m_connection; - private IDbTransaction m_transaction; + /// + /// The database instance used for storage operations. + /// + private LocalDatabase m_db = null!; - private IDbCommand m_insertPreviousElementCommand; - private IDbCommand m_insertCurrentElementCommand; + /// + /// Command for inserting elements into the previous table. + /// + private SqliteCommand m_insertPreviousElementCommand = null!; + /// + /// Command for inserting elements into the current table. + /// + private SqliteCommand m_insertCurrentElementCommand = null!; - private string m_previousTable; - private string m_currentTable; + /// + /// The name of the temporary table for previous elements. + /// + private string m_previousTable = null!; + /// + /// The name of the temporary table for current elements. + /// + private string m_currentTable = null!; - public StorageHelper(IDbConnection con) + /// + /// Private constructor to prevent direct instantiation. + /// This constructor is obsolete and will throw an exception if called. + /// Use the method to create an instance instead. + /// + [Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")] + public StorageHelper(SqliteConnection con) { } + + /// + /// Private constructor to prevent direct instantiation. + /// This class should be created using the CreateAsync method. + /// + private StorageHelper() { } + + /// + /// Asynchronously creates a new instance of the class. + /// + /// The local database instance to use. + /// A cancellation token to cancel the operation. + /// A task that, when awaited, returns a new instance of . + public static async Task CreateAsync(LocalDatabase db, CancellationToken token) { - m_connection = con; - m_previousTable = "Previous-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - m_currentTable = "Current-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - - m_transaction = m_connection.BeginTransactionSafe(); - - using (var cmd = m_connection.CreateCommand(m_transaction)) + var sh = new StorageHelper { - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_previousTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_currentTable}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ")); + m_db = db, + m_previousTable = "Previous-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()), + m_currentTable = "Current-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()) + }; + + await using (var cmd = sh.m_db.Connection.CreateCommand(db.Transaction)) + { + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{sh.m_previousTable}"" ( + ""Path"" TEXT NOT NULL, + ""FileHash"" TEXT NULL, + ""MetaHash"" TEXT NOT NULL, + ""Size"" INTEGER NOT NULL, + ""Type"" INTEGER NOT NULL + ) + ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{sh.m_currentTable}"" ( + ""Path"" TEXT NOT NULL, + ""FileHash"" TEXT NULL, + ""MetaHash"" TEXT NOT NULL, + ""Size"" INTEGER NOT NULL, + ""Type"" INTEGER NOT NULL + ) + ", token) + .ConfigureAwait(false); } - m_insertPreviousElementCommand = m_connection.CreateCommand(m_transaction, FormatInvariant($@"INSERT INTO ""{m_previousTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)")); - m_insertCurrentElementCommand = m_connection.CreateCommand(FormatInvariant($@"INSERT INTO ""{m_currentTable}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)")); + sh.m_insertPreviousElementCommand = await sh.m_db.Connection + .CreateCommandAsync($@" + INSERT INTO ""{sh.m_previousTable}"" ( + ""Path"", + ""FileHash"", + ""MetaHash"", + ""Size"", + ""Type"" + ) + VALUES ( + @Path, + @FileHash, + @MetaHash, + @Size, + @Type + ) + ", token) + .ConfigureAwait(false); + + sh.m_insertCurrentElementCommand = await sh.m_db.Connection + .CreateCommandAsync($@" + INSERT INTO ""{sh.m_currentTable}"" ( + ""Path"", + ""FileHash"", + ""MetaHash"", + ""Size"", + ""Type"" + ) + VALUES ( + @Path, + @FileHash, + @MetaHash, + @Size, + @Type + ) + ", token) + .ConfigureAwait(false); + + return sh; } - public void AddFromDb(long filesetId, bool asNew, IFilter filter) + /// + public async Task AddFromDb(long filesetId, bool asNew, IFilter filter, CancellationToken token) { var tablename = asNew ? m_currentTable : m_previousTable; - var folders = FormatInvariant($@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {(int)Interface.ListChangesElementType.Folder} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -100 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "); - var symlinks = FormatInvariant($@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {(int)Interface.ListChangesElementType.Symlink} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -200 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" "); - var files = FormatInvariant($@"SELECT ""File"".""Path"" AS ""Path"", ""FB"".""FullHash"" AS ""FileHash"", ""MB"".""Fullhash"" AS ""MetaHash"", ""FB"".""Length"" AS ""Size"", {(int)Interface.ListChangesElementType.File} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" MB, ""Blockset"" FB WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" >= 0 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""MB"".""ID"" AND ""File"".""BlocksetID"" = ""FB"".""ID"" "); - var combined = "(" + folders + " UNION " + symlinks + " UNION " + files + ")"; + var folders = $@" + SELECT + ""File"".""Path"" AS ""Path"", + NULL AS ""FileHash"", + ""Blockset"".""Fullhash"" AS ""MetaHash"", + -1 AS ""Size"", + {(int)Interface.ListChangesElementType.Folder} AS ""Type"", + ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" + FROM + ""File"", + ""FilesetEntry"", + ""Metadataset"", + ""Blockset"" + WHERE + ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""File"".""BlocksetID"" = -100 + AND ""Metadataset"".""ID""=""File"".""MetadataID"" + AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" + "; + var symlinks = $@" + SELECT + ""File"".""Path"" AS ""Path"", + NULL AS ""FileHash"", + ""Blockset"".""Fullhash"" AS ""MetaHash"", + -1 AS ""Size"", + {(int)Interface.ListChangesElementType.Symlink} AS ""Type"", + ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" + FROM + ""File"", + ""FilesetEntry"", + ""Metadataset"", + ""Blockset"" + WHERE + ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""File"".""BlocksetID"" = -200 + AND ""Metadataset"".""ID""=""File"".""MetadataID"" + AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" + "; - using (var cmd = m_connection.CreateCommand(m_transaction)) + var files = $@" + SELECT + ""File"".""Path"" AS ""Path"", + ""FB"".""FullHash"" AS ""FileHash"", + ""MB"".""Fullhash"" AS ""MetaHash"", + ""FB"".""Length"" AS ""Size"", + {(int)Interface.ListChangesElementType.File} AS ""Type"", + ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" + FROM + ""File"", + ""FilesetEntry"", + ""Metadataset"", + ""Blockset"" MB, + ""Blockset"" FB + WHERE + ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""File"".""BlocksetID"" >= 0 + AND ""Metadataset"".""ID""=""File"".""MetadataID"" + AND ""Metadataset"".""BlocksetID"" = ""MB"".""ID"" + AND ""File"".""BlocksetID"" = ""FB"".""ID"" "; + + var combined = $"({folders} UNION {symlinks} UNION {files})"; + + await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + if (filter == null || filter.Empty) { - if (filter == null || filter.Empty) + // Simple case, select everything + await cmd.SetCommandAndParameters($@" + INSERT INTO ""{tablename}"" ( + ""Path"", + ""FileHash"", + ""MetaHash"", + ""Size"", + ""Type"" + ) + SELECT + ""Path"", + ""FileHash"", + ""MetaHash"", + ""Size"", + ""Type"" + FROM {combined} ""A"" + WHERE ""A"".""FilesetID"" = @FilesetId + ") + .SetParameterValue("@FilesetId", filesetId) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } + else if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == Duplicati.Library.Utility.FilterType.Simple) + { + // File list based + // unfortunately we cannot do this if the filesystem is case sensitive as + // SQLite only supports ASCII compares + var p = expression.GetSimpleList(); + var filenamestable = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{filenamestable}"" ( + ""Path"" TEXT NOT NULL + ) + ", token) + .ConfigureAwait(false); + + await cmd.SetCommandAndParameters($@" + INSERT INTO ""{filenamestable}"" (""Path"") + VALUES (@Path) + ") + .PrepareAsync(token) + .ConfigureAwait(false); + + foreach (var s in p) + await cmd + .SetParameterValue("@Path", s) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + string whereClause; + if (expression.Result) { - // Simple case, select everything - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {combined} A WHERE ""A"".""FilesetID"" = @FilesetId ")) - .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQuery(); - } - else if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == Duplicati.Library.Utility.FilterType.Simple) - { - // File list based - // unfortunately we cannot do this if the filesystem is case sensitive as - // SQLite only supports ASCII compares - var p = expression.GetSimpleList(); - var filenamestable = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{filenamestable}"" (""Path"" TEXT NOT NULL) ")); - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{filenamestable}"" (""Path"") VALUES (@Path)")); - - foreach (var s in p) - cmd.SetParameterValue("@Path", s) - .ExecuteNonQuery(); - - string whereClause; - if (expression.Result) - { - // Include filter - whereClause = FormatInvariant($@"""A"".""FilesetID"" = @FilesetId AND ""A"".""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{filenamestable}"")"); - } - else - { - // Exclude filter - whereClause = FormatInvariant($@"""A"".""FilesetID"" = @FilesetId AND ""A"".""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""{filenamestable}"")"); - } - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {combined} A WHERE {whereClause} ")) - .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQuery(); - - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{filenamestable}"" ")); + // Include filter + whereClause = $@" + ""A"".""FilesetID"" = @FilesetId + AND ""A"".""Path"" IN ( + SELECT DISTINCT ""Path"" + FROM ""{filenamestable}"" + ) + "; } else { - // Do row-wise iteration - var values = new object[5]; - cmd.SetCommandAndParameters(FormatInvariant($@"SELECT ""A"".""Path"", ""A"".""FileHash"", ""A"".""MetaHash"", ""A"".""Size"", ""A"".""Type"" FROM {combined} A WHERE ""A"".""FilesetID"" = @FilesetId")) - .SetParameterValue("@FilesetId", filesetId); - using (var cmd2 = m_connection.CreateCommand(m_transaction, FormatInvariant($@"INSERT INTO ""{tablename}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (@Path,@FileHash,@MetaHash,@Size,@Type)"))) - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - { - rd.GetValues(values); - var path = values[0] as string; - if (path != null && FilterExpression.Matches(filter, path.ToString())) - { - cmd2.SetParameterValue("@Path", values[0]) - .SetParameterValue("@FileHash", values[1]) - .SetParameterValue("@MetaHash", values[2]) - .SetParameterValue("@Size", values[3]) - .SetParameterValue("@Type", values[4]) - .ExecuteNonQuery(); - } - } + // Exclude filter + whereClause = $@" + ""A"".""FilesetID"" = @FilesetId + AND ""A"".""Path"" NOT IN ( + SELECT DISTINCT ""Path"" + FROM ""{filenamestable}"" + ) + "; + } + await cmd.SetCommandAndParameters($@" + INSERT INTO ""{tablename}"" ( + ""Path"", + ""FileHash"", + ""MetaHash"", + ""Size"", + ""Type"" + ) + SELECT + ""Path"", + ""FileHash"", + ""MetaHash"", + ""Size"", + ""Type"" + FROM {combined} ""A"" + WHERE {whereClause} + ") + .SetParameterValue("@FilesetId", filesetId) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + await cmd + .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filenamestable}"" ", token) + .ConfigureAwait(false); + } + else + { + // Do row-wise iteration + var values = new object[5]; + await cmd.SetCommandAndParameters($@" + SELECT + ""A"".""Path"", + ""A"".""FileHash"", + ""A"".""MetaHash"", + ""A"".""Size"", + ""A"".""Type"" + FROM {combined} ""A"" + WHERE ""A"".""FilesetID"" = @FilesetId + ") + .SetParameterValue("@FilesetId", filesetId) + .PrepareAsync(token) + .ConfigureAwait(false); + + await using var cmd2 = m_db.Connection.CreateCommand(m_db.Transaction) + .SetCommandAndParameters($@" + INSERT INTO ""{tablename}"" ( + ""Path"", + ""FileHash"", + ""MetaHash"", + ""Size"", + ""Type"" + ) + VALUES ( + @Path, + @FileHash, + @MetaHash, + @Size, + @Type + ) + "); + await cmd2.PrepareAsync(token).ConfigureAwait(false); + + await using var rd = await cmd + .ExecuteReaderAsync(token) + .ConfigureAwait(false); + + while (await rd.ReadAsync(token).ConfigureAwait(false)) + { + rd.GetValues(values); + var path = values[0] as string; + if (path != null && FilterExpression.Matches(filter, path.ToString())) + { + await cmd2 + .SetParameterValue("@Path", values[0]) + .SetParameterValue("@FileHash", values[1]) + .SetParameterValue("@MetaHash", values[2]) + .SetParameterValue("@Size", values[3]) + .SetParameterValue("@Type", values[4]) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } } } } - public void AddElement(string path, string filehash, string metahash, long size, Interface.ListChangesElementType type, bool asNew) + /// + public async Task AddElement(string path, string filehash, string metahash, long size, Interface.ListChangesElementType type, bool asNew, CancellationToken token) { var cmd = asNew ? m_insertCurrentElementCommand : m_insertPreviousElementCommand; - cmd.SetParameterValue("@Path", path); - cmd.SetParameterValue("@FileHash", filehash); - cmd.SetParameterValue("@MetaHash", metahash); - cmd.SetParameterValue("@Size", size); - cmd.SetParameterValue("@Type", (int)type); - cmd.ExecuteNonQuery(); + await cmd + .SetParameterValue("@Path", path) + .SetParameterValue("@FileHash", filehash) + .SetParameterValue("@MetaHash", metahash) + .SetParameterValue("@Size", size) + .SetParameterValue("@Type", (int)type) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); } - private static IEnumerable ReaderToStringList(IDataReader rd) + /// + /// Converts a SqliteDataReader to an asynchronous enumerable of strings. + /// + /// The SqliteDataReader to read from. + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of strings, where each string is a value from the first column of the reader. + private static async IAsyncEnumerable ReaderToStringList(SqliteDataReader rd, [EnumeratorCancellation] CancellationToken token) { - using (rd) - while (rd.Read()) + await using (rd) + while (await rd.ReadAsync(token).ConfigureAwait(false)) { var v = rd.GetValue(0); if (v == null || v == DBNull.Value) @@ -224,85 +590,181 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Retrieves SQL queries for added, deleted, and modified files based on the current and previous tables. + /// + /// If true, retrieves all types of changes; otherwise, filters by type. + /// A tuple containing SQL queries for added, deleted, and modified files. private (string Added, string Deleted, string Modified) GetSqls(bool allTypes) { return ( - FormatInvariant($@"SELECT ""Path"" FROM ""{m_currentTable}"" WHERE {(allTypes ? "" : FormatInvariant(@$" ""{m_currentTable}"".""Type"" = @Type AND "))} ""{m_currentTable}"".""Path"" NOT IN (SELECT ""Path"" FROM ""{m_previousTable}"")"), - FormatInvariant($@"SELECT ""Path"" FROM ""{m_previousTable}"" WHERE {(allTypes ? "" : FormatInvariant(@$" ""{m_previousTable}"".""Type"" = @Type AND "))} ""{m_previousTable}"".""Path"" NOT IN (SELECT ""Path"" FROM ""{m_currentTable}"")"), - FormatInvariant($@"SELECT ""{m_currentTable}"".""Path"" FROM ""{m_currentTable}"",""{m_previousTable}"" WHERE {(allTypes ? "" : FormatInvariant($@" ""{m_currentTable}"".""Type"" = @Type AND "))} ""{m_currentTable}"".""Path"" = ""{m_previousTable}"".""Path"" AND (""{m_currentTable}"".""FileHash"" != ""{m_previousTable}"".""FileHash"" OR ""{m_currentTable}"".""MetaHash"" != ""{m_previousTable}"".""MetaHash"" OR ""{m_currentTable}"".""Type"" != ""{m_previousTable}"".""Type"") ") + $@" + SELECT ""Path"" + FROM ""{m_currentTable}"" + WHERE + {(allTypes ? "" : @$" ""{m_currentTable}"".""Type"" = @Type AND ")} + ""{m_currentTable}"".""Path"" NOT IN ( + SELECT ""Path"" + FROM ""{m_previousTable}"" + ) + ", + + $@" + SELECT ""Path"" + FROM ""{m_previousTable}"" + WHERE + {(allTypes ? "" : @$" ""{m_previousTable}"".""Type"" = @Type AND ")} + ""{m_previousTable}"".""Path"" NOT IN ( + SELECT ""Path"" + FROM ""{m_currentTable}"" + ) + ", + + $@" + SELECT ""{m_currentTable}"".""Path"" + FROM ""{m_currentTable}"",""{m_previousTable}"" + WHERE + {(allTypes ? "" : $@" ""{m_currentTable}"".""Type"" = @Type AND ")} + ""{m_currentTable}"".""Path"" = ""{m_previousTable}"".""Path"" + AND ( + ""{m_currentTable}"".""FileHash"" != ""{m_previousTable}"".""FileHash"" + OR ""{m_currentTable}"".""MetaHash"" != ""{m_previousTable}"".""MetaHash"" + OR ""{m_currentTable}"".""Type"" != ""{m_previousTable}"".""Type"" + ) + " ); } - public IChangeSizeReport CreateChangeSizeReport() + /// + /// Creates a report of changes in file sizes, including added, deleted, previous, and current sizes. + /// + /// A cancellation token to cancel the operation. + /// A task that, when awaited, returns an with the size details. + public async Task CreateChangeSizeReport(CancellationToken token) { - var sqls = GetSqls(true); - var added = sqls.Added; - var deleted = sqls.Deleted; - //var modified = sqls.Modified; + var (Added, Deleted, Modified) = GetSqls(true); - using (var cmd = m_connection.CreateCommand(m_transaction)) + await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + var result = new ChangeSizeReport { - var result = new ChangeSizeReport(); + PreviousSize = await cmd.ExecuteScalarInt64Async($@" + SELECT SUM(""Size"") + FROM ""{m_previousTable}"" + ", 0, token) + .ConfigureAwait(false), - result.PreviousSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_previousTable}"" "), 0); - result.CurrentSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_currentTable}"" "), 0); + CurrentSize = await cmd.ExecuteScalarInt64Async($@" + SELECT SUM(""Size"") + FROM ""{m_currentTable}"" + ", 0, token) + .ConfigureAwait(false), - result.AddedSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_currentTable}"" WHERE ""{m_currentTable}"".""Path"" IN ({added}) "), 0); - result.DeletedSize = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_previousTable}"" WHERE ""{m_previousTable}"".""Path"" IN ({deleted}) "), 0); + AddedSize = await cmd.ExecuteScalarInt64Async($@" + SELECT SUM(""Size"") + FROM ""{m_currentTable}"" + WHERE ""{m_currentTable}"".""Path"" IN ({Added}) + ", 0, token) + .ConfigureAwait(false), - return result; - } + DeletedSize = await cmd.ExecuteScalarInt64Async($@" + SELECT SUM(""Size"") + FROM ""{m_previousTable}"" + WHERE ""{m_previousTable}"".""Path"" IN ({Deleted}) + ", 0, token) + .ConfigureAwait(false) + }; + + return result; } - public IChangeCountReport CreateChangeCountReport() + /// + /// Creates a report containing the count of added, deleted, and modified elements. + /// + /// A cancellation token to cancel the operation. + /// A task that, when awaited, returns an with the change counts. + public async Task CreateChangeCountReport(CancellationToken token) { - var sqls = GetSqls(false); - var added = @"SELECT COUNT(*) FROM (" + sqls.Added + ")"; - var deleted = @"SELECT COUNT(*) FROM (" + sqls.Deleted + ")"; - var modified = @"SELECT COUNT(*) FROM (" + sqls.Modified + ")"; + var (Added, Deleted, Modified) = GetSqls(false); - using (var cmd = m_connection.CreateCommand(m_transaction)) + var added = @$" + SELECT COUNT(*) + FROM ({Added}) + "; + + var deleted = @$" + SELECT COUNT(*) + FROM ({Deleted}) + "; + + var modified = @$" + SELECT COUNT(*) + FROM ({Modified}) + "; + + await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + + var result = new ChangeCountReport { - var result = new ChangeCountReport(); - result.AddedFolders = cmd.SetCommandAndParameters(added) + AddedFolders = await cmd + .SetCommandAndParameters(added) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Folder) - .ExecuteScalarInt64(0); - result.AddedSymlinks = cmd.SetCommandAndParameters(added) + .ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false), + AddedSymlinks = await cmd + .SetCommandAndParameters(added) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Symlink) - .ExecuteScalarInt64(0); - result.AddedFiles = cmd.SetCommandAndParameters(added) + .ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false), + AddedFiles = await cmd + .SetCommandAndParameters(added) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.File) - .ExecuteScalarInt64(0); - - result.DeletedFolders = cmd.SetCommandAndParameters(deleted) + .ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false), + DeletedFolders = await cmd + .SetCommandAndParameters(deleted) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Folder) - .ExecuteScalarInt64(0); - result.DeletedSymlinks = cmd.SetCommandAndParameters(deleted) + .ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false), + DeletedSymlinks = await cmd + .SetCommandAndParameters(deleted) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Symlink) - .ExecuteScalarInt64(0); - result.DeletedFiles = cmd.SetCommandAndParameters(deleted) + .ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false), + DeletedFiles = await cmd + .SetCommandAndParameters(deleted) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.File) - .ExecuteScalarInt64(0); - - result.ModifiedFolders = cmd.SetCommandAndParameters(modified) + .ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false), + ModifiedFolders = await cmd + .SetCommandAndParameters(modified) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Folder) - .ExecuteScalarInt64(0); - result.ModifiedSymlinks = cmd.SetCommandAndParameters(modified) + .ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false), + ModifiedSymlinks = await cmd + .SetCommandAndParameters(modified) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.Symlink) - .ExecuteScalarInt64(0); - result.ModifiedFiles = cmd.SetCommandAndParameters(modified) + .ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false), + ModifiedFiles = await cmd + .SetCommandAndParameters(modified) .SetParameterValue("@Type", (int)Interface.ListChangesElementType.File) - .ExecuteScalarInt64(0); + .ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false) + }; - return result; - } + return result; } - public IEnumerable> CreateChangedFileReport() + /// + /// Asynchronously creates a report of changed files, yielding tuples that describe the change type, element type, and file path. + /// + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of tuples containing the change type, element type, and file path. + public async IAsyncEnumerable> CreateChangedFileReport([EnumeratorCancellation] CancellationToken token) { - var sqls = GetSqls(false); + var (Added, Deleted, Modified) = GetSqls(false); - using (var cmd = m_connection.CreateCommand(m_transaction)) + await using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) { var elTypes = new[] { Interface.ListChangesElementType.Folder, @@ -310,56 +772,88 @@ namespace Duplicati.Library.Main.Database Interface.ListChangesElementType.File }; - IEnumerable> BuildResult(IDbCommand cmd, string sql, Interface.ListChangesChangeType changeType) + async IAsyncEnumerable> BuildResult(SqliteCommand cmd, string sql, Interface.ListChangesChangeType changeType) { cmd.SetCommandAndParameters(sql); foreach (var type in elTypes) - foreach (var s in ReaderToStringList(cmd.SetParameterValue("@Type", (int)type).ExecuteReader())) + await foreach (var s in ReaderToStringList(await cmd.SetParameterValue("@Type", (int)type).ExecuteReaderAsync(token).ConfigureAwait(false), token).ConfigureAwait(false)) yield return new Tuple(changeType, type, s ?? ""); } - foreach (var r in BuildResult(cmd, sqls.Added, Interface.ListChangesChangeType.Added)) + await foreach (var r in + BuildResult(cmd, Added, Interface.ListChangesChangeType.Added) + .ConfigureAwait(false) + ) yield return r; - foreach (var r in BuildResult(cmd, sqls.Deleted, Interface.ListChangesChangeType.Deleted)) + + await foreach (var r in + BuildResult(cmd, Deleted, Interface.ListChangesChangeType.Deleted) + .ConfigureAwait(false) + ) yield return r; - foreach (var r in BuildResult(cmd, sqls.Modified, Interface.ListChangesChangeType.Modified)) + + await foreach (var r in + BuildResult(cmd, Modified, Interface.ListChangesChangeType.Modified) + .ConfigureAwait(false) + ) yield return r; } } public void Dispose() + { + DisposeAsync().AsTask().Await(); + } + + public async ValueTask DisposeAsync() { if (m_insertPreviousElementCommand != null) { - try { m_insertPreviousElementCommand.Dispose(); } + try + { + await m_insertPreviousElementCommand + .DisposeAsync() + .ConfigureAwait(false); + } catch { } finally { m_insertPreviousElementCommand = null!; } } if (m_insertCurrentElementCommand != null) { - try { m_insertCurrentElementCommand.Dispose(); } + try + { + await m_insertCurrentElementCommand + .DisposeAsync() + .ConfigureAwait(false); + } catch { } finally { m_insertCurrentElementCommand = null!; } } - if (m_transaction != null) + try { - try { m_transaction.Rollback(); } - catch { } - finally - { - m_previousTable = null!; - m_currentTable = null!; - m_transaction = null!; - } + await m_db.Transaction + .RollBackAsync() + .ConfigureAwait(false); + } + catch { } + finally + { + m_previousTable = null!; + m_currentTable = null!; } } } - public IStorageHelper CreateStorageHelper() + /// + /// Creates a new instance of the for managing temporary storage of file changes. + /// + /// A cancellation token to cancel the operation. + /// A task that, when awaited, returns an instance of . + public async Task CreateStorageHelper(CancellationToken token) { - return new StorageHelper(m_connection); + return await StorageHelper.CreateAsync(this, token).ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index 33d18348a..95fb130ac 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -1,99 +1,268 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable using System; -using System.Data; using System.Linq; using System.Collections.Generic; +using System.Threading.Tasks; using Duplicati.Library.Common.IO; using Duplicati.Library.Utility; using Duplicati.Library.Interface; +using Microsoft.Data.Sqlite; +using System.Threading; +using System.Runtime.CompilerServices; namespace Duplicati.Library.Main.Database { + /// + /// A local database that stores file lists and filesets. + /// internal class LocalListDatabase : LocalDatabase { - public LocalListDatabase(string path, long pagecachesize) - : base(path, "List", false, pagecachesize) + /// + /// Creates a new instance of the LocalListDatabase. + /// + /// The path to the database file. + /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains a new instance of LocalListDatabase. + public static async Task CreateAsync(string path, LocalListDatabase? dbnew, CancellationToken token) { - ShouldCloseConnection = true; + dbnew ??= new LocalListDatabase(); + + dbnew = (LocalListDatabase) + await CreateLocalDatabaseAsync(path, "List", false, dbnew, token) + .ConfigureAwait(false); + dbnew.ShouldCloseConnection = true; + + return dbnew; } + /// + /// Interface for file versions. + /// public interface IFileversion { + /// + /// Gets the path of the file, or null if not available. + /// string? Path { get; } - IEnumerable Sizes { get; } + /// + /// Asynchronously retrieves the sizes of the file versions. + /// + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of file sizes. + IAsyncEnumerable Sizes(CancellationToken token); } + /// + /// Interface for filesets. + /// public interface IFileset { + /// + /// Gets the version of the fileset. + /// long Version { get; } + /// + /// Gets or sets a value indicating whether the fileset is a full backup. + /// int IsFullBackup { get; set; } + /// + /// Gets the timestamp of the fileset. + /// DateTime Time { get; } + /// + /// Gets the count of files in the fileset. + /// long FileCount { get; } + /// + /// Gets the total sizes of files in the fileset. + /// long FileSizes { get; } } - public interface IFileSets : IDisposable + /// + /// Interface for managing file sets in the local database. + /// + public interface IFileSets : IDisposable, IAsyncDisposable { - IEnumerable Sets { get; } - IEnumerable QuickSets { get; } - IEnumerable SelectFiles(IFilter filter); - IEnumerable GetLargestPrefix(IFilter filter); - IEnumerable SelectFolderContents(IFilter filter); - void TakeFirst(); + /// + /// Gets all of the filesets in the database as an asynchronous enumerable, including file count and total size. + /// + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of filesets. + IAsyncEnumerable Sets(CancellationToken token); + /// + /// Gets all of the filesets in the database as an asynchronous enumerable, omitting file count and total size. + /// + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of filesets. + IAsyncEnumerable QuickSets(CancellationToken token); + /// + /// Gets the files in the filesets that match the provided filter as an asynchronous enumerable. + /// + /// The filter to apply to the files. + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of file versions representing the selected files. + IAsyncEnumerable SelectFiles(IFilter filter, CancellationToken token); + /// + /// Gets the largest prefix of files in the filesets that match the provided filter as an asynchronous enumerable. + /// + /// The filter to apply to the files. + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of file versions representing the largest prefix. + IAsyncEnumerable GetLargestPrefix(IFilter filter, CancellationToken token); + /// + /// Selects the contents of a folder in the filesets that match the provided filter as an asynchronous enumerable. + /// + /// The filter to apply to the folder contents. + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of file versions representing the folder contents. + IAsyncEnumerable SelectFolderContents(IFilter filter, CancellationToken token); + /// + /// Deletes all filesets except the most recent one. + /// + /// A cancellation token to cancel the operation. + /// A task that completes when the filesets have been deleted. + Task TakeFirst(CancellationToken token); } + /// + /// Creates a new instance of the class implementing the interface. + /// private class FileSets : IFileSets { - private readonly IDbConnection m_connection; - private string m_tablename; - private readonly KeyValuePair[] m_filesets; + /// + /// The database instance to execute commands against. + /// + private LocalDatabase m_db = null!; + /// + /// The name of the temporary table used to store filesets. + /// + private string m_tablename = null!; + /// + /// An array of key-value pairs representing filesets, where the key is the fileset ID and the value is the timestamp. + /// + private KeyValuePair[] m_filesets = []; + /// + /// Private constructor to prevent instantiation without using CreateAsync. + /// + [Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")] public FileSets(LocalListDatabase owner, DateTime time, long[] versions) { - m_connection = owner.m_connection; - m_filesets = owner.FilesetTimes.ToArray(); - m_tablename = "Filesets-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var tmp = owner.GetFilelistWhereClause(time, versions, m_filesets); + throw new NotSupportedException("Use CreateAsync instead."); + } + + /// + /// Private constructor to prevent instantiation without using CreateAsync. + /// + private FileSets() { } + + /// + /// Asynchronously creates a new instance of the class. + /// + /// The owner database instance. + /// The time to filter filesets by. + /// An array of versions to filter filesets by. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains a new instance of . + public static async Task CreateAsync(LocalListDatabase owner, DateTime time, long[] versions, CancellationToken token) + { + var fs = new FileSets + { + m_db = owner, + m_filesets = await owner + .FilesetTimes(token) + .ToArrayAsync(cancellationToken: token) + .ConfigureAwait(false), + m_tablename = "Filesets-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()) + }; + var tmp = await owner.GetFilelistWhereClause(time, versions, fs.m_filesets, false, token) + .ConfigureAwait(false); var query = tmp.Item1; var args = tmp.Item2; - using (var cmd = m_connection.CreateCommand()) + await using (var cmd = fs.m_db.Connection.CreateCommand()) { - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tablename}"" AS SELECT DISTINCT ""ID"" AS ""FilesetID"", ""IsFullBackup"" AS ""IsFullBackup"" , ""Timestamp"" AS ""Timestamp"" FROM ""Fileset"" {query}"), args); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tablename}_FilesetIDTimestampIndex"" ON ""{m_tablename}"" (""FilesetID"", ""Timestamp"" DESC)")); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{fs.m_tablename}"" AS + SELECT DISTINCT + ""ID"" AS ""FilesetID"", + ""IsFullBackup"" AS ""IsFullBackup"", + ""Timestamp"" AS ""Timestamp"" + FROM ""Fileset"" {query} + ", args, token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{fs.m_tablename}_FilesetIDTimestampIndex"" + ON ""{fs.m_tablename}"" ( + ""FilesetID"", + ""Timestamp"" DESC + ) + ", token) + .ConfigureAwait(false); } + + return fs; } + /// + /// Internal class representing a fileset. + /// private class Fileset : IFileset { + /// + /// The backup version this fileset is part of. + /// public long Version { get; private set; } + /// + /// Indicates whether this fileset is a full backup (1) or a partial backup (0). + /// public int IsFullBackup { get; set; } + /// + /// The timestamp of the fileset, indicating when it was created. + /// public DateTime Time { get; private set; } + /// + /// The count of files in this fileset. + /// public long FileCount { get; private set; } + /// + /// The total sizes of files in this fileset. + /// public long FileSizes { get; private set; } + /// + /// Initializes a new instance of the class. + /// + /// The backup version of the fileset. + /// Indicates whether this fileset is a full backup (1) or a partial backup (0). + /// The timestamp of the fileset, indicating when it was created. + /// The count of files in this fileset. + /// The total sizes of files in this fileset. public Fileset(long version, int isFullBackup, DateTime time, long filecount, long filesizes) { Version = version; @@ -104,130 +273,232 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Implementation of the interface. + /// private class Fileversion : IFileversion { - private readonly IDataReader m_reader; + /// + /// The SqliteDataReader used to read file version data. + /// + private readonly SqliteDataReader m_reader; + /// + /// The path of the file, or null if not available. + /// public string? Path { get; private set; } + /// + /// Indicates whether there are more file versions to read. + /// public bool More { get; private set; } - public Fileversion(IDataReader reader) + /// + /// Initializes a new instance of the class. + /// + /// The SqliteDataReader used to read file version data. + public Fileversion(SqliteDataReader reader) { m_reader = reader; Path = reader.ConvertValueToString(0); More = true; } - public IEnumerable Sizes + /// + public async IAsyncEnumerable Sizes([EnumeratorCancellation] CancellationToken token) { - get + while (More && Path == m_reader.ConvertValueToString(0)) { - while (More && Path == m_reader.ConvertValueToString(0)) - { - yield return m_reader.ConvertValueToInt64(1, -1); - More = m_reader.Read(); - } + yield return m_reader.ConvertValueToInt64(1, -1); + More = await m_reader.ReadAsync(token).ConfigureAwait(false); } } } + /// + /// Internal class representing a fixed file version with no sizes. + /// private class FileversionFixed : IFileversion { + /// public string? Path { get; internal set; } - public IEnumerable Sizes { get { return new long[0]; } } + /// + public IAsyncEnumerable Sizes(CancellationToken token) => AsyncEnumerable.Empty(); } - public IEnumerable GetLargestPrefix(IFilter filter) + /// + public IAsyncEnumerable GetLargestPrefix(IFilter filter, CancellationToken token) { - return GetLargestPrefix(filter, null); + return GetLargestPrefix(filter, null, token); } - private IEnumerable GetLargestPrefix(IFilter filter, string? prefixrule) + /// + /// Recursively finds the largest prefix of files in the filesets that match the provided filter. + /// + /// The filter to apply to the files. + /// The prefix rule to apply, or null if no prefix rule is specified. + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of file versions representing the largest prefix. + private async IAsyncEnumerable GetLargestPrefix(IFilter filter, string? prefixrule, [EnumeratorCancellation] CancellationToken token) { - using (var tmpnames = new FilteredFilenameTable(m_connection, filter, null)) - using (var cmd = m_connection.CreateCommand()) + await using var tmpnames = await FilteredFilenameTable + .CreateFilteredFilenameTableAsync(m_db, filter, token) + .ConfigureAwait(false); + await using var cmd = m_db.Connection.CreateCommand(); + //First we trim the filelist to exclude filenames not found in any of the filesets + await cmd.ExecuteNonQueryAsync($@" + DELETE FROM ""{tmpnames.Tablename}"" + WHERE ""Path"" NOT IN ( + SELECT DISTINCT ""Path"" + FROM + ""File"", + ""FilesetEntry"" + WHERE + ""FilesetEntry"".""FileID"" = ""File"".""ID"" + AND ""FilesetEntry"".""FilesetID"" IN ( + SELECT ""FilesetID"" + FROM ""{m_tablename}"" + ) + ) + ", token) + .ConfigureAwait(false); + + //If we have a prefix rule, apply it + if (!string.IsNullOrWhiteSpace(prefixrule)) + await cmd.SetCommandAndParameters($@" + DELETE FROM ""{tmpnames.Tablename}"" + WHERE SUBSTR(""Path"", 1, {prefixrule.Length}) != @Rule + ") + .SetParameterValue("@Rule", prefixrule) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + // Then we recursively find the largest prefix + var v0 = await cmd.ExecuteScalarAsync($@" + SELECT ""Path"" + FROM ""{tmpnames.Tablename}"" + ORDER BY LENGTH(""Path"") DESC + LIMIT 1 + ", token) + .ConfigureAwait(false); + + var maxpath = ""; + if (v0 != null) + maxpath = v0.ToString() ?? ""; + + var dirsep = Util.GuessDirSeparator(maxpath); + + var filecount = await cmd.ExecuteScalarInt64Async($@" + SELECT COUNT(*) + FROM ""{tmpnames.Tablename}"" + ", 0, token) + .ConfigureAwait(false); + + var foundfiles = -1L; + + //TODO: Handle FS case-sensitive? + await cmd.SetCommandAndParameters($@" + SELECT COUNT(*) + FROM ""{tmpnames.Tablename}"" + WHERE SUBSTR(""Path"", 1, @PrefixLength) = @Prefix + ") + .PrepareAsync(token) + .ConfigureAwait(false); + + while (filecount != foundfiles && maxpath.Length > 0) { - //First we trim the filelist to exclude filenames not found in any of the filesets - cmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) ")); + var mp = Util.AppendDirSeparator(maxpath, dirsep); + foundfiles = await cmd + .SetParameterValue("@PrefixLength", mp.Length) + .SetParameterValue("@Prefix", mp) + .ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false); - //If we have a prefix rule, apply it - if (!string.IsNullOrWhiteSpace(prefixrule)) - cmd.SetCommandAndParameters(FormatInvariant($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE SUBSTR(""Path"", 1, {prefixrule.Length}) != @Rule")) - .SetParameterValue("@Rule", prefixrule) - .ExecuteNonQuery(); - - // Then we recursively find the largest prefix - var v0 = cmd.ExecuteScalar(FormatInvariant($@"SELECT ""Path"" FROM ""{tmpnames.Tablename}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1")); - var maxpath = ""; - if (v0 != null) - maxpath = v0.ToString() ?? ""; - - var dirsep = Util.GuessDirSeparator(maxpath); - - var filecount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""{tmpnames.Tablename}"""), 0); - var foundfiles = -1L; - - //TODO: Handle FS case-sensitive? - cmd.SetCommandAndParameters(FormatInvariant($@"SELECT COUNT(*) FROM ""{tmpnames.Tablename}"" WHERE SUBSTR(""Path"", 1, @PrefixLength) = @Prefix")); - - while (filecount != foundfiles && maxpath.Length > 0) + if (filecount != foundfiles) { - var mp = Util.AppendDirSeparator(maxpath, dirsep); - foundfiles = cmd.SetParameterValue("@PrefixLength", mp.Length) - .SetParameterValue("@Prefix", mp) - .ExecuteScalarInt64(0); + var oldlen = maxpath.Length; + var lix = maxpath.LastIndexOf(dirsep, maxpath.Length - 2, StringComparison.Ordinal); - if (filecount != foundfiles) - { - var oldlen = maxpath.Length; - var lix = maxpath.LastIndexOf(dirsep, maxpath.Length - 2, StringComparison.Ordinal); - - maxpath = maxpath.Substring(0, lix + 1); - if (string.IsNullOrWhiteSpace(maxpath) || maxpath.Length == oldlen || maxpath == "\\\\") - maxpath = ""; - } + maxpath = maxpath.Substring(0, lix + 1); + if (string.IsNullOrWhiteSpace(maxpath) || maxpath.Length == oldlen || maxpath == "\\\\") + maxpath = ""; } - - // Special handling for Windows and multi-drive/UNC backups as they do not have a single common root - if (string.IsNullOrWhiteSpace(maxpath) && string.IsNullOrWhiteSpace(prefixrule)) - { - var paths = cmd.ExecuteReaderEnumerable(FormatInvariant($@"SELECT Path FROM ""{tmpnames.Tablename}""")).Select(x => x.ConvertValueToString(0) ?? "").ToArray(); - var roots = paths.Select(x => x.Substring(0, 1)).Distinct().Where(x => x != "\\").ToArray(); - - //unc path like \\server.domain\ - var regexUNCPrefix = new System.Text.RegularExpressions.Regex(@"^\\\\.*?\\"); - var rootsUNC = paths.Select(x => regexUNCPrefix.Match(x)).Where(x => x.Success).Select(x => x.Value).Distinct().ToArray(); - - return roots.Concat(rootsUNC).Select(x => GetLargestPrefix(filter, x).First()).Distinct().ToArray(); - } - - return [new FileversionFixed { Path = maxpath == "" ? "" : Util.AppendDirSeparator(maxpath, dirsep) }]; } + + // Special handling for Windows and multi-drive/UNC backups as they do not have a single common root + if (string.IsNullOrWhiteSpace(maxpath) && string.IsNullOrWhiteSpace(prefixrule)) + { + var paths = cmd.ExecuteReaderEnumerableAsync($@" + SELECT ""Path"" + FROM ""{tmpnames.Tablename}"" + ", token) + .Select(x => x.ConvertValueToString(0) ?? ""); + + var roots = paths + .Select(x => x.Substring(0, 1)) + .Distinct() + .Where(x => x != "\\"); + + //unc path like \\server.domain\ + var regexUNCPrefix = new System.Text.RegularExpressions.Regex(@"^\\\\.*?\\"); + var rootsUNC = paths + .Select(x => regexUNCPrefix.Match(x)) + .Where(x => x.Success) + .Select(x => x.Value) + .Distinct(); + + var result = roots + .Concat(rootsUNC) + .Select(x => GetLargestPrefix(filter, x, token) + .FirstAsync()) + .Distinct(); + + await foreach (var el in result.ConfigureAwait(false)) + yield return await el; + } + + yield return new FileversionFixed + { + Path = maxpath == "" ? "" : Util.AppendDirSeparator(maxpath, dirsep) + }; } - private IEnumerable SelectFolderEntries(IDbCommand cmd, string prefix, string table) + /// + /// Asynchronously retrieves all folder entries from the specified table that match the given prefix. + /// + /// The SqliteCommand to execute. + /// The prefix to filter folder entries by. + /// The name of the table to query. + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of folder entries. + private static async IAsyncEnumerable SelectFolderEntries(SqliteCommand cmd, string prefix, string table, [EnumeratorCancellation] CancellationToken token) { if (!string.IsNullOrEmpty(prefix)) prefix = Util.AppendDirSeparator(prefix, Util.GuessDirSeparator(prefix)); var ppl = prefix.Length; - using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT DISTINCT ""Path"" FROM ""{table}"" "))) - while (rd.Read()) - { - var s = rd.ConvertValueToString(0) ?? ""; - if (!s.StartsWith(prefix, StringComparison.Ordinal)) - continue; + cmd.SetCommandAndParameters($@" + SELECT DISTINCT ""Path"" + FROM ""{table}"" + "); + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + { + var s = rd.ConvertValueToString(0) ?? ""; + if (!s.StartsWith(prefix, StringComparison.Ordinal)) + continue; - var dirsep = Util.GuessDirSeparator(s); + var dirsep = Util.GuessDirSeparator(s); - s = s.Substring(ppl); - var ix = s.IndexOf(dirsep, StringComparison.Ordinal); - if (ix > 0 && ix != s.Length - 1) - s = s.Substring(0, ix + 1); - yield return prefix + s; - } + s = s.Substring(ppl); + var ix = s.IndexOf(dirsep, StringComparison.Ordinal); + if (ix > 0 && ix != s.Length - 1) + s = s.Substring(0, ix + 1); + + yield return prefix + s; + } } - public IEnumerable SelectFolderContents(IFilter filter) + /// + public async IAsyncEnumerable SelectFolderContents(IFilter filter, [EnumeratorCancellation] CancellationToken token) { var tbname = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); try @@ -235,7 +506,9 @@ namespace Duplicati.Library.Main.Database string pathprefix; if (filter == null || filter.Empty) pathprefix = ""; - else if (filter as FilterExpression == null || ((FilterExpression)filter).Type != FilterType.Simple || ((FilterExpression)filter).GetSimpleList().Length != 1) + else if (filter as FilterExpression == null + || ((FilterExpression)filter).Type != FilterType.Simple + || ((FilterExpression)filter).GetSimpleList().Length != 1) throw new ArgumentException("Filter for list-folder-contents must be a path prefix with no wildcards", nameof(filter)); else pathprefix = ((FilterExpression)filter).GetSimpleList().First(); @@ -245,67 +518,145 @@ namespace Duplicati.Library.Main.Database if (pathprefix.Length > 0 || dirsep == "/") pathprefix = Util.AppendDirSeparator(pathprefix, dirsep); - using (var tmpnames = new FilteredFilenameTable(m_connection, new FilterExpression(new string[] { pathprefix + "*" }, true), null)) - using (var cmd = m_connection.CreateCommand()) + await using var tmpnames = await FilteredFilenameTable + .CreateFilteredFilenameTableAsync(m_db, new FilterExpression(new string[] { pathprefix + "*" }, true), token) + .ConfigureAwait(false); + await using var cmd = m_db.Connection.CreateCommand(); + //First we trim the filelist to exclude filenames not found in any of the filesets + await cmd.ExecuteNonQueryAsync($@" + DELETE FROM ""{tmpnames.Tablename}"" + WHERE ""Path"" NOT IN ( + SELECT DISTINCT ""Path"" + FROM + ""File"", + ""FilesetEntry"" + WHERE + ""FilesetEntry"".""FileID"" = ""File"".""ID"" + AND ""FilesetEntry"".""FilesetID"" IN ( + SELECT ""FilesetID"" + FROM ""{m_tablename}"" + ) + ) + ", token) + .ConfigureAwait(false); + + // If we had instr support this would work: + /*var distinctPaths = @"SELECT DISTINCT :1 || " + + @"CASE(INSTR(SUBSTR(""Path"", :2), '/')) " + + @"WHEN 0 THEN SUBSTR(""Path"", :2) " + + @"ELSE SUBSTR(""Path"", :2, INSTR(SUBSTR(path, :2), '/')) " + + @"END AS ""Path"", ""FilesetID"" " + + @" FROM (" + cartesianPathFileset + @")";*/ + + // Instead we manually iterate the paths + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{tbname}"" ( + ""Path"" TEXT NOT NULL + ) + ", token) + .ConfigureAwait(false); + + await using (var c2 = m_db.Connection.CreateCommand()) { - //First we trim the filelist to exclude filenames not found in any of the filesets - cmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) ")); + c2.SetCommandAndParameters($@" + INSERT INTO ""{tbname}"" (""Path"") + VALUES (@Path) + "); - // If we had instr support this would work: - /*var distinctPaths = @"SELECT DISTINCT :1 || " + - @"CASE(INSTR(SUBSTR(""Path"", :2), '/')) " + - @"WHEN 0 THEN SUBSTR(""Path"", :2) " + - @"ELSE SUBSTR(""Path"", :2, INSTR(SUBSTR(path, :2), '/')) " + - @"END AS ""Path"", ""FilesetID"" " + - @" FROM (" + cartesianPathFileset + @")";*/ + await foreach (var n in SelectFolderEntries(cmd, pathprefix, tmpnames.Tablename, token).Distinct().ConfigureAwait(false)) + await c2 + .SetParameterValue("@Path", n) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); - // Instead we manually iterate the paths - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tbname}"" (""Path"" TEXT NOT NULL)")); + await c2.ExecuteNonQueryAsync($@" + CREATE INDEX ""{tbname}_PathIndex"" + ON ""{tbname}"" (""Path"") + ", token) + .ConfigureAwait(false); + } - using (var c2 = m_connection.CreateCommand()) + //Then we select the matching results + var filesets = $@" + SELECT + ""FilesetID"", + ""Timestamp"" + FROM ""{m_tablename}"" + ORDER BY ""Timestamp"" DESC + "; + + var cartesianPathFileset = $@" + SELECT + ""A"".""Path"", + ""B"".""FilesetID"" + FROM + ""{tbname}"" ""A"", + ({filesets}) ""B"" + ORDER BY + ""A"".""Path"" ASC, + ""B"".""Timestamp"" DESC + "; + + var filesWithSizes = $@" + SELECT + ""Length"", + ""FilesetEntry"".""FilesetID"", + ""File"".""Path"" + FROM + ""Blockset"", + ""FilesetEntry"", + ""File"" + WHERE + ""File"".""BlocksetID"" = ""Blockset"".""ID"" + AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" + AND FilesetEntry.""FilesetID"" IN ( + SELECT DISTINCT ""FilesetID"" + FROM ""{m_tablename}"" + ) + "; + + var query = @$" + SELECT + ""C"".""Path"", + ""D"".""Length"", + ""C"".""FilesetID"" + FROM ({cartesianPathFileset}) ""C"" + LEFT OUTER JOIN ({filesWithSizes}) ""D"" + ON ""C"".""FilesetID"" = ""D"".""FilesetID"" + AND ""C"".""Path"" = ""D"".""Path"" + "; + + await using var rd = await cmd + .ExecuteReaderAsync(query, token) + .ConfigureAwait(false); + + if (await rd.ReadAsync(token).ConfigureAwait(false)) + { + bool more; + do { - c2.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{tbname}"" (""Path"") VALUES (@Path)")); - foreach (var n in SelectFolderEntries(cmd, pathprefix, tmpnames.Tablename).Distinct()) - c2.SetParameterValue("@Path", n) - .ExecuteNonQuery(); - - c2.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{tbname}_PathIndex"" ON ""{tbname}"" (""Path"")")); - } - - //Then we select the matching results - var filesets = FormatInvariant($@"SELECT ""FilesetID"", ""Timestamp"" FROM ""{m_tablename}"" ORDER BY ""Timestamp"" DESC"); - var cartesianPathFileset = FormatInvariant($@"SELECT ""A"".""Path"", ""B"".""FilesetID"" FROM ""{tbname}"" A, ({filesets}) B ORDER BY ""A"".""Path"" ASC, ""B"".""Timestamp"" DESC"); - - var filesWithSizes = FormatInvariant($@"SELECT ""Length"", ""FilesetEntry"".""FilesetID"", ""File"".""Path"" FROM ""Blockset"", ""FilesetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND FilesetEntry.""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") "); - var query = @"SELECT ""C"".""Path"", ""D"".""Length"", ""C"".""FilesetID"" FROM (" + cartesianPathFileset + @") C LEFT OUTER JOIN (" + filesWithSizes + @") D ON ""C"".""FilesetID"" = ""D"".""FilesetID"" AND ""C"".""Path"" = ""D"".""Path"""; - - using (var rd = cmd.ExecuteReader(query)) - if (rd.Read()) + var f = new Fileversion(rd); + if (!(string.IsNullOrWhiteSpace(f.Path) || f.Path == pathprefix)) { - bool more; - do - { - var f = new Fileversion(rd); - if (!(string.IsNullOrWhiteSpace(f.Path) || f.Path == pathprefix)) - { - yield return f; - more = f.More; - } - else - { - more = rd.Read(); - } - - } while (more); + yield return f; + more = f.More; } + else + { + more = await rd.ReadAsync(token) + .ConfigureAwait(false); + } + + } while (more); } } finally { try { - using (var c = m_connection.CreateCommand()) - c.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tbname}""")); + await using var c = m_db.Connection.CreateCommand(); + await c.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tbname}""", token) + .ConfigureAwait(false); } catch { @@ -313,157 +664,288 @@ namespace Duplicati.Library.Main.Database } } - public IEnumerable SelectFiles(IFilter filter) + /// + public async IAsyncEnumerable SelectFiles(IFilter filter, [EnumeratorCancellation] CancellationToken token) { - using (var tmpnames = new FilteredFilenameTable(m_connection, filter, null)) - using (var cmd = m_connection.CreateCommand()) - { - //First we trim the filelist to exclude filenames not found in any of the filesets - cmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{tmpnames.Tablename}"" WHERE ""Path"" NOT IN (SELECT DISTINCT ""Path"" FROM ""File"", ""FilesetEntry"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""FilesetEntry"".""FilesetID"" IN (SELECT ""FilesetID"" FROM ""{m_tablename}"") ) ")); + await using var tmpnames = await FilteredFilenameTable + .CreateFilteredFilenameTableAsync(m_db, filter, token) + .ConfigureAwait(false); + await using var cmd = m_db.Connection.CreateCommand(); + //First we trim the filelist to exclude filenames not found in any of the filesets + await cmd.ExecuteNonQueryAsync($@" + DELETE FROM ""{tmpnames.Tablename}"" + WHERE ""Path"" NOT IN ( + SELECT DISTINCT ""Path"" + FROM + ""File"", + ""FilesetEntry"" + WHERE + ""FilesetEntry"".""FileID"" = ""File"".""ID"" + AND ""FilesetEntry"".""FilesetID"" IN ( + SELECT ""FilesetID"" + FROM ""{m_tablename}"" + ) + ) + ", token) + .ConfigureAwait(false); - //Then we select the matching results - var filesets = FormatInvariant($@"SELECT ""FilesetID"", ""Timestamp"" FROM ""{m_tablename}"" ORDER BY ""Timestamp"" DESC"); - var cartesianPathFileset = FormatInvariant($@"SELECT ""A"".""Path"", ""B"".""FilesetID"" FROM ""{tmpnames.Tablename}"" A, ({filesets}) B ORDER BY ""A"".""Path"" ASC, ""B"".""Timestamp"" DESC"); - var filesWithSizes = FormatInvariant($@"SELECT ""Length"", ""FilesetEntry"".""FilesetID"", ""File"".""Path"" FROM ""Blockset"", ""FilesetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND FilesetEntry.""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") "); - var query = @"SELECT ""C"".""Path"", ""D"".""Length"", ""C"".""FilesetID"" FROM (" + cartesianPathFileset + @") C LEFT OUTER JOIN (" + filesWithSizes + @") D ON ""C"".""FilesetID"" = ""D"".""FilesetID"" AND ""C"".""Path"" = ""D"".""Path"""; - using (var rd = cmd.ExecuteReader(query)) - if (rd.Read()) - { - bool more; - do - { - var f = new Fileversion(rd); - yield return f; - more = f.More; - } while (more); - } + //Then we select the matching results + var filesets = $@" + SELECT + ""FilesetID"", + ""Timestamp"" + FROM ""{m_tablename}"" + ORDER BY ""Timestamp"" DESC + "; + + var cartesianPathFileset = $@" + SELECT + ""A"".""Path"", + ""B"".""FilesetID"" + FROM + ""{tmpnames.Tablename}"" ""A"", + ({filesets}) ""B"" + ORDER BY + ""A"".""Path"" ASC, + ""B"".""Timestamp"" DESC + "; + + var filesWithSizes = $@" + SELECT + ""Length"", + ""FilesetEntry"".""FilesetID"", + ""File"".""Path"" + FROM + ""Blockset"", + ""FilesetEntry"", + ""File"" + WHERE + ""File"".""BlocksetID"" = ""Blockset"".""ID"" + AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" + AND FilesetEntry.""FilesetID"" IN ( + SELECT DISTINCT ""FilesetID"" + FROM ""{m_tablename}"" + ) + "; + + var query = @$" + SELECT + ""C"".""Path"", + ""D"".""Length"", + ""C"".""FilesetID"" + FROM ({cartesianPathFileset}) ""C"" + LEFT OUTER JOIN ({filesWithSizes}) ""D"" + ON ""C"".""FilesetID"" = ""D"".""FilesetID"" + AND ""C"".""Path"" = ""D"".""Path"" + "; + + await using var rd = await cmd.ExecuteReaderAsync(query, token) + .ConfigureAwait(false); + if (await rd.ReadAsync(token).ConfigureAwait(false)) + { + bool more; + do + { + var f = new Fileversion(rd); + yield return f; + more = f.More; + } while (more); } } - public void TakeFirst() + /// + public async Task TakeFirst(CancellationToken token) { - using (var cmd = m_connection.CreateCommand()) - cmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{m_tablename}"" WHERE ""FilesetID"" NOT IN (SELECT ""FilesetID"" FROM ""{m_tablename}"" ORDER BY ""Timestamp"" DESC LIMIT 1 )")); - } - - public IEnumerable QuickSets - { - get - { - var dict = new Dictionary(); - for (var i = 0; i < m_filesets.Length; i++) - dict[m_filesets[i].Key] = i; - - using (var cmd = m_connection.CreateCommand()) - using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""ID"", ""IsFullBackup"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC ")) - while (rd.Read()) - { - var id = rd.ConvertValueToInt64(0); - var backupType = rd.GetInt32(1); - var e = dict[id]; - yield return new Fileset(e, backupType, m_filesets[e].Value, -1L, -1L); - } - } - } - - public IEnumerable Sets - { - get - { - var dict = new Dictionary(); - for (var i = 0; i < m_filesets.Length; i++) - dict[m_filesets[i].Key] = i; - - var summation = FormatInvariant( - $@"SELECT ""A"".""FilesetID"" AS ""FilesetID"", COUNT(*) AS ""FileCount"", SUM(""C"".""Length"") AS ""FileSizes"" FROM ""FilesetEntry"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" AND ""A"".""FilesetID"" IN (SELECT DISTINCT ""FilesetID"" FROM ""{m_tablename}"") GROUP BY ""A"".""FilesetID"" "); - - using (var cmd = m_connection.CreateCommand()) - using (var rd = cmd.ExecuteReader(FormatInvariant( - $@"SELECT DISTINCT ""A"".""FilesetID"", ""A"".""IsFullBackup"", ""B"".""FileCount"", ""B"".""FileSizes"" FROM ""{m_tablename}"" A LEFT OUTER JOIN ( {summation} ) B ON ""A"".""FilesetID"" = ""B"".""FilesetID"" ORDER BY ""A"".""Timestamp"" DESC ")) + await using var cmd = m_db.Connection.CreateCommand(); + await cmd.ExecuteNonQueryAsync($@" + DELETE FROM ""{m_tablename}"" + WHERE ""FilesetID"" NOT IN ( + SELECT ""FilesetID"" + FROM ""{m_tablename}"" + ORDER BY ""Timestamp"" DESC + LIMIT 1 ) - while (rd.Read()) - { - var id = rd.ConvertValueToInt64(0); - var isFullBackup = rd.GetInt32(1); - var e = dict[id]; - var filecount = rd.ConvertValueToInt64(2, -1L); - var filesizes = rd.ConvertValueToInt64(3, -1L); + ", token) + .ConfigureAwait(false); + } - yield return new Fileset(e, isFullBackup, m_filesets[e].Value, filecount, filesizes); - } + /// + public async IAsyncEnumerable QuickSets([EnumeratorCancellation] CancellationToken token) + { + var dict = new Dictionary(); + for (var i = 0; i < m_filesets.Length; i++) + dict[m_filesets[i].Key] = i; + + await using var cmd = m_db.Connection.CreateCommand(); + await using var rd = await cmd.ExecuteReaderAsync(@" + SELECT DISTINCT + ""ID"", + ""IsFullBackup"" + FROM ""Fileset"" + ORDER BY ""Timestamp"" DESC + ", token) + .ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + { + var id = rd.ConvertValueToInt64(0); + var backupType = rd.GetInt32(1); + var e = dict[id]; + + yield return new Fileset( + e, + backupType, + m_filesets[e].Value, + -1L, + -1L + ); + } + } + + /// + public async IAsyncEnumerable Sets([EnumeratorCancellation] CancellationToken token) + { + var dict = new Dictionary(); + for (var i = 0; i < m_filesets.Length; i++) + dict[m_filesets[i].Key] = i; + + var summation = $@" + SELECT + ""A"".""FilesetID"" AS ""FilesetID"", + COUNT(*) AS ""FileCount"", + SUM(""C"".""Length"") AS ""FileSizes"" + FROM + ""FilesetEntry"" ""A"", + ""File"" ""B"", + ""Blockset"" ""C"" + WHERE + ""A"".""FileID"" = ""B"".""ID"" + AND ""B"".""BlocksetID"" = ""C"".""ID"" + AND ""A"".""FilesetID"" IN ( + SELECT DISTINCT ""FilesetID"" + FROM ""{m_tablename}"") + GROUP BY ""A"".""FilesetID"" + "; + + await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + await using var rd = await cmd.ExecuteReaderAsync($@" + SELECT DISTINCT + ""A"".""FilesetID"", + ""A"".""IsFullBackup"", + ""B"".""FileCount"", + ""B"".""FileSizes"" + FROM ""{m_tablename}"" ""A"" + LEFT OUTER JOIN ( {summation} ) ""B"" + ON ""A"".""FilesetID"" = ""B"".""FilesetID"" + ORDER BY ""A"".""Timestamp"" DESC + ", token) + .ConfigureAwait(false); + + while (await rd.ReadAsync(token).ConfigureAwait(false)) + { + var id = rd.ConvertValueToInt64(0); + var isFullBackup = rd.GetInt32(1); + var e = dict[id]; + var filecount = rd.ConvertValueToInt64(2, -1L); + var filesizes = rd.ConvertValueToInt64(3, -1L); + + yield return new Fileset( + e, + isFullBackup, + m_filesets[e].Value, + filecount, + filesizes + ); } } public void Dispose() + { + DisposeAsync().AsTask().Await(); + } + + public async ValueTask DisposeAsync() { if (m_tablename != null) { try { - using (var cmd = m_connection.CreateCommand()) - cmd.ExecuteNonQuery(FormatInvariant(@$"DROP TABLE IF EXISTS ""{m_tablename}"" ")); + await using var cmd = m_db.Connection.CreateCommand(); + await cmd + .ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tablename}"" ", default) + .ConfigureAwait(false); } catch { } finally { m_tablename = null!; } } - } } - public IFileSets SelectFileSets(DateTime time, long[] versions) + /// + /// Creates a new instance of with the specified time and versions. + /// + /// The time to filter filesets by. + /// An array of versions to filter filesets by. + /// A task that when awaited returns a new instance of . + public async Task SelectFileSets(DateTime time, long[] versions, CancellationToken token) { - return new FileSets(this, time, versions); + return await FileSets.CreateAsync(this, time, versions, token) + .ConfigureAwait(false); } /// /// Represents a fileset entry in the list fileset results. /// - /// The fileset version + /// The fileset version. /// The fileset ID - /// Flag indicating if the backup is a full backup or synthetic backup - /// The timestamp of the fileset - /// The number of files in the fileset - /// >The total size of files in the fileset + /// Flag indicating if the backup is a full backup or synthetic backup. + /// The timestamp of the fileset. + /// The number of files in the fileset. + /// >The total size of files in the fileset. private sealed record FilesetEntry(long Version, long ID, bool? IsFullBackup, DateTime Time, long? FileCount, long? FileSizes) : IListFilesetResultFileset; /// - /// Lists all filesets with summary data + /// Lists all filesets with summary data. /// - /// A collection of fileset entries - public IEnumerable ListFilesetsExtended() + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of fileset entries. + public async IAsyncEnumerable ListFilesetsExtended([EnumeratorCancellation] CancellationToken token) { const string query = @" -SELECT - f.""ID"" AS ""FilesetID"", - f.""Timestamp"", - f.""IsFullBackup"", - COUNT(fe.""FileID"") AS ""NumberOfFiles"", - COALESCE(SUM(b.""Length""), 0) AS ""TotalFileSize"" -FROM - ""Fileset"" f -LEFT JOIN - ""FilesetEntry"" fe ON f.""ID"" = fe.""FilesetID"" -LEFT JOIN - ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" -LEFT JOIN - ""Blockset"" b ON fl.""BlocksetID"" = b.""ID"" -GROUP BY - f.""ID"" -ORDER BY - f.""Timestamp"" DESC; -"; + SELECT + ""f"".""ID"" AS ""FilesetID"", + ""f"".""Timestamp"", + ""f"".""IsFullBackup"", + COUNT(""fe"".""FileID"") AS ""NumberOfFiles"", + COALESCE(SUM(""b"".""Length""), 0) AS ""TotalFileSize"" + FROM ""Fileset"" ""f"" + LEFT JOIN ""FilesetEntry"" ""fe"" + ON ""f"".""ID"" = ""fe"".""FilesetID"" + LEFT JOIN ""FileLookup"" ""fl"" + ON ""fe"".""FileID"" = ""fl"".""ID"" + LEFT JOIN ""Blockset"" ""b"" + ON ""fl"".""BlocksetID"" = ""b"".""ID"" + GROUP BY ""f"".""ID"" + ORDER BY ""f"".""Timestamp"" DESC; + "; var version = 0L; - using (var cmd = m_connection.CreateCommand(query)) - foreach (var rd in cmd.ExecuteReaderEnumerable()) - { - var id = rd.ConvertValueToInt64(0); - var time = ParseFromEpochSeconds(rd.ConvertValueToInt64(1)); - var isFullBackup = rd.GetInt32(2) == BackupType.FULL_BACKUP; - var filecount = rd.ConvertValueToInt64(3, -1L); - var filesizes = rd.ConvertValueToInt64(4, -1L); - yield return new FilesetEntry(version++, id, isFullBackup, time, filecount, filesizes); - } + await using var cmd = m_connection.CreateCommand(query); + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) + { + var id = rd.ConvertValueToInt64(0); + var time = ParseFromEpochSeconds(rd.ConvertValueToInt64(1)); + var isFullBackup = rd.GetInt32(2) == BackupType.FULL_BACKUP; + var filecount = rd.ConvertValueToInt64(3, -1L); + var filesizes = rd.ConvertValueToInt64(4, -1L); + + yield return new FilesetEntry( + version++, id, + isFullBackup, + time, + filecount, + filesizes + ); + } } /// @@ -483,44 +965,61 @@ ORDER BY /// The fileset ID to filter the folder entries. /// The offset for pagination. /// >The limit for pagination. - /// A paginated result set of folder entries. - public IPaginatedResults ListFolder(IEnumerable prefixIds, long filesetid, long offset, long limit) + /// A cancellation token to cancel the operation. + /// A task that when awaited returns a paginated result set of folder entries. + public async Task> ListFolder(IEnumerable prefixIds, long filesetid, long offset, long limit, CancellationToken token) { if (offset != 0 && limit <= 0) throw new ArgumentException("Cannot use offset without limit specified.", nameof(offset)); if (limit <= 0) limit = long.MaxValue; - using var cmd = m_connection.CreateCommand(); + await using var cmd = m_connection.CreateCommand(); if (prefixIds.Count() == 0) return new PaginatedResults(0, (int)limit, 0, 0, Enumerable.Empty()); // Then query the matching files cmd.SetCommandAndParameters($@" - SELECT - pp.""Prefix"" || fl.""Path"" AS ""FullPath"", - b.""Length"", - (CASE WHEN fl.""BlocksetID"" = @FolderBlocksetId THEN 1 ELSE 0 END) AS ""IsDirectory"", - (CASE WHEN fl.""BlocksetID"" = @SymlinkBlocksetId THEN 1 ELSE 0 END) AS ""IsSymlink"", - fe.""Lastmodified"" - FROM ""FilesetEntry"" fe - INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" - INNER JOIN ""PathPrefix"" pp ON fl.""PrefixID"" = pp.""ID"" - LEFT JOIN ""Blockset"" b ON fl.""BlocksetID"" = b.""ID"" - WHERE fe.""FilesetID"" = @filesetid AND fl.""PrefixID"" IN (@PrefixIds) - ORDER BY pp.""Prefix"" ASC, fl.""Path"" ASC - LIMIT @limit OFFSET @offset - ") + SELECT + ""pp"".""Prefix"" || ""fl"".""Path"" AS ""FullPath"", + ""b"".""Length"", + (CASE + WHEN ""fl"".""BlocksetID"" = @FolderBlocksetId + THEN 1 + ELSE 0 + END) AS ""IsDirectory"", + (CASE + WHEN ""fl"".""BlocksetID"" = @SymlinkBlocksetId + THEN 1 + ELSE 0 + END) AS ""IsSymlink"", + ""fe"".""Lastmodified"" + FROM ""FilesetEntry"" ""fe"" + INNER JOIN ""FileLookup"" ""fl"" + ON ""fe"".""FileID"" = ""fl"".""ID"" + INNER JOIN ""PathPrefix"" ""pp"" + ON ""fl"".""PrefixID"" = ""pp"".""ID"" + LEFT JOIN ""Blockset"" ""b"" + ON ""fl"".""BlocksetID"" = ""b"".""ID"" + WHERE + ""fe"".""FilesetID"" = @filesetid + AND ""fl"".""PrefixID"" IN (@PrefixIds) + ORDER BY + ""pp"".""Prefix"" ASC, + ""fl"".""Path"" ASC + LIMIT @limit + OFFSET @offset + ") .SetParameterValue("@FolderBlocksetId", FOLDER_BLOCKSET_ID) .SetParameterValue("@SymlinkBlocksetId", SYMLINK_BLOCKSET_ID) - .ExpandInClauseParameter("@PrefixIds", prefixIds) + .ExpandInClauseParameterMssqlite("@PrefixIds", prefixIds) .SetParameterValue("@filesetid", filesetid) .SetParameterValue("@limit", limit) .SetParameterValue("@offset", offset); var results = new List(); - foreach (var rd in cmd.ExecuteReaderEnumerable()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) { var path = rd.ConvertValueToString(0) ?? string.Empty; var size = rd.ConvertValueToInt64(-1); @@ -528,7 +1027,13 @@ ORDER BY var isSymlink = rd.GetInt32(3) != 0; var lastModified = new DateTime(rd.ConvertValueToInt64(4, 0), DateTimeKind.Utc); - results.Add(new FolderEntry(path, size, isDir, isSymlink, lastModified)); + results.Add(new FolderEntry( + path, + size, + isDir, + isSymlink, + lastModified + )); } long totalCount; @@ -545,16 +1050,19 @@ ORDER BY else { // Get the total count of files for the given fileset and prefix IDs - totalCount = cmd.SetCommandAndParameters($@" + totalCount = await cmd.SetCommandAndParameters($@" SELECT COUNT(*) - FROM FilesetEntry fe - INNER JOIN FileLookup fl ON fe.FileID = fl.ID - WHERE fe.FilesetID = @filesetid - AND fl.PrefixID IN (@PrefixIds) + FROM ""FilesetEntry"" ""fe"" + INNER JOIN ""FileLookup"" ""fl"" + ON ""fe"".""FileID"" = ""fl"".""ID"" + WHERE + ""fe"".""FilesetID"" = @filesetid + AND ""fl"".""PrefixID"" IN (@PrefixIds) ") - .ExpandInClauseParameter("@PrefixIds", prefixIds) - .SetParameterValue("@filesetid", filesetid) - .ExecuteScalarInt64(0); + .ExpandInClauseParameterMssqlite("@PrefixIds", prefixIds) + .SetParameterValue("@filesetid", filesetid) + .ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false); } // Return paginated results @@ -567,63 +1075,83 @@ ORDER BY /// Gets the prefix IDs for a given set of folder-prefixes. /// /// The prefixes to get the IDs for. - /// >A list of prefix IDs. - public IEnumerable GetPrefixIds(IEnumerable prefixes) + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of prefix IDs. + public async IAsyncEnumerable GetPrefixIds(IEnumerable prefixes, [EnumeratorCancellation] CancellationToken token) { - using var cmd = m_connection.CreateCommand(); + await using var cmd = m_connection.CreateCommand(); // Map folders to prefix IDs - cmd.SetCommandAndParameters($@"SELECT ""ID"" FROM ""PathPrefix"" WHERE ""Prefix"" IN (@Prefixes)") + cmd.SetCommandAndParameters($@" + SELECT ""ID"" + FROM ""PathPrefix"" + WHERE ""Prefix"" IN (@Prefixes) + ") .ExpandInClauseParameter("@Prefixes", prefixes); - return cmd.ExecuteReaderEnumerable() - .Select(x => x.ConvertValueToInt64(0)) - .ToList(); + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) + { + yield return rd.ConvertValueToInt64(0); + } } /// /// Gets the minimal unique prefix entries for a given fileset ID. /// This returns the roots of all unique paths in the fileset. /// - /// The fileset id - /// The entries that are unique roots - public IList GetMinimalUniquePrefixEntries(long filesetId) + /// The fileset id. + /// A cancellation token to cancel the operation. + /// A task that when awaited returns a list of the entries that are unique roots. + public async IAsyncEnumerable GetMinimalUniquePrefixEntries(long filesetId, [EnumeratorCancellation] CancellationToken token) { - var results = new List(); - - using var cmd = m_connection.CreateCommand(); + await using var cmd = m_connection.CreateCommand(); cmd.SetCommandAndParameters(@" WITH AllPaths AS ( - SELECT fl.""ID"" AS ""FileID"", + SELECT + fl.""ID"" AS ""FileID"", pp.""Prefix"" || fl.""Path"" AS ""FullPath"", - fl.""BlocksetID"", - b.""Length"", + fl.""BlocksetID"", + b.""Length"", fe.""Lastmodified"" FROM ""FilesetEntry"" fe - JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" - JOIN ""PathPrefix"" pp ON fl.""PrefixID"" = pp.""ID"" - LEFT JOIN ""Blockset"" b ON fl.""BlocksetID"" = b.""ID"" + JOIN ""FileLookup"" fl + ON fe.""FileID"" = fl.""ID"" + JOIN ""PathPrefix"" pp + ON fl.""PrefixID"" = pp.""ID"" + LEFT JOIN ""Blockset"" b + ON fl.""BlocksetID"" = b.""ID"" WHERE fe.""FilesetID"" = @FilesetId ), RootCandidates AS ( SELECT a1.* FROM AllPaths a1 WHERE NOT EXISTS ( - SELECT 1 FROM AllPaths a2 - WHERE a2.""FullPath"" != a1.""FullPath"" - AND a1.""FullPath"" LIKE a2.""FullPath"" || '%' + SELECT 1 + FROM AllPaths a2 + WHERE + a2.""FullPath"" != a1.""FullPath"" + AND a1.""FullPath"" LIKE a2.""FullPath"" || '%' ) ) SELECT ""FullPath"", ""Length"", - CASE WHEN ""BlocksetID"" = -100 THEN 1 ELSE 0 END AS ""IsDirectory"", - CASE WHEN ""BlocksetID"" = -200 THEN 1 ELSE 0 END AS ""IsSymlink"", + CASE + WHEN ""BlocksetID"" = -100 + THEN 1 + ELSE 0 + END AS ""IsDirectory"", + CASE + WHEN ""BlocksetID"" = -200 + THEN 1 + ELSE 0 + END AS ""IsSymlink"", ""Lastmodified"" FROM RootCandidates ORDER BY ""FullPath"" - "); - cmd.SetParameterValue("@FilesetId", filesetId); + ") + .SetTransaction(m_rtr) + .SetParameterValue("@FilesetId", filesetId); - foreach (var rd in cmd.ExecuteReaderEnumerable()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token)) { var path = rd.ConvertValueToString(0) ?? string.Empty; var size = rd.ConvertValueToInt64(1, -1); @@ -631,10 +1159,8 @@ ORDER BY var isSymlink = rd.GetInt32(3) != 0; var lastModified = new DateTime(rd.ConvertValueToInt64(4, 0), DateTimeKind.Utc); - results.Add(new FolderEntry(path, size, isDir, isSymlink, lastModified)); + yield return new FolderEntry(path, size, isDir, isSymlink, lastModified); } - - return results; } /// @@ -644,8 +1170,9 @@ ORDER BY /// Optional fileset IDs to restrict the search to. /// Pagination offset. /// Pagination limit. - /// All matching file versions ordered by path and then version time. - public IPaginatedResults ListFileVersions(IEnumerable paths, long[]? filesetIds, long offset, long limit) + /// A cancellation token to cancel the operation. + /// A task that when awaited returns all matching file versions ordered by path and then version time. + public async Task> ListFileVersions(IEnumerable paths, long[]? filesetIds, long offset, long limit, CancellationToken token) { if (paths == null || !paths.Any()) return new PaginatedResults(0, (int)(limit > 0 ? limit : long.MaxValue), 0, 0, Enumerable.Empty()); @@ -655,35 +1182,56 @@ ORDER BY if (limit <= 0) limit = long.MaxValue; - using var cmd = m_connection.CreateCommand(); + await using var cmd = m_connection.CreateCommand(); var countWhere = "WHERE fl.\"Path\" IN (@Paths)"; if (filesetIds != null && filesetIds.Length > 0) countWhere += " AND fe.\"FilesetID\" IN (@FilesetIds)"; - cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC"); + cmd.SetCommandAndParameters(@" + SELECT ""ID"" + FROM ""Fileset"" + ORDER BY ""Timestamp"" DESC + "); - var versionMap = cmd.ExecuteReaderEnumerable() + var versionMap = await cmd + .ExecuteReaderEnumerableAsync(token) .Select((x, i) => (Version: i, FilesetId: x.ConvertValueToInt64(0))) - .ToDictionary(x => x.FilesetId, x => x.Version); + .ToDictionaryAsync(x => x.FilesetId, x => x.Version, cancellationToken: token) + .ConfigureAwait(false); // Then fetch the actual data - var fetchSql = @" - SELECT - fe.""FilesetID"", - f.""Timestamp"", - fl.""Path"", - COALESCE(b.""Length"", 0) AS ""Size"", - CASE WHEN fl.""BlocksetID"" = @FolderBlocksetId THEN 1 ELSE 0 END AS ""IsDirectory"", - CASE WHEN fl.""BlocksetID"" = @SymlinkBlocksetId THEN 1 ELSE 0 END AS ""IsSymlink"", - fe.""Lastmodified"" - FROM ""FilesetEntry"" fe - INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" - INNER JOIN ""Fileset"" f ON fe.""FilesetID"" = f.""ID"" - LEFT JOIN ""Blockset"" b ON fl.""BlocksetID"" = b.""ID"" - " + countWhere + @" - ORDER BY fl.""Path"" ASC, f.""Timestamp"" ASC - LIMIT @limit OFFSET @offset"; + var fetchSql = @$" + SELECT + ""fe"".""FilesetID"", + ""f"".""Timestamp"", + ""fl"".""Path"", + COALESCE(""b"".""Length"", 0) AS ""Size"", + CASE + WHEN ""fl"".""BlocksetID"" = @FolderBlocksetId + THEN 1 + ELSE 0 + END AS ""IsDirectory"", + CASE + WHEN ""fl"".""BlocksetID"" = @SymlinkBlocksetId + THEN 1 + ELSE 0 + END AS ""IsSymlink"", + ""fe"".""Lastmodified"" + FROM ""FilesetEntry"" ""fe"" + INNER JOIN ""FileLookup"" ""fl"" + ON ""fe"".""FileID"" = ""fl"".""ID"" + INNER JOIN ""Fileset"" ""f"" + ON ""fe"".""FilesetID"" = ""f"".""ID"" + LEFT JOIN ""Blockset"" ""b"" + ON ""fl"".""BlocksetID"" = ""b"".""ID"" + {countWhere} + ORDER BY + ""fl"".""Path"" ASC, + ""f"".""Timestamp"" ASC + LIMIT @limit + OFFSET @offset + "; cmd.SetCommandAndParameters(fetchSql) .ExpandInClauseParameter("@Paths", paths) @@ -693,10 +1241,10 @@ ORDER BY .SetParameterValue("@offset", offset); if (filesetIds != null && filesetIds.Length > 0) - cmd.ExpandInClauseParameter("@FilesetIds", filesetIds); + cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds); var results = new List(); - foreach (var rd in cmd.ExecuteReaderEnumerable()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) { var version = versionMap[rd.ConvertValueToInt64(0, -1)]; var time = new DateTime(rd.ConvertValueToInt64(1, 0), DateTimeKind.Utc); @@ -706,7 +1254,15 @@ ORDER BY var isSymlink = rd.GetInt32(5) != 0; var lastModified = new DateTime(rd.ConvertValueToInt64(6, 0), DateTimeKind.Utc); - results.Add(new ListFileVersion(version, time, path, size, isDirectory, isSymlink, lastModified)); + results.Add(new ListFileVersion( + version, + time, + path, + size, + isDirectory, + isSymlink, + lastModified + )); } long totalCount; @@ -725,17 +1281,19 @@ ORDER BY // Get total count var countSql = @" SELECT COUNT(*) - FROM ""FilesetEntry"" fe - INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" + FROM ""FilesetEntry"" ""fe"" + INNER JOIN ""FileLookup"" ""fl"" + ON ""fe"".""FileID"" = ""fl"".""ID"" "; cmd.SetCommandAndParameters(countSql + "\n" + countWhere) .ExpandInClauseParameter("@Paths", paths); if (filesetIds != null && filesetIds.Length > 0) - cmd.ExpandInClauseParameter("@FilesetIds", filesetIds); + cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds); - totalCount = cmd.ExecuteScalarInt64(0); + totalCount = await cmd.ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false); } var intLimit = limit > int.MaxValue ? int.MaxValue : (int)limit; @@ -758,15 +1316,16 @@ ORDER BY /// Optional fileset IDs to restrict the search to. /// The offset for pagination. /// The limit for pagination. - /// All matching file versions ordered by path and then version time. - public IPaginatedResults SearchEntries(IEnumerable? pathprefixes, IFilter filter, long[]? filesetIds, long offset, long limit) + /// A cancellation token to cancel the operation. + /// A task that when awaited returns all matching file versions ordered by path and then version time. + public async Task> SearchEntries(IEnumerable? pathprefixes, IFilter filter, long[]? filesetIds, long offset, long limit, CancellationToken token) { if (offset != 0 && limit <= 0) throw new ArgumentException("Cannot use offset without limit specified.", nameof(offset)); if (limit <= 0) limit = long.MaxValue; - using var cmd = m_connection.CreateCommand(); + await using var cmd = m_connection.CreateCommand(); // Analyze the filter to determine the default behavior FilterExpression.AnalyzeFilters(filter, out var includes, out var excludes); @@ -818,9 +1377,15 @@ ORDER BY var propName = $"@Part{filterProps.Count}"; filterProps[propName] = part; if (caseSensitive) - caseWhenParts.Add($@"WHEN instr(LOWER(pp.""Prefix"" || fl.""Path""), LOWER({propName})) > 0 THEN {(filterExpression.Result ? 0 : 1)}"); + caseWhenParts.Add($@" + WHEN instr(LOWER(pp.""Prefix"" || fl.""Path""), LOWER({propName})) > 0 + THEN {(filterExpression.Result ? 0 : 1)} + "); else - caseWhenParts.Add($@"WHEN instr(pp.""Prefix"" || fl.""Path"", {propName}) > 0 THEN {(filterExpression.Result ? 0 : 1)}"); + caseWhenParts.Add($@" + WHEN instr(pp.""Prefix"" || fl.""Path"", {propName}) > 0 + THEN {(filterExpression.Result ? 0 : 1)} + "); } } else if (filterExpression.Type == FilterType.Wildcard) @@ -831,9 +1396,15 @@ ORDER BY var propName = $"@Part{filterProps.Count}"; filterProps[propName] = part; if (caseSensitive) - caseWhenParts.Add($@"WHEN LOWER(pp.""Prefix"" || fl.""Path"") GLOB LOWER({propName}) THEN {(filterExpression.Result ? 0 : 1)}"); + caseWhenParts.Add($@" + WHEN LOWER(pp.""Prefix"" || fl.""Path"") GLOB LOWER({propName}) + THEN {(filterExpression.Result ? 0 : 1)} + "); else - caseWhenParts.Add($@"WHEN pp.""Prefix"" || fl.""Path"" GLOB {propName} THEN {(filterExpression.Result ? 0 : 1)}"); + caseWhenParts.Add($@" + WHEN pp.""Prefix"" || fl.""Path"" GLOB {propName} + THEN {(filterExpression.Result ? 0 : 1)} + "); } } else @@ -852,13 +1423,15 @@ ORDER BY if (caseWhenParts.Any()) { - whereClauses.Add($@"COALESCE( - CASE - {string.Join("\n ", caseWhenParts)} - ELSE NULL - END, - @DefaultBehavior - ) = 0"); + whereClauses.Add($@" + COALESCE( + CASE + {string.Join("\n ", caseWhenParts)} + ELSE NULL + END, + @DefaultBehavior + ) = 0 + "); } else { @@ -872,31 +1445,53 @@ ORDER BY var whereClause = string.Join("\n AND ", whereClauses); - cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC"); + cmd.SetCommandAndParameters(@" + SELECT ""ID"" + FROM ""Fileset"" + ORDER BY ""Timestamp"" DESC + "); - var versionMap = cmd.ExecuteReaderEnumerable() + var versionMap = await cmd + .ExecuteReaderEnumerableAsync(token) .Select((x, i) => (Version: i, FilesetId: x.ConvertValueToInt64(0))) - .ToDictionary(x => x.FilesetId, x => x.Version); + .ToDictionaryAsync(x => x.FilesetId, x => x.Version, cancellationToken: token) + .ConfigureAwait(false); // Fetch results var fetchSql = $@" - SELECT - fe.""FilesetID"", - f.""Timestamp"", - pp.""Prefix"" || fl.""Path"" AS ""FullPath"", - COALESCE(b.""Length"", 0) AS ""Size"", - CASE WHEN fl.""BlocksetID"" = @FolderBlocksetId THEN 1 ELSE 0 END AS ""IsDirectory"", - CASE WHEN fl.""BlocksetID"" = @SymlinkBlocksetId THEN 1 ELSE 0 END AS ""IsSymlink"", - fe.""Lastmodified"" - FROM ""FilesetEntry"" fe - INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" - INNER JOIN ""Fileset"" f ON fe.""FilesetID"" = f.""ID"" - LEFT JOIN ""Blockset"" b ON fl.""BlocksetID"" = b.""ID"" - INNER JOIN ""PathPrefix"" pp ON fl.""PrefixID"" = pp.""ID"" - WHERE {whereClause} - ORDER BY pp.""Prefix"" ASC, fl.""Path"" ASC, f.""Timestamp"" ASC - LIMIT @limit OFFSET @offset - "; + SELECT + ""fe"".""FilesetID"", + ""f"".""Timestamp"", + ""pp"".""Prefix"" || ""fl"".""Path"" AS ""FullPath"", + COALESCE(""b"".""Length"", 0) AS ""Size"", + CASE + WHEN ""fl"".""BlocksetID"" = @FolderBlocksetId + THEN 1 + ELSE 0 + END AS ""IsDirectory"", + CASE + WHEN ""fl"".""BlocksetID"" = @SymlinkBlocksetId + THEN 1 + ELSE 0 + END AS ""IsSymlink"", + ""fe"".""Lastmodified"" + FROM ""FilesetEntry"" ""fe"" + INNER JOIN ""FileLookup"" ""fl"" + ON ""fe"".""FileID"" = ""fl"".""ID"" + INNER JOIN ""Fileset"" ""f"" + ON ""fe"".""FilesetID"" = ""f"".""ID"" + LEFT JOIN ""Blockset"" ""b"" + ON ""fl"".""BlocksetID"" = ""b"".""ID"" + INNER JOIN ""PathPrefix"" ""pp"" + ON ""fl"".""PrefixID"" = ""pp"".""ID"" + WHERE {whereClause} + ORDER BY + ""pp"".""Prefix"" ASC, + ""fl"".""Path"" ASC, + ""f"".""Timestamp"" ASC + LIMIT @limit + OFFSET @offset + "; cmd.SetCommandAndParameters(fetchSql) .SetParameterValues(filterProps) @@ -910,10 +1505,10 @@ ORDER BY cmd.ExpandInClauseParameter("@PathPrefixes", pathprefixes); if (filesetIds != null && filesetIds.Length > 0) - cmd.ExpandInClauseParameter("@FilesetIds", filesetIds); + cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds); var results = new List(); - foreach (var rd in cmd.ExecuteReaderEnumerable()) + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) { var version = versionMap[rd.ConvertValueToInt64(0, -1)]; var time = new DateTime(rd.ConvertValueToInt64(1, 0), DateTimeKind.Utc); @@ -944,9 +1539,11 @@ ORDER BY // Calculate the total var countSql = $@" SELECT COUNT(*) - FROM ""FilesetEntry"" fe - INNER JOIN ""FileLookup"" fl ON fe.""FileID"" = fl.""ID"" - INNER JOIN ""PathPrefix"" pp ON fl.""PrefixID"" = pp.""ID"" + FROM ""FilesetEntry"" ""fe"" + INNER JOIN ""FileLookup"" ""fl"" + ON ""fe"".""FileID"" = ""fl"".""ID"" + INNER JOIN ""PathPrefix"" ""pp"" + ON ""fl"".""PrefixID"" = ""pp"".""ID"" WHERE {whereClause} "; @@ -958,9 +1555,10 @@ ORDER BY cmd.ExpandInClauseParameter("@PathPrefixes", pathprefixes); if (filesetIds != null && filesetIds.Length > 0) - cmd.ExpandInClauseParameter("@FilesetIds", filesetIds); + cmd.ExpandInClauseParameterMssqlite("@FilesetIds", filesetIds); - totalCount = cmd.ExecuteScalarInt64(0); + totalCount = await cmd.ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false); } var intLimit = limit > int.MaxValue ? int.MaxValue : (int)limit; @@ -976,4 +1574,3 @@ ORDER BY } } } - diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index 5e8cfe288..a5ed28302 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -1,119 +1,277 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable using System; -using System.Data; using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; using Duplicati.Library.Utility; +using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database { + /// + /// A local database for purging files from a fileset. + /// internal class LocalPurgeDatabase : LocalDeleteDatabase { - public LocalPurgeDatabase(string path, long pagecachesize) - : base(path, "Purge", pagecachesize) + /// + /// Creates a new instance of the class. + /// + /// The path to the database file. + /// The optional existing database instance to use. Used to mimic constructor chaining. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains a new instance of . + public static async Task CreateAsync(string path, LocalPurgeDatabase? dbnew, CancellationToken token) { - ShouldCloseConnection = true; + dbnew ??= new LocalPurgeDatabase(); + + dbnew = (LocalPurgeDatabase) + await LocalDeleteDatabase.CreateAsync(path, "Purge", dbnew, token) + .ConfigureAwait(false); + + dbnew.ShouldCloseConnection = true; + + return dbnew; } - public LocalPurgeDatabase(LocalDatabase db) - : base(db) + /// + /// Creates a new instance of the class. + /// + /// The parent database to use. + /// An optional existing database instance to use. Used to mimic constructor chaining. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains a new instance of . + public static async Task CreateAsync(LocalDatabase dbparent, LocalPurgeDatabase? dbnew, CancellationToken token) { + dbnew ??= new LocalPurgeDatabase(); + + dbnew = (LocalPurgeDatabase) + await LocalDeleteDatabase.CreateAsync(dbparent, dbnew, token) + .ConfigureAwait(false); + + return dbnew; } - public ITemporaryFileset CreateTemporaryFileset(long parentid, IDbTransaction transaction) + /// + /// Creates a new instance of the class. + /// + /// The ID of the parent fileset. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains a new instance of . + public async Task CreateTemporaryFileset(long parentid, CancellationToken token) { - return new TemporaryFileset(parentid, this, m_connection, transaction); + return await TemporaryFileset.CreateAsync(parentid, this, token) + .ConfigureAwait(false); } - public string GetRemoteVolumeNameForFileset(long id, IDbTransaction transaction) + /// + /// Gets the remote volume name for a fileset with the specified ID. + /// + /// The ID of the fileset. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains the name of the remote volume. + public async Task GetRemoteVolumeNameForFileset(long id, CancellationToken token) { - using var cmd = m_connection.CreateCommand(transaction, @"SELECT ""B"".""Name"" FROM ""Fileset"" A, ""RemoteVolume"" B WHERE ""A"".""VolumeID"" = ""B"".""ID"" AND ""A"".""ID"" = @FilesetId ") - .SetParameterValue("@FilesetId", id); - using (var rd = cmd.ExecuteReader()) - if (!rd.Read()) - throw new Exception($"No remote volume found for fileset with id {id}"); - else - return rd.ConvertValueToString(0) ?? throw new Exception($"Remote volume name for fileset with id {id} is null"); + await using var cmd = m_connection.CreateCommand(@" + SELECT + ""B"".""Name"" + FROM + ""Fileset"" ""A"", + ""RemoteVolume"" ""B"" + WHERE + ""A"".""VolumeID"" = ""B"".""ID"" + AND ""A"".""ID"" = @FilesetId + ") + .SetParameterValue("@FilesetId", id); + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + if (!await rd.ReadAsync(token).ConfigureAwait(false)) + throw new Exception($"No remote volume found for fileset with id {id}"); + else + return rd.ConvertValueToString(0) ?? throw new Exception($"Remote volume name for fileset with id {id} is null"); } - internal long CountOrphanFiles(IDbTransaction transaction) + /// + /// Counts the number of orphan files in the database. + /// + /// A cancellation token to cancel the operation. + /// A task that when awaited contains the count of orphan files. + internal async Task CountOrphanFiles(CancellationToken token) { - using (var cmd = m_connection.CreateCommand(transaction)) - using (var rd = cmd.ExecuteReader(@"SELECT COUNT(*) FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FileID"" FROM ""FilesetEntry"")")) - if (rd.Read()) - return rd.ConvertValueToInt64(0, 0); - else - return 0; + await using var cmd = m_connection.CreateCommand(m_rtr); + cmd.SetCommandAndParameters(@" + SELECT COUNT(*) + FROM ""FileLookup"" + WHERE ""ID"" NOT IN ( + SELECT DISTINCT ""FileID"" + FROM ""FilesetEntry"" + ) + "); + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + if (await rd.ReadAsync(token).ConfigureAwait(false)) + return rd.ConvertValueToInt64(0, 0); + else + return 0; } - public interface ITemporaryFileset : IDisposable + /// + /// Interface for a temporary fileset used for purging files from a fileset. + /// + public interface ITemporaryFileset : IDisposable, IAsyncDisposable { + /// + /// Gets the ID of the parent fileset. + /// long ParentID { get; } + /// + /// Gets the count of removed files. + /// long RemovedFileCount { get; } + /// + /// Gets the total size of removed files. + /// long RemovedFileSize { get; } + /// + /// Gets the count of updated files. + /// long UpdatedFileCount { get; } - void ApplyFilter(Library.Utility.IFilter filter); - void ApplyFilter(Func filtercommand); - Tuple ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup); - IEnumerable> ListAllDeletedFiles(); + /// + /// Applies a filter to the temporary fileset. + /// + /// The filter to apply. + /// A cancellation token to cancel the operation. + /// A task that completes when the filter has been applied. + Task ApplyFilter(Library.Utility.IFilter filter, CancellationToken token); + /// + /// Applies a filter using a custom command. + /// + /// The command to execute for filtering. + /// A cancellation token to cancel the operation. + /// A task that completes when the filter has been applied. + Task ApplyFilter(Func> filtercommand, CancellationToken token); + /// + /// Converts the temporary fileset to a permanent fileset. + /// + /// The name of the new fileset. + /// The timestamp for the new fileset. + /// Indicates if this is a full backup. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains a tuple with the remote volume ID and the new fileset ID. + Task> ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup, CancellationToken token); + /// + /// Lists all deleted files in the temporary fileset. + /// + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of key-value pairs where the key is the file path and the value is the file size. + IAsyncEnumerable> ListAllDeletedFiles(CancellationToken token); } + /// + /// A temporary fileset implementation for purging files from a fileset. + /// private class TemporaryFileset : ITemporaryFileset { - private readonly IDbConnection m_connection; - private readonly IDbTransaction m_transaction; - private readonly string m_tablename; - private readonly LocalPurgeDatabase m_parentdb; + /// + /// The name of the temporary table used for storing deleted files. + /// + private string m_tablename = null!; + /// + /// The database instance used for queries. + /// + private LocalDatabase m_db = null!; public long ParentID { get; private set; } public long RemovedFileCount { get; private set; } public long RemovedFileSize { get; private set; } public long UpdatedFileCount { get; private set; } - public TemporaryFileset(long parentid, LocalPurgeDatabase parentdb, IDbConnection connection, IDbTransaction transaction) + /// + /// Calling this constructor will throw an exception. Use CreateAsync instead. + /// + [Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")] + public TemporaryFileset(long parentid, LocalPurgeDatabase parentdb, SqliteConnection connection, SqliteTransaction transaction) { - ParentID = parentid; - m_parentdb = parentdb; - m_connection = connection; - m_transaction = transaction; - m_tablename = "TempDeletedFilesTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - - using (var cmd = m_connection.CreateCommand(m_transaction)) - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tablename}"" (""FileID"" INTEGER PRIMARY KEY) ")); + throw new NotImplementedException("Use CreateAsync instead"); } - public void ApplyFilter(Func filtercommand) + /// + /// Private constructor to prevent direct instantiation. + /// + private TemporaryFileset() { } + + /// + /// Creates a new instance of the class asynchronously. + /// + /// The ID of the parent fileset. + /// The database instance to use. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains a new instance of . + public static async Task CreateAsync(long parentid, LocalDatabase db, CancellationToken token) + { + var tempf = new TemporaryFileset() + { + ParentID = parentid, + m_db = db, + m_tablename = "TempDeletedFilesTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()) + }; + + await using (var cmd = db.Connection.CreateCommand()) + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{tempf.m_tablename}"" ( + ""FileID"" INTEGER PRIMARY KEY + ) + ", token) + .ConfigureAwait(false); + + return tempf; + } + + /// + /// Applies a filter to the temporary fileset using a custom command. + /// + /// The command to execute for filtering. + /// A cancellation token to cancel the operation. + /// A task that completes when the filter has been applied. + public async Task ApplyFilter(Func> filtercommand, CancellationToken token) { int updated; - using (var cmd = m_connection.CreateCommand(m_transaction)) - updated = filtercommand(cmd, ParentID, m_tablename); + await using (var cmd = m_db.Connection.CreateCommand()) + updated = await filtercommand(cmd, ParentID, m_tablename) + .ConfigureAwait(false); - PostFilterChecks(updated); + await PostFilterChecks(updated, token).ConfigureAwait(false); } - public void ApplyFilter(Library.Utility.IFilter filter) + /// + /// Applies a filter to the temporary fileset. + /// + /// The filter to apply. + /// A cancellation token to cancel the operation. + /// A task that completes when the filter has been applied. + public async Task ApplyFilter(Library.Utility.IFilter filter, CancellationToken token) { if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == Duplicati.Library.Utility.FilterType.Simple) { @@ -122,103 +280,219 @@ namespace Duplicati.Library.Main.Database // SQLite only supports ASCII compares var p = expression.GetSimpleList(); var filenamestable = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using (var cmd = m_connection.CreateCommand(m_transaction)) - { - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{filenamestable}"" (""Path"" TEXT NOT NULL) ")); - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{filenamestable}"" (""Path"") VALUES (@Path)")); - foreach (var s in p) - cmd.SetParameterValue("@Path", s) - .ExecuteNonQuery(); + await using var cmd = m_db.Connection.CreateCommand(); + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{filenamestable}"" ( + ""Path"" TEXT NOT NULL + ) + ", token) + .ConfigureAwait(false); - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_tablename}"" (""FileID"") SELECT DISTINCT ""A"".""FileID"" FROM ""FilesetEntry"" A, ""File"" B WHERE ""A"".""FilesetID"" = @FilesetId AND ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""Path"" IN ""{filenamestable}""")) - .SetParameterValue("@FilesetId", ParentID) - .ExecuteNonQuery(); - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{filenamestable}"" ")); - } + await cmd.SetCommandAndParameters($@" + INSERT INTO ""{filenamestable}"" (""Path"") + VALUES (@Path) + ") + .PrepareAsync(token) + .ConfigureAwait(false); + + foreach (var s in p) + await cmd.SetParameterValue("@Path", s) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + await cmd.SetCommandAndParameters($@" + INSERT INTO ""{m_tablename}"" (""FileID"") + SELECT DISTINCT ""A"".""FileID"" + FROM + ""FilesetEntry"" ""A"", + ""File"" ""B"" + WHERE + ""A"".""FilesetID"" = @FilesetId + AND ""A"".""FileID"" = ""B"".""ID"" + AND ""B"".""Path"" IN ""{filenamestable}"" + ") + .SetParameterValue("@FilesetId", ParentID) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + await cmd + .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filenamestable}"" ", token) + .ConfigureAwait(false); } else { // Do row-wise iteration var values = new object[2]; - using (var cmd = m_connection.CreateCommand(m_transaction)) - using (var cmd2 = m_connection.CreateCommand(m_transaction)) + await using var cmd = m_db.Connection.CreateCommand(); + await using var cmd2 = m_db.Connection.CreateCommand(); + await cmd2.SetCommandAndParameters($@" + INSERT INTO ""{m_tablename}"" (""FileID"") + VALUES (@FileId) + ") + .PrepareAsync(token) + .ConfigureAwait(false); + + cmd.SetCommandAndParameters(@" + SELECT + ""B"".""Path"", + ""A"".""FileID"" + FROM + ""FilesetEntry"" ""A"", + ""File"" ""B"" + WHERE + ""A"".""FilesetID"" = @FilesetId + AND ""A"".""FileID"" = ""B"".""ID"" + ") + .SetParameterValue("@FilesetId", ParentID); + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) { - cmd2.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_tablename}"" (""FileID"") VALUES (@FileId)")); - cmd.SetCommandAndParameters(@"SELECT ""B"".""Path"", ""A"".""FileID"" FROM ""FilesetEntry"" A, ""File"" B WHERE ""A"".""FilesetID"" = @FilesetId AND ""A"".""FileID"" = ""B"".""ID"" ") - .SetParameterValue("@FilesetId", ParentID); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - { - rd.GetValues(values); - var path = values[0] as string; - if (path != null && Library.Utility.FilterExpression.Matches(filter, path.ToString())) - { - cmd2.SetParameterValue("@FileId", values[1]) - .ExecuteNonQuery(); - } - } + rd.GetValues(values); + var path = values[0] as string; + if (path != null && Library.Utility.FilterExpression.Matches(filter, path.ToString())) + { + await cmd2 + .SetParameterValue("@FileId", values[1]) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } } } - PostFilterChecks(0); + await PostFilterChecks(0, token).ConfigureAwait(false); } - private void PostFilterChecks(int updated) + /// + /// Applies the filter checks after filtering. + /// + /// The number of files updated by the filter. + /// A cancellation token to cancel the operation. + /// A task that completes when the checks have been applied. + private async Task PostFilterChecks(int updated, CancellationToken token) { - using (var cmd = m_connection.CreateCommand(m_transaction)) - { - UpdatedFileCount = updated; - RemovedFileCount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""{m_tablename}"""), 0); - RemovedFileSize = cmd.ExecuteScalarInt64(FormatInvariant($@" - SELECT - SUM(""C"".""Length"") + await using var cmd = m_db.Connection.CreateCommand(); + UpdatedFileCount = updated; + + RemovedFileCount = await cmd.ExecuteScalarInt64Async($@" + SELECT COUNT(*) + FROM ""{m_tablename}"" + ", 0, token) + .ConfigureAwait(false); + + RemovedFileSize = await cmd.ExecuteScalarInt64Async($@" + SELECT SUM(""C"".""Length"") FROM - ""{m_tablename}"" A, ""FileLookup"" B, ""Blockset"" C, ""Metadataset"" D + ""{m_tablename}"" ""A"", + ""FileLookup"" ""B"", + ""Blockset"" ""C"", + ""Metadataset"" ""D"" WHERE ""A"".""FileID"" = ""B"".""ID"" - AND(""B"".""BlocksetID"" = ""C"".""ID"" OR(""B"".""MetadataID"" = ""D"".""ID"" AND ""D"".""BlocksetID"" = ""C"".""ID"")) - "), 0); - var filesetcount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = {ParentID}"), 0); - if (filesetcount == RemovedFileCount) - throw new Interface.UserInformationException($"Refusing to purge {RemovedFileCount} files from fileset with ID {ParentID}, as that would remove the entire fileset.\nTo delete a fileset, use the \"delete\" command.", "PurgeWouldRemoveEntireFileset"); - } + AND ( + ""B"".""BlocksetID"" = ""C"".""ID"" + OR ( + ""B"".""MetadataID"" = ""D"".""ID"" + AND ""D"".""BlocksetID"" = ""C"".""ID"" + ) + ) + ", 0, token) + .ConfigureAwait(false); + + var filesetcount = await cmd.SetCommandAndParameters($@" + SELECT COUNT(*) + FROM ""FilesetEntry"" + WHERE ""FilesetID"" = @ParentID + ") + .SetParameterValue("@ParentID", ParentID) + .ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false); + + if (filesetcount == RemovedFileCount) + throw new Interface.UserInformationException($"Refusing to purge {RemovedFileCount} files from fileset with ID {ParentID}, as that would remove the entire fileset.\nTo delete a fileset, use the \"delete\" command.", "PurgeWouldRemoveEntireFileset"); } - public Tuple ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup) + /// + public async Task> ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup, CancellationToken token) { - var remotevolid = m_parentdb.RegisterRemoteVolume(name, RemoteVolumeType.Files, RemoteVolumeState.Temporary, m_transaction); - var filesetid = m_parentdb.CreateFileset(remotevolid, timestamp, m_transaction); - m_parentdb.UpdateFullBackupStateInFileset(filesetid, isFullBackup); + var remotevolid = + await m_db + .RegisterRemoteVolume(name, RemoteVolumeType.Files, RemoteVolumeState.Temporary, token) + .ConfigureAwait(false); - using (var cmd = m_connection.CreateCommand(m_transaction)) - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") SELECT @TargetFilesetId, ""FileID"", ""LastModified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @SourceFilesetId AND ""FileID"" NOT IN ""{m_tablename}"" ")) - .SetParameterValue("@TargetFilesetId", filesetid) - .SetParameterValue("@SourceFilesetId", ParentID) - .ExecuteNonQuery(); + var filesetid = await m_db + .CreateFileset(remotevolid, timestamp, token) + .ConfigureAwait(false); + + await m_db + .UpdateFullBackupStateInFileset(filesetid, isFullBackup, token) + .ConfigureAwait(false); + + await using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction)) + await cmd.SetCommandAndParameters($@" + INSERT INTO ""FilesetEntry"" ( + ""FilesetID"", + ""FileID"", + ""Lastmodified"" + ) + SELECT + @TargetFilesetId, + ""FileID"", + ""LastModified"" + FROM ""FilesetEntry"" + WHERE + ""FilesetID"" = @SourceFilesetId + AND ""FileID"" NOT IN ""{m_tablename}"" + ") + .SetParameterValue("@TargetFilesetId", filesetid) + .SetParameterValue("@SourceFilesetId", ParentID) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); return new Tuple(remotevolid, filesetid); } - public IEnumerable> ListAllDeletedFiles() + /// + public async IAsyncEnumerable> ListAllDeletedFiles([EnumeratorCancellation] CancellationToken token) { - using (var cmd = m_connection.CreateCommand(m_transaction)) - using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT ""B"".""Path"", ""C"".""Length"" FROM ""{m_tablename}"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" "))) - while (rd.Read()) - yield return new KeyValuePair(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); + await using var cmd = m_db.Connection.CreateCommand(); + await using var rd = await cmd.ExecuteReaderAsync($@" + SELECT + ""B"".""Path"", + ""C"".""Length"" + FROM + ""{m_tablename}"" ""A"", + ""File"" ""B"", + ""Blockset"" ""C"" + WHERE + ""A"".""FileID"" = ""B"".""ID"" + AND ""B"".""BlocksetID"" = ""C"".""ID"" + ", token) + .ConfigureAwait(false); + + while (await rd.ReadAsync(token).ConfigureAwait(false)) + yield return new KeyValuePair( + rd.ConvertValueToString(0) ?? "", + rd.ConvertValueToInt64(1) + ); } public void Dispose() + { + DisposeAsync().AsTask().Await(); + } + + public async ValueTask DisposeAsync() { try { - using (var cmd = m_connection.CreateCommand(m_transaction)) - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tablename}""")); - } - catch - { + await using var cmd = m_db.Connection.CreateCommand(); + await cmd + .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tablename}""", default) + .ConfigureAwait(false); } + catch { } } } - } } diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index 16900bd2e..807ff4e4e 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -1,44 +1,58 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable using System; -using System.Data; using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.Utility; +using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Database { + /// + /// This class is used to recreate a local database from a backup. + /// internal class LocalRecreateDatabase : LocalRestoreDatabase { /// - /// The tag used for logging + /// The tag used for logging. /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalRecreateDatabase)); + /// + /// Keeps track of the path entries in a sorted manner. + /// This is used to ensure that the entries are unique and sorted by their keys. + /// private class PathEntryKeeper { + /// + /// Comparer for KeyValuePair to sort by key and value. + /// private struct KeyValueComparer : IComparer> { - public int Compare(KeyValuePair x, KeyValuePair y) + public readonly int Compare(KeyValuePair x, KeyValuePair y) { return x.Key == y.Key ? (x.Value == y.Value ? @@ -49,172 +63,562 @@ namespace Duplicati.Library.Main.Database } } - private readonly IDbCommand m_insertFileCommand; - private readonly IDbCommand m_insertFilesetEntryCommand; - private readonly IDbCommand m_insertMetadatasetCommand; - private readonly IDbCommand m_insertBlocksetCommand; - private readonly IDbCommand m_insertBlocklistHashCommand; - private readonly IDbCommand m_updateBlockVolumeCommand; - private readonly IDbCommand m_insertTempBlockListHash; - private readonly IDbCommand m_insertSmallBlockset; - private readonly IDbCommand m_findBlocksetCommand; - private readonly IDbCommand m_findMetadatasetCommand; - private readonly IDbCommand m_findFilesetCommand; - private readonly IDbCommand m_findTempBlockListHashCommand; - private readonly IDbCommand m_findHashBlockCommand; - private readonly IDbCommand m_insertBlockCommand; - private readonly IDbCommand m_insertDuplicateBlockCommand; - - private string m_tempblocklist; - private string m_tempsmalllist; + /// + /// Inserts a new file entry into the FileLookup table and returns the new row ID. + /// + private SqliteCommand m_insertFileCommand = null!; + /// + /// Inserts a new entry into the FilesetEntry table linking a fileset to a file with a last-modified timestamp. + /// + private SqliteCommand m_insertFilesetEntryCommand = null!; + /// + /// Inserts a new row into the Metadataset table for a given blockset and returns the new row ID. + /// + private SqliteCommand m_insertMetadatasetCommand = null!; + /// + /// Inserts a new blockset with its length and full hash into the Blockset table and returns the new row ID. + /// + private SqliteCommand m_insertBlocksetCommand = null!; + /// + /// Inserts a new blocklist hash entry into the BlocklistHash table for a given blockset and index. + /// + private SqliteCommand m_insertBlocklistHashCommand = null!; + /// + /// Updates the VolumeID of a block in the Block table matching a specific hash and size. + /// + private SqliteCommand m_updateBlockVolumeCommand = null!; + /// + /// Inserts a blocklist hash, block hash, and index into the temporary blocklist table. + /// + private SqliteCommand m_insertTempBlockListHash = null!; + /// + /// Inserts or ignores a file hash, block hash, and block size into the temporary small blockset table. + /// + private SqliteCommand m_insertSmallBlockset = null!; + /// + /// Selects the ID of a blockset from the Blockset table matching a given length and full hash. + /// + private SqliteCommand m_findBlocksetCommand = null!; + /// + /// Selects the ID of a Metadataset whose blockset matches a given full hash and length. + /// + private SqliteCommand m_findMetadatasetCommand = null!; + /// + /// Selects the ID of a file from the FileLookup table matching a given prefix, path, blockset, and metadata. + /// + private SqliteCommand m_findFilesetCommand = null!; + /// + /// Selects distinct blocklist hashes from the temporary blocklist table matching a given hash. + /// + private SqliteCommand m_findTempBlockListHashCommand = null!; + /// + /// Selects the VolumeID of a block in the Block table matching a specific hash and size. + /// + private SqliteCommand m_findHashBlockCommand = null!; + /// + /// Inserts a new block with hash, size, and volume ID into the Block table. + /// + private SqliteCommand m_insertBlockCommand = null!; + /// + /// Inserts or ignores a duplicate block entry linking a block and a volume into the DuplicateBlock table. + /// + private SqliteCommand m_insertDuplicateBlockCommand = null!; /// - /// A lookup table that prevents multiple downloads of the same volume + /// Temporary table name for blocklist hashes and their indices. + /// + private string m_tempblocklist = null!; + /// + /// Temporary table name for small blocksets, containing file hashes, block hashes, and block sizes. + /// + private string m_tempsmalllist = null!; + + /// + /// A lookup table that prevents multiple downloads of the same volume. /// private readonly Dictionary m_proccessedVolumes = new Dictionary(); - // SQL that finds index and block size for all blocklist hashes, based on the temporary hash list - // with vars Used: - // {0} --> Blocksize - // {1} --> BlockHash-Size - // {2} --> Temp-Table - // {3} --> FullBlocklist-BlockCount [equals ({0} / {1}), if SQLite pays respect to ints] - private static string SELECT_BLOCKLIST_ENTRIES(long blocksize, long blockhashsize, string temptable, long fullBlockListBlockCount) => - FormatInvariant($@" - SELECT - ""E"".""BlocksetID"", - ""F"".""Index"" + (""E"".""BlocklistIndex"" * {fullBlockListBlockCount}) AS ""FullIndex"", - ""F"".""BlockHash"", - MIN({blocksize}, ""E"".""Length"" - ((""F"".""Index"" + (""E"".""BlocklistIndex"" * {fullBlockListBlockCount})) * {blocksize})) AS ""BlockSize"", - ""E"".""Hash"", - ""E"".""BlocklistSize"", - ""E"".""BlocklistHash"" - FROM - ( - SELECT * FROM - ( - SELECT - ""A"".""BlocksetID"", - ""A"".""Index"" AS ""BlocklistIndex"", - MIN({fullBlockListBlockCount} * {blockhashsize}, (((""B"".""Length"" + {blocksize} - 1) / {blocksize}) - (""A"".""Index"" * ({fullBlockListBlockCount}))) * {blockhashsize}) AS ""BlocklistSize"", - ""A"".""Hash"" AS ""BlocklistHash"", - ""B"".""Length"" - FROM - ""BlocklistHash"" A, - ""Blockset"" B - WHERE - ""B"".""ID"" = ""A"".""BlocksetID"" - ) C, - ""Block"" D - WHERE - ""C"".""BlocklistHash"" = ""D"".""Hash"" - AND - ""C"".""BlocklistSize"" = ""D"".""Size"" - ) E, - ""{temptable}"" F - WHERE - ""F"".""BlocklistHash"" = ""E"".""Hash"" - ORDER BY - ""E"".""BlocksetID"", - ""FullIndex"" -"); + /// + /// SQL that finds index and block size for all blocklist hashes, based on the temporary hash list. + /// + /// The size of the blocks in the blocklist. + /// The size of the block hashes. + /// The name of the temporary table containing blocklist hashes and indices. + /// The number of blocks in the full block list, used to calculate the full index. + /// A SQL query string that selects blocklist entries with their full index and block size. + private static string SELECT_BLOCKLIST_ENTRIES(long blocksize, long blockhashsize, string temptable, long fullBlockListBlockCount) => $@" + SELECT + ""E"".""BlocksetID"", + ""F"".""Index"" + (""E"".""BlocklistIndex"" * {fullBlockListBlockCount}) AS ""FullIndex"", + ""F"".""BlockHash"", + MIN({blocksize}, ""E"".""Length"" - ((""F"".""Index"" + (""E"".""BlocklistIndex"" * {fullBlockListBlockCount})) * {blocksize})) AS ""BlockSize"", + ""E"".""Hash"", + ""E"".""BlocklistSize"", + ""E"".""BlocklistHash"" + FROM + ( + SELECT * + FROM + ( + SELECT + ""A"".""BlocksetID"", + ""A"".""Index"" AS ""BlocklistIndex"", + MIN({fullBlockListBlockCount} * {blockhashsize}, (((""B"".""Length"" + {blocksize} - 1) / {blocksize}) - (""A"".""Index"" * ({fullBlockListBlockCount}))) * {blockhashsize}) AS ""BlocklistSize"", + ""A"".""Hash"" AS ""BlocklistHash"", + ""B"".""Length"" + FROM + ""BlocklistHash"" ""A"", + ""Blockset"" ""B"" + WHERE + ""B"".""ID"" = ""A"".""BlocksetID"" + ) ""C"", + ""Block"" ""D"" + WHERE + ""C"".""BlocklistHash"" = ""D"".""Hash"" + AND ""C"".""BlocklistSize"" = ""D"".""Size"" + ) ""E"", + ""{temptable}"" ""F"" + WHERE + ""F"".""BlocklistHash"" = ""E"".""Hash"" + ORDER BY + ""E"".""BlocksetID"", + ""FullIndex"" + "; - public LocalRecreateDatabase(LocalDatabase parentdb, Options options) - : base(parentdb) + /// + /// Asynchronously creates a new instance of the class. + /// + /// The parent database from which to restore. + /// The options to use for the database creation. + /// An optional existing instance of to use; if null, a new instance will be created. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that represents the asynchronous operation, containing the newly created instance. + public static async Task CreateAsync(LocalDatabase parentdb, Options options, LocalRecreateDatabase? dbnew, CancellationToken token) { - m_tempblocklist = "TempBlocklist_" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - m_tempsmalllist = "TempSmalllist_" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + dbnew ??= new LocalRecreateDatabase(); - using (var cmd = m_connection.CreateCommand()) - { - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tempblocklist}"" (""BlockListHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""Index_{m_tempblocklist}"" ON ""{m_tempblocklist}"" (""BlockListHash"");")); + dbnew = (LocalRecreateDatabase) + await LocalRestoreDatabase.CreateAsync(parentdb, dbnew, token) + .ConfigureAwait(false); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tempsmalllist}"" (""FileHash"" TEXT NOT NULL, ""BlockHash"" TEXT NOT NULL, ""BlockSize"" INTEGER NOT NULL)")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE UNIQUE INDEX ""Index_File_{m_tempsmalllist}"" ON ""{m_tempsmalllist}"" (""FileHash"", ""BlockSize"");")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE UNIQUE INDEX ""Index_Block_{m_tempsmalllist}"" ON ""{m_tempsmalllist}"" (""BlockHash"", ""BlockSize"");")); - } + dbnew.m_tempblocklist = "TempBlocklist_" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + dbnew.m_tempsmalllist = "TempSmalllist_" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - m_insertFileCommand = m_connection.CreateCommand(@"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") VALUES (@PrefixId,@Path,@BlocksetId,@MetadataId); SELECT last_insert_rowid();"); - m_insertFilesetEntryCommand = m_connection.CreateCommand(@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (@FilesetId,@FileId,@LastModified)"); - m_insertMetadatasetCommand = m_connection.CreateCommand(@"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (@BlocksetId); SELECT last_insert_rowid();"); - m_insertBlocksetCommand = m_connection.CreateCommand(@"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (@Length,@FullHash); SELECT last_insert_rowid();"); - m_insertBlocklistHashCommand = m_connection.CreateCommand(@"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (@BlocksetId,@Index,@Hash)"); - m_updateBlockVolumeCommand = m_connection.CreateCommand(@"UPDATE ""Block"" SET ""VolumeID"" = @VolumeId WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); - m_insertTempBlockListHash = m_connection.CreateCommand(FormatInvariant($@"INSERT INTO ""{m_tempblocklist}"" (""BlocklistHash"", ""BlockHash"", ""Index"") VALUES (@BlocklistHash,@BlockHash,@Index) ")); - m_insertSmallBlockset = m_connection.CreateCommand(FormatInvariant($@"INSERT OR IGNORE INTO ""{m_tempsmalllist}"" (""FileHash"", ""BlockHash"", ""BlockSize"") VALUES (@FileHash,@BlockHash,@BlockSize) ")); - m_findBlocksetCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" = @Length AND ""FullHash"" = @FullHash "); - m_findMetadatasetCommand = m_connection.CreateCommand(@"SELECT ""Metadataset"".""ID"" FROM ""Metadataset"",""Blockset"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Blockset"".""FullHash"" = @FullHash AND ""Blockset"".""Length"" = @Length "); - m_findFilesetCommand = m_connection.CreateCommand(@"SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId "); - m_findTempBlockListHashCommand = m_connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""BlockListHash"" FROM ""{m_tempblocklist}"" WHERE ""BlockListHash"" = @BlocklistHash ")); - m_findHashBlockCommand = m_connection.CreateCommand(@"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size "); - m_insertBlockCommand = m_connection.CreateCommand(@"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") VALUES (@Hash,@Size,@VolumeId)"); - m_insertDuplicateBlockCommand = m_connection.CreateCommand(@"INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") VALUES ((SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size), @VolumeId)"); + if (dbnew.m_connection == null) + throw new Exception("Connection is null"); + await using var cmd = dbnew.m_connection.CreateCommand(); + + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{dbnew.m_tempblocklist}"" ( + ""BlockListHash"" TEXT NOT NULL, + ""BlockHash"" TEXT NOT NULL, + ""Index"" INTEGER NOT NULL + ) + ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""Index_{dbnew.m_tempblocklist}"" + ON ""{dbnew.m_tempblocklist}"" (""BlockListHash""); + ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{dbnew.m_tempsmalllist}"" ( + ""FileHash"" TEXT NOT NULL, + ""BlockHash"" TEXT NOT NULL, + ""BlockSize"" INTEGER NOT NULL + ) + ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@" + CREATE UNIQUE INDEX ""Index_File_{dbnew.m_tempsmalllist}"" + ON ""{dbnew.m_tempsmalllist}"" ( + ""FileHash"", + ""BlockSize"" + ); + ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@" + CREATE UNIQUE INDEX ""Index_Block_{dbnew.m_tempsmalllist}"" + ON ""{dbnew.m_tempsmalllist}"" ( + ""BlockHash"", + ""BlockSize"" + ); + ", token) + .ConfigureAwait(false); + + dbnew.m_insertFileCommand = await dbnew.m_connection.CreateCommandAsync(@" + INSERT INTO ""FileLookup"" ( + ""PrefixID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"" + ) + VALUES ( + @PrefixId, + @Path, + @BlocksetId, + @MetadataId + ); + SELECT last_insert_rowid(); + ", token) + .ConfigureAwait(false); + + dbnew.m_insertFilesetEntryCommand = await dbnew.m_connection.CreateCommandAsync(@" + INSERT INTO ""FilesetEntry"" ( + ""FilesetID"", + ""FileID"", + ""Lastmodified"" + ) + VALUES ( + @FilesetId, + @FileId, + @LastModified + ) + ", token) + .ConfigureAwait(false); + + dbnew.m_insertMetadatasetCommand = await dbnew.m_connection.CreateCommandAsync(@" + INSERT INTO ""Metadataset"" (""BlocksetID"") + VALUES (@BlocksetId); + SELECT last_insert_rowid(); + ", token) + .ConfigureAwait(false); + + dbnew.m_insertBlocksetCommand = await dbnew.m_connection.CreateCommandAsync(@" + INSERT INTO ""Blockset"" ( + ""Length"", + ""FullHash"" + ) + VALUES ( + @Length, + @FullHash + ); + SELECT last_insert_rowid(); + ", token) + .ConfigureAwait(false); + + dbnew.m_insertBlocklistHashCommand = await dbnew.m_connection.CreateCommandAsync(@" + INSERT INTO ""BlocklistHash"" ( + ""BlocksetID"", + ""Index"", + ""Hash"" + ) + VALUES ( + @BlocksetId, + @Index, + @Hash + ) + ", token) + .ConfigureAwait(false); + + dbnew.m_updateBlockVolumeCommand = await dbnew.m_connection.CreateCommandAsync(@" + UPDATE ""Block"" + SET ""VolumeID"" = @VolumeId + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size + ", token) + .ConfigureAwait(false); + + dbnew.m_insertTempBlockListHash = await dbnew.m_connection.CreateCommandAsync($@" + INSERT INTO ""{dbnew.m_tempblocklist}"" ( + ""BlocklistHash"", + ""BlockHash"", + ""Index"" + ) + VALUES ( + @BlocklistHash, + @BlockHash, + @Index + ) + ", token) + .ConfigureAwait(false); + + dbnew.m_insertSmallBlockset = await dbnew.m_connection.CreateCommandAsync($@" + INSERT OR IGNORE INTO ""{dbnew.m_tempsmalllist}"" ( + ""FileHash"", + ""BlockHash"", + ""BlockSize"" + ) + VALUES ( + @FileHash, + @BlockHash, + @BlockSize + ) + ", token) + .ConfigureAwait(false); + + dbnew.m_findBlocksetCommand = await dbnew.m_connection.CreateCommandAsync(@" + SELECT ""ID"" + FROM ""Blockset"" + WHERE + ""Length"" = @Length + AND ""FullHash"" = @FullHash + ", token) + .ConfigureAwait(false); + + dbnew.m_findMetadatasetCommand = await dbnew.m_connection.CreateCommandAsync(@" + SELECT ""Metadataset"".""ID"" + FROM + ""Metadataset"", + ""Blockset"" + WHERE + ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" + AND ""Blockset"".""FullHash"" = @FullHash + AND ""Blockset"".""Length"" = @Length + ", token) + .ConfigureAwait(false); + + dbnew.m_findFilesetCommand = await dbnew.m_connection.CreateCommandAsync(@" + SELECT ""ID"" + FROM ""FileLookup"" + WHERE + ""PrefixID"" = @PrefixId + AND ""Path"" = @Path + AND ""BlocksetID"" = @BlocksetId + AND ""MetadataID"" = @MetadataId + ", token) + .ConfigureAwait(false); + + dbnew.m_findTempBlockListHashCommand = await dbnew.m_connection.CreateCommandAsync($@" + SELECT DISTINCT ""BlockListHash"" + FROM ""{dbnew.m_tempblocklist}"" + WHERE ""BlockListHash"" = @BlocklistHash + ", token) + .ConfigureAwait(false); + + dbnew.m_findHashBlockCommand = await dbnew.m_connection.CreateCommandAsync(@" + SELECT ""VolumeID"" + FROM ""Block"" + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size + ", token) + .ConfigureAwait(false); + + dbnew.m_insertBlockCommand = await dbnew.m_connection.CreateCommandAsync(@" + INSERT INTO ""Block"" ( + ""Hash"", + ""Size"", + ""VolumeID"" + ) + VALUES ( + @Hash, + @Size, + @VolumeId + ) + ", token) + .ConfigureAwait(false); + + dbnew.m_insertDuplicateBlockCommand = await dbnew.m_connection.CreateCommandAsync(@" + INSERT OR IGNORE INTO ""DuplicateBlock"" ( + ""BlockID"", + ""VolumeID"" + ) + VALUES ( + ( + SELECT ""ID"" + FROM ""Block"" + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size + ), + @VolumeId + ) + ", token) + .ConfigureAwait(false); + + return dbnew; } - public void FindMissingBlocklistHashes(long hashsize, long blocksize, IDbTransaction transaction) + /// + /// Identifies and inserts any missing blocks and blockset entries into the database by comparing temporary tables of blocklist and small blockset data with the main tables, ensuring that all required block and blockset relationships are present for database consistency. + /// + /// The size of the hashes in the blocklist. + /// The size of the blocks in the blocklist. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that completes when the operation is finished. + public async Task FindMissingBlocklistHashes(long hashsize, long blocksize, CancellationToken token) { - using (var cmd = m_connection.CreateCommand(transaction)) + await using var cmd = m_connection.CreateCommand(m_rtr); + //Update all small blocklists and matching blocks + + var selectSmallBlocks = $@" + SELECT + ""BlockHash"", + ""BlockSize"" + FROM ""{m_tempsmalllist}"" + "; + + var selectBlockHashes = $@" + SELECT + ""BlockHash"" AS ""FullHash"", + ""BlockSize"" AS ""Length"" + FROM ( + {SELECT_BLOCKLIST_ENTRIES(blocksize, hashsize, m_tempblocklist, blocksize / hashsize)} + ) + "; + + var selectAllBlocks = @$" + SELECT DISTINCT + ""FullHash"", + ""Length"" + FROM ( + {selectBlockHashes} + UNION {selectSmallBlocks} + ) + "; + + var selectNewBlocks = $@" + SELECT + ""FullHash"" AS ""Hash"", + ""Length"" AS ""Size"", + -1 AS ""VolumeID"" + FROM ( + SELECT + ""A"".""FullHash"", + ""A"".""Length"", + CASE + WHEN ""B"".""Hash"" IS NULL + THEN '' + ELSE ""B"".""Hash"" + END AS ""Hash"", + CASE + WHEN ""B"".""Size"" is NULL + THEN -1 + ELSE ""B"".""Size"" + END AS ""Size"" + FROM ({selectAllBlocks}) ""A"" + LEFT OUTER JOIN ""Block"" ""B"" + ON ""B"".""Hash"" = ""A"".""FullHash"" + AND ""B"".""Size"" = ""A"".""Length"" + ) + WHERE + ""FullHash"" != ""Hash"" + AND ""Length"" != ""Size"" + "; + + var insertBlocksCommand = @$" + INSERT INTO ""Block"" ( + ""Hash"", + ""Size"", + ""VolumeID"" + ) + {selectNewBlocks} + "; + + // Insert all known blocks into block table with volumeid = -1 + await cmd.ExecuteNonQueryAsync(insertBlocksCommand, token) + .ConfigureAwait(false); + + // TODO: The BlocklistHash join seems to be unnecessary, but the join might be required to work around a from really old versions of Duplicati + // this could be used instead + //@") D, ""Block"" WHERE ""BlockQuery"".""BlockHash"" = ""Block"".""Hash"" AND ""BlockQuery"".""BlockSize"" = ""Block"".""Size"" "; + var selectBlocklistBlocksetEntries = $@" + SELECT + ""E"".""BlocksetID"" AS ""BlocksetID"", + ""D"".""FullIndex"" AS ""Index"", + ""F"".""ID"" AS ""BlockID"" + FROM + ({SELECT_BLOCKLIST_ENTRIES(blocksize, hashsize, m_tempblocklist, blocksize / hashsize)}) ""D"", + ""BlocklistHash"" ""E"", + ""Block"" ""F"", + ""Block"" ""G"" + WHERE + ""D"".""BlocksetID"" = ""E"".""BlocksetID"" + AND ""D"".""BlocklistHash"" = ""E"".""Hash"" + AND ""D"".""BlocklistSize"" = ""G"".""Size"" + AND ""D"".""BlocklistHash"" = ""G"".""Hash"" + AND ""D"".""Blockhash"" = ""F"".""Hash"" + AND ""D"".""BlockSize"" = ""F"".""Size"" + "; + + var selectBlocksetEntries = $@" + SELECT + ""Blockset"".""ID"" AS ""BlocksetID"", + 0 AS ""Index"", + ""Block"".""ID"" AS ""BlockID"" + FROM + ""Blockset"", + ""Block"", + ""{m_tempsmalllist}"" ""S"" + WHERE + ""Blockset"".""Fullhash"" = ""S"".""FileHash"" + AND ""S"".""BlockHash"" = ""Block"".""Hash"" + AND ""S"".""BlockSize"" = ""Block"".""Size"" + AND ""Blockset"".""Length"" = ""S"".""BlockSize"" + AND ""Blockset"".""Length"" <= {blocksize} + "; + + var selectAllBlocksetEntries = @$" + {selectBlocklistBlocksetEntries} + UNION + {selectBlocksetEntries} + "; + + var selectFiltered = @$" + SELECT DISTINCT + ""H"".""BlocksetID"", + ""H"".""Index"", + ""H"".""BlockID"" + FROM ({selectAllBlocksetEntries}) H + WHERE ( + ""H"".""BlocksetID"" + || ':' + || ""H"".""Index"" + ) NOT IN ( + SELECT ( + ""ExistingBlocksetEntries"".""BlocksetID"" + || ':' + || ""ExistingBlocksetEntries"".""Index"" + ) + FROM ""BlocksetEntry"" ""ExistingBlocksetEntries"" + ) + "; + + var insertBlocksetEntriesCommand = @$" + INSERT INTO ""BlocksetEntry"" ( + ""BlocksetID"", + ""Index"", + ""BlockID"" + ) + {selectFiltered} + "; + + try { - //Update all small blocklists and matching blocks - - var selectSmallBlocks = FormatInvariant($@"SELECT ""BlockHash"", ""BlockSize"" FROM ""{m_tempsmalllist}"""); - - var selectBlockHashes = FormatInvariant($@"SELECT ""BlockHash"" AS ""FullHash"", ""BlockSize"" AS ""Length"" FROM ( {SELECT_BLOCKLIST_ENTRIES(blocksize, hashsize, m_tempblocklist, blocksize / hashsize)} )"); - - var selectAllBlocks = @"SELECT DISTINCT ""FullHash"", ""Length"" FROM (" + selectBlockHashes + " UNION " + selectSmallBlocks + " )"; - - var selectNewBlocks = FormatInvariant($@"SELECT ""FullHash"" AS ""Hash"", ""Length"" AS ""Size"", -1 AS ""VolumeID"" -FROM (SELECT ""A"".""FullHash"", ""A"".""Length"", CASE WHEN ""B"".""Hash"" IS NULL THEN '' ELSE ""B"".""Hash"" END AS ""Hash"", CASE WHEN ""B"".""Size"" is NULL THEN -1 ELSE ""B"".""Size"" END AS ""Size"" FROM ({selectAllBlocks}) A -LEFT OUTER JOIN ""Block"" B ON ""B"".""Hash"" = ""A"".""FullHash"" AND ""B"".""Size"" = ""A"".""Length"" ) -WHERE ""FullHash"" != ""Hash"" AND ""Length"" != ""Size"" "); - - var insertBlocksCommand = - @"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") " + - selectNewBlocks; - - // Insert all known blocks into block table with volumeid = -1 - cmd.ExecuteNonQuery(insertBlocksCommand); - - var selectBlocklistBlocksetEntries = FormatInvariant( - // TODO: The BlocklistHash join seems to be unnecessary, but the join might be required to work around a from really old versions of Duplicati - // this could be used instead - //@") D, ""Block"" WHERE ""BlockQuery"".""BlockHash"" = ""Block"".""Hash"" AND ""BlockQuery"".""BlockSize"" = ""Block"".""Size"" "; - $@"SELECT ""E"".""BlocksetID"" AS ""BlocksetID"", ""D"".""FullIndex"" AS ""Index"", ""F"".""ID"" AS ""BlockID"" FROM ( {SELECT_BLOCKLIST_ENTRIES(blocksize, hashsize, m_tempblocklist, blocksize / hashsize)}) D, ""BlocklistHash"" E, ""Block"" F, ""Block"" G WHERE ""D"".""BlocksetID"" = ""E"".""BlocksetID"" AND ""D"".""BlocklistHash"" = ""E"".""Hash"" AND ""D"".""BlocklistSize"" = ""G"".""Size"" AND ""D"".""BlocklistHash"" = ""G"".""Hash"" AND ""D"".""Blockhash"" = ""F"".""Hash"" AND ""D"".""BlockSize"" = ""F"".""Size"" " - ); - - var selectBlocksetEntries = FormatInvariant($@"SELECT ""Blockset"".""ID"" AS ""BlocksetID"", 0 AS ""Index"", ""Block"".""ID"" AS ""BlockID"" FROM ""Blockset"", ""Block"", ""{m_tempsmalllist}"" S WHERE ""Blockset"".""Fullhash"" = ""S"".""FileHash"" AND ""S"".""BlockHash"" = ""Block"".""Hash"" AND ""S"".""BlockSize"" = ""Block"".""Size"" AND ""Blockset"".""Length"" = ""S"".""BlockSize"" AND ""Blockset"".""Length"" <= {blocksize} "); - - var selectAllBlocksetEntries = - selectBlocklistBlocksetEntries + - @" UNION " + - selectBlocksetEntries; - - var selectFiltered = - @"SELECT DISTINCT ""H"".""BlocksetID"", ""H"".""Index"", ""H"".""BlockID"" FROM (" + - selectAllBlocksetEntries + - @") H WHERE (""H"".""BlocksetID"" || ':' || ""H"".""Index"") NOT IN (SELECT (""ExistingBlocksetEntries"".""BlocksetID"" || ':' || ""ExistingBlocksetEntries"".""Index"") FROM ""BlocksetEntry"" ""ExistingBlocksetEntries"" )"; - - var insertBlocksetEntriesCommand = - @"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") " + selectFiltered; - - try - { - cmd.ExecuteNonQuery(insertBlocksetEntriesCommand); - } - catch (Exception ex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "BlocksetInsertFailed", ex, "Blockset insert failed, committing temporary tables for trace purposes"); - - using (var fixcmd = m_connection.CreateCommand()) - { - fixcmd.ExecuteNonQuery(FormatInvariant($@"CREATE TABLE ""{m_tempblocklist}-Failure"" AS SELECT * FROM ""{m_tempblocklist}"" ")); - fixcmd.ExecuteNonQuery(FormatInvariant($@"CREATE TABLE ""{m_tempsmalllist}-Failure"" AS SELECT * FROM ""{m_tempsmalllist}"" ")); - } - - throw new Exception("The recreate failed, please create a bug-report from this database and send it to the developers for further analysis"); - } + await cmd.ExecuteNonQueryAsync(insertBlocksetEntriesCommand, token) + .ConfigureAwait(false); } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "BlocksetInsertFailed", ex, "Blockset insert failed, committing temporary tables for trace purposes"); + + await using (var fixcmd = m_connection.CreateCommand(m_rtr)) + { + await fixcmd.ExecuteNonQueryAsync($@" + CREATE TABLE ""{m_tempblocklist}-Failure"" AS + SELECT * + FROM ""{m_tempblocklist}"" + ", token) + .ConfigureAwait(false); + + await fixcmd.ExecuteNonQueryAsync($@" + CREATE TABLE ""{m_tempsmalllist}-Failure"" AS + SELECT * + FROM ""{m_tempsmalllist}"" + ", token) + .ConfigureAwait(false); + } + + throw new Exception("The recreate failed, please create a bug-report from this database and send it to the developers for further analysis"); + } + + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } /// @@ -233,159 +637,312 @@ WHERE ""FullHash"" != ""Hash"" AND ""Length"" != ""Size"" "); /// temp small list structure: filehash, blockhash, blocksize: as the small files are defined /// by the fact that they are contained in a single block, blockhash is the same as the filehash, /// and blocksize can vary from 0 to the configured block size for the backup - public void AddBlockAndBlockSetEntryFromTemp(long hashsize, long blocksize, IDbTransaction transaction, bool hashOnly = false) + /// The size of the hashes in the blocklist. + /// The size of the blocks in the blocklist. + /// If true, only hash entries are processed, ignoring small blocks. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that completes when the operation is finished. + public async Task AddBlockAndBlockSetEntryFromTemp(long hashsize, long blocksize, bool hashOnly, CancellationToken token) { + // TODO should values be parameters, rather than hardcoded into the SQL queries? + await using var cmd = m_connection.CreateCommand(m_rtr); + var extra = hashOnly ? "" : $@" + UNION + SELECT + ""TS"".""BlockHash"", + ""TS"".""BlockSize"" + FROM ""{m_tempsmalllist}"" ""TS"" + WHERE NOT EXISTS ( + SELECT ""X"" + FROM Block AS ""B"" + WHERE + ""B"".""Hash"" = ""TS"".""BlockHash"" + AND ""B"".""Size"" = ""TS"".""BlockSize"" + ) + "; + var insertBlocksCommand = $@" + INSERT INTO ""Block"" ( + ""Hash"", + ""Size"", + ""VolumeID"" + ) + SELECT DISTINCT + ""BlockHash"" AS ""Hash"", + ""BlockSize"" AS ""Size"", + -1 AS ""VolumeID"" + FROM ( + SELECT + ""NB"".""BlockHash"", + MIN({blocksize}, ""BS"".""Length"" - ((""NB"".""Index"" + (BH.""Index"" * {blocksize / hashsize})) * {blocksize})) AS ""BlockSize"" + FROM ( + SELECT + ""TBL"".""BlockListHash"", + ""TBL"".""BlockHash"", + ""TBL"".""Index"" FROM ""{m_tempblocklist}"" ""TBL"" + LEFT OUTER JOIN ""Block"" ""B"" + ON (""B"".""Hash"" = ""TBL"".""BlockHash"") + WHERE ""B"".""Hash"" IS NULL + ) ""NB"" + JOIN ""BlocklistHash"" ""BH"" + ON (""BH"".""Hash"" = ""NB"".""BlocklistHash"") + JOIN ""Blockset"" ""BS"" + ON (""BS"".""ID"" = ""BH"".""Blocksetid"") + {extra} + ) + "; - using (var cmd = m_connection.CreateCommand(transaction)) + extra = hashOnly ? "" : $@" + UNION + SELECT + ""BS"".""ID"" AS ""BlocksetID"", + 0 AS ""Index"", + ""BL"".""ID"" AS ""BlockID"" + FROM ""{m_tempsmalllist}"" ""TS"" + JOIN ""Blockset"" ""BS"" + ON ( + ""BS"".""FullHash"" = ""TS"".""FileHash"" + AND ""BS"".""Length"" = ""TS"".""BlockSize"" + AND ""BS"".""Length"" <= {blocksize} + ) + JOIN ""Block"" ""BL"" + ON ( + ""BL"".""Hash"" = ""TS"".""BlockHash"" + AND ""BL"".""Size"" = ""TS"".""BlockSize"" + ) + LEFT OUTER JOIN ""BlocksetEntry"" ""BE"" + ON ( + ""BE"".""BlocksetID"" = ""BS"".""ID"" + AND ""BE"".""Index"" = 0 + ) + WHERE ""BE"".""BlocksetID"" IS NULL + "; + var insertBlocksetEntriesCommand = $@" + INSERT INTO ""BlocksetEntry"" ( + ""BlocksetID"", + ""Index"", + ""BlockID"" + ) + SELECT DISTINCT + ""BH"".""blocksetid"", + (""BH"".""Index"" * {blocksize / hashsize}) + ""TBL"".""Index"" as ""FullIndex"", + ""BK"".""ID"" AS ""BlockID"" + FROM ""{m_tempblocklist}"" ""TBL"" + JOIN ""blocklisthash"" ""BH"" + ON (""BH"".""Hash"" = ""TBL"".""blocklisthash"") + JOIN ""Block"" ""BK"" + ON (""BK"".""Hash"" = ""TBL"".""BlockHash"") + LEFT OUTER JOIN ""BlocksetEntry"" ""BE"" + ON ( + BE.""BlockSetID"" = ""BH"".""BlocksetID"" + AND BE.""Index"" = (""BH"".""Index"" * {blocksize / hashsize})+""TBL"".""Index"" + ) + WHERE ""BE"".""BlockSetID"" IS NULL + {extra} + "; + + try { - var insertBlocksCommand = FormatInvariant($@"INSERT INTO BLOCK (Hash, Size, VolumeID) -SELECT DISTINCT BlockHash AS Hash, BlockSize AS Size, -1 AS VolumeID FROM -( -SELECT NB.BlockHash, -MIN({blocksize}, BS.Length - ((NB.""Index"" + (BH.""Index"" * {blocksize / hashsize})) * {blocksize})) AS BlockSize -FROM ( - SELECT TBL.BlockListHash, TBL.BlockHash, TBL.""Index"" FROM {m_tempblocklist} TBL - LEFT OUTER JOIN Block B ON (B.Hash = TBL.BlockHash) - WHERE B.Hash IS NULL - ) NB -JOIN BlocklistHash BH ON (BH.Hash = NB.BlocklistHash) -JOIN Blockset BS ON (BS.ID = BH.Blocksetid) "); - if (!hashOnly) + // Insert discovered new blocks into block table with volumeid = -1 + await cmd.ExecuteNonQueryAsync(insertBlocksCommand, token) + .ConfigureAwait(false); + + // Insert corresponding entries into blockset + await cmd.ExecuteNonQueryAsync(insertBlocksetEntriesCommand, token) + .ConfigureAwait(false); + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "BlockOrBlocksetInsertFailed", ex, "Block or Blockset insert failed, committing temporary tables for trace purposes"); + + await using (var fixcmd = m_connection.CreateCommand(m_rtr)) { - insertBlocksCommand += FormatInvariant($@" UNION -SELECT TS.BlockHash, TS.BlockSize FROM -{m_tempsmalllist} TS -WHERE NOT EXISTS (SELECT ""X"" FROM Block AS B WHERE - B.Hash = TS.BlockHash AND - B.Size = TS.BlockSize) -)"); + await fixcmd.ExecuteNonQueryAsync($@" + CREATE TABLE ""{m_tempblocklist}_Failure"" AS + SELECT * + FROM ""{m_tempblocklist}"" + ", token) + .ConfigureAwait(false); + + await fixcmd.ExecuteNonQueryAsync($@" + CREATE TABLE ""{m_tempsmalllist}_Failure"" + AS SELECT * + FROM ""{m_tempsmalllist}"" + ", token) + .ConfigureAwait(false); } - var insertBlocksetEntriesCommand = FormatInvariant($@"INSERT INTO BlocksetEntry (BlocksetID, ""Index"", BlockID) -SELECT DISTINCT BH.blocksetid, (BH.""Index"" * {blocksize / hashsize})+TBL.""Index"" as FullIndex, BK.ID AS BlockID -FROM {m_tempblocklist} TBL - JOIN blocklisthash BH ON (BH.hash = TBL.blocklisthash) - JOIN block BK ON (BK.Hash = TBL.BlockHash) - LEFT OUTER JOIN BlocksetEntry BE ON (BE.BlockSetID = BH.BlocksetID AND BE.""Index"" = (BH.""Index"" * {blocksize / hashsize})+TBL.""Index"") -WHERE -BE.BlockSetID IS NULL "); - if (!hashOnly) - { - insertBlocksetEntriesCommand += FormatInvariant($@" UNION -SELECT BS.ID AS BlocksetID, 0 AS ""Index"", BL.ID AS BlockID -FROM {m_tempsmalllist} TS - JOIN Blockset BS ON (BS.FullHash = TS.FileHash AND - BS.Length = TS.BlockSize AND - BS.Length <= {blocksize}) - JOIN Block BL ON (BL.Hash = TS.BlockHash AND - BL.Size = TS.BlockSize) - LEFT OUTER JOIN BlocksetEntry BE ON (BE.BlocksetID = BS.ID AND BE.""Index"" = 0) -WHERE -BE.BlocksetID IS NULL "); - } - - try - { - // Insert discovered new blocks into block table with volumeid = -1 - cmd.ExecuteNonQuery(insertBlocksCommand); - // Insert corresponding entries into blockset - cmd.ExecuteNonQuery(insertBlocksetEntriesCommand); - } - catch (Exception ex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "BlockOrBlocksetInsertFailed", ex, "Block or Blockset insert failed, committing temporary tables for trace purposes"); - - using (var fixcmd = m_connection.CreateCommand()) - { - fixcmd.ExecuteNonQuery(FormatInvariant($@"CREATE TABLE ""{m_tempblocklist}_Failure"" AS SELECT * FROM ""{m_tempblocklist}"" ")); - fixcmd.ExecuteNonQuery(FormatInvariant($@"CREATE TABLE ""{m_tempsmalllist}_Failure"" AS SELECT * FROM ""{m_tempsmalllist}"" ")); - } - - throw new Exception("The recreate failed, please create a bug-report from this database and send it to the developers for further analysis"); - } + throw new Exception("The recreate failed, please create a bug-report from this database and send it to the developers for further analysis"); } } - public void AddDirectoryEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid, IDbTransaction transaction) + /// + /// Adds a directory entry to the FilesetEntry table, linking it to a fileset and a path prefix. + /// + /// The ID of the fileset to which the entry belongs. + /// The ID of the path prefix for the entry. + /// The path of the directory entry. + /// The last modified time of the directory entry. + /// The ID of the metadata associated with the directory entry. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that when completed, indicates that the directory entry has been added. + public async Task AddDirectoryEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid, CancellationToken token) { - AddEntry(filesetid, pathprefixid, path, time, FOLDER_BLOCKSET_ID, metadataid, transaction); + await AddEntry(filesetid, pathprefixid, path, time, FOLDER_BLOCKSET_ID, metadataid, token) + .ConfigureAwait(false); } - public void AddSymlinkEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid, IDbTransaction transaction) + /// + /// Adds a symlink entry to the FilesetEntry table, linking it to a fileset and a path prefix. + /// + /// The ID of the fileset to which the entry belongs. + /// The ID of the path prefix for the entry. + /// The path of the symlink entry. + /// The last modified time of the symlink entry. + /// The ID of the metadata associated with the symlink entry. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that when completed, indicates that the symlink entry has been added. + public async Task AddSymlinkEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid, CancellationToken token) { - AddEntry(filesetid, pathprefixid, path, time, SYMLINK_BLOCKSET_ID, metadataid, transaction); + await AddEntry(filesetid, pathprefixid, path, time, SYMLINK_BLOCKSET_ID, metadataid, token) + .ConfigureAwait(false); } - public void AddFileEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, IDbTransaction transaction) + /// + /// Adds a file entry to the FilesetEntry table, linking it to a fileset and a path prefix. + /// + /// The ID of the fileset to which the entry belongs. + /// The ID of the path prefix for the entry. + /// The path of the file entry. + /// The last modified time of the file entry. + /// The ID of the blockset associated with the file entry. + /// The ID of the metadata associated with the file entry. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that when completed, indicates that the file entry has been added. + public async Task AddFileEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, CancellationToken token) { - AddEntry(filesetid, pathprefixid, path, time, blocksetid, metadataid, transaction); + await AddEntry(filesetid, pathprefixid, path, time, blocksetid, metadataid, token) + .ConfigureAwait(false); } - private void AddEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, IDbTransaction transaction) + /// + /// Adds a file and fileset entry to the FilesetEntry table, linking it to a fileset and a path prefix. + /// If the file already exists, it retrieves its ID; otherwise, it inserts a new file entry. + /// Then, it inserts the fileset entry linking the file to the fileset. + /// + /// The ID of the fileset to which the entry belongs. + /// The ID of the path prefix for the entry. + /// The path of the file entry. + /// The last modified time of the file entry. + /// The ID of the blockset associated with the file entry. + /// The ID of the metadata associated with the file entry. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that when completed, indicates that the file and fileset entry has been added. + private async Task AddEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, CancellationToken token) { - var fileid = m_findFilesetCommand.SetTransaction(transaction) + var fileid = await m_findFilesetCommand + .SetTransaction(m_rtr) .SetParameterValue("@PrefixId", pathprefixid) .SetParameterValue("@Path", path) .SetParameterValue("@BlocksetId", blocksetid) .SetParameterValue("@MetadataId", metadataid) - .ExecuteScalarInt64(-1); + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); if (fileid < 0) { - fileid = m_insertFileCommand.SetTransaction(transaction) + fileid = await m_insertFileCommand + .SetTransaction(m_rtr) .SetParameterValue("@PrefixId", pathprefixid) .SetParameterValue("@Path", path) .SetParameterValue("@BlocksetId", blocksetid) .SetParameterValue("@MetadataId", metadataid) - .ExecuteScalarInt64(-1); + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); } - m_insertFilesetEntryCommand.SetTransaction(transaction) + await m_insertFilesetEntryCommand + .SetTransaction(m_rtr) .SetParameterValue("@FilesetId", filesetid) .SetParameterValue("@FileId", fileid) .SetParameterValue("@LastModified", time.ToUniversalTime().Ticks) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); } - public long AddMetadataset(string metahash, long metahashsize, IEnumerable metablocklisthashes, long expectedmetablocklisthashes, IDbTransaction transaction) + /// + /// Adds a new metadataset to the database, creating a blockset if it does not already exist. + /// If the metadataset already exists, it returns its ID. + /// + /// The full hash of the metadataset. + /// The size of the metadataset in bytes. + /// A collection of blocklist hashes associated with the metadataset. + /// The expected number of blocklist hashes for the metadataset. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that returns the ID of the added or existing metadataset. + public async Task AddMetadataset(string metahash, long metahashsize, IEnumerable metablocklisthashes, long expectedmetablocklisthashes, CancellationToken token) { var metadataid = -1L; if (metahash == null) return metadataid; - metadataid = m_findMetadatasetCommand.SetTransaction(transaction) + metadataid = await m_findMetadatasetCommand + .SetTransaction(m_rtr) .SetParameterValue("@FullHash", metahash) .SetParameterValue("@Length", metahashsize) - .ExecuteScalarInt64(-1); + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); + if (metadataid != -1) return metadataid; - var blocksetid = AddBlockset(metahash, metahashsize, metablocklisthashes, expectedmetablocklisthashes, transaction); + var blocksetid = + await AddBlockset(metahash, metahashsize, metablocklisthashes, expectedmetablocklisthashes, token) + .ConfigureAwait(false); - return m_insertMetadatasetCommand.SetTransaction(transaction) + metadataid = await m_insertMetadatasetCommand + .SetTransaction(m_rtr) .SetParameterValue("@BlocksetId", blocksetid) - .ExecuteScalarInt64(-1); + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); + + return metadataid; } - public long AddBlockset(string fullhash, long size, IEnumerable blocklisthashes, long expectedblocklisthashes, IDbTransaction transaction) + /// + /// Adds a new blockset to the database, creating it if it does not already exist. + /// If the blockset already exists, it returns its ID. + /// + /// The full hash of the blockset. + /// The size of the blockset in bytes. + /// A collection of blocklist hashes associated with the blockset. + /// The expected number of blocklist hashes for the blockset. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that returns the ID of the added or existing blockset. + public async Task AddBlockset(string fullhash, long size, IEnumerable blocklisthashes, long expectedblocklisthashes, CancellationToken token) { - var blocksetid = m_findBlocksetCommand.SetTransaction(transaction) + var blocksetid = await m_findBlocksetCommand + .SetTransaction(m_rtr) .SetParameterValue("@Length", size) .SetParameterValue("@FullHash", fullhash) - .ExecuteScalarInt64(-1); + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); + if (blocksetid != -1) return blocksetid; - blocksetid = m_insertBlocksetCommand.SetTransaction(transaction) + blocksetid = await m_insertBlocksetCommand + .SetTransaction(m_rtr) .SetParameterValue("@Length", size) .SetParameterValue("@FullHash", fullhash) - .ExecuteScalarInt64(-1); + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); long c = 0; if (blocklisthashes != null) { var index = 0L; - m_insertBlocklistHashCommand.SetTransaction(transaction) + m_insertBlocklistHashCommand + .SetTransaction(m_rtr) .SetParameterValue("@BlocksetId", blocksetid); foreach (var hash in blocklisthashes) @@ -395,9 +952,11 @@ BE.BlocksetID IS NULL "); c++; if (c <= expectedblocklisthashes) { - m_insertBlocklistHashCommand.SetParameterValue("@Index", index++) + await m_insertBlocklistHashCommand + .SetParameterValue("@Index", index++) .SetParameterValue("@Hash", hash) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); } } } @@ -409,173 +968,297 @@ BE.BlocksetID IS NULL "); return blocksetid; } - public bool UpdateBlock(string hash, long size, long volumeID, IDbTransaction transaction, ref bool anyChange) + /// + /// Updates the block information in the database, inserting a new block if it does not exist, + /// updating the volume ID if it does, or inserting a duplicate block entry if the block already exists with a different volume ID. + /// + /// The hash of the block to update. + /// The size of the block in bytes. + /// The ID of the volume to which the block belongs. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that returns a tuple indicating whether any changes were made and whether the block was newly inserted. + public async Task<(bool, bool)> UpdateBlock(string hash, long size, long volumeID, CancellationToken token) { - var currentVolumeId = m_findHashBlockCommand.SetTransaction(transaction) + var anyChange = false; + var currentVolumeId = await m_findHashBlockCommand + .SetTransaction(m_rtr) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .ExecuteScalarInt64(-2); + .ExecuteScalarInt64Async(-2, token) + .ConfigureAwait(false); if (currentVolumeId == volumeID) - return false; + return (anyChange, false); anyChange = true; if (currentVolumeId == -2) { //Insert - m_insertBlockCommand.SetTransaction(transaction) + await m_insertBlockCommand + .SetTransaction(m_rtr) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) .SetParameterValue("@VolumeId", volumeID) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); - return true; + return (anyChange, true); } else if (currentVolumeId == -1) { //Update - var c = m_updateBlockVolumeCommand.SetTransaction(transaction) + var c = await m_updateBlockVolumeCommand + .SetTransaction(m_rtr) .SetParameterValue("@VolumeId", volumeID) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + if (c != 1) throw new Exception($"Failed to update table, found {c} entries for key {hash} with size {size}"); - return true; + return (anyChange, true); } else { - m_insertDuplicateBlockCommand.SetParameterValue("@Hash", hash) + await m_insertDuplicateBlockCommand + .SetTransaction(m_rtr) + .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", size) .SetParameterValue("@VolumeId", volumeID) - .ExecuteNonQuery(transaction); + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); - return false; + return (anyChange, false); } } - public void AddSmallBlocksetLink(string filehash, string blockhash, long blocksize, IDbTransaction transaction) + /// + /// Adds a link between a small blockset and a block in the database. + /// This method is used to track small blocks that are part of a larger blockset, allowing for efficient storage and retrieval of small files. + /// + /// The hash of the file that contains the small block. + /// The hash of the small block. + /// The size of the small block in bytes. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that completes when the small blockset link has been added. + public async Task AddSmallBlocksetLink(string filehash, string blockhash, long blocksize, CancellationToken token) { - m_insertSmallBlockset.SetParameterValue("@FileHash", filehash) + await m_insertSmallBlockset + .SetTransaction(m_rtr) + .SetParameterValue("@FileHash", filehash) .SetParameterValue("@BlockHash", blockhash) .SetParameterValue("@BlockSize", blocksize) - .ExecuteNonQuery(transaction); + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); } - public bool AddTempBlockListHash(string hash, IEnumerable blocklisthashes, IDbTransaction transaction) + /// + /// Adds a temporary blocklist hash to the database, along with its associated block hashes. + /// This method is used to track temporary blocklists that are being processed, allowing for efficient management of blocklist data. + /// + /// The hash of the temporary blocklist. + /// A collection of block hashes associated with the temporary blocklist. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that returns a boolean indicating whether the temporary blocklist hash was successfully added. + public async Task AddTempBlockListHash(string hash, IEnumerable blocklisthashes, CancellationToken token) { - var r = m_findTempBlockListHashCommand.SetTransaction(transaction) + var r = await m_findTempBlockListHashCommand + .SetTransaction(m_rtr) .SetParameterValue("@BlocklistHash", hash) - .ExecuteScalar(); + .ExecuteScalarAsync(token) + .ConfigureAwait(false); + if (r != null && r != DBNull.Value) return false; - m_insertTempBlockListHash.SetTransaction(transaction) + m_insertTempBlockListHash + .SetTransaction(m_rtr) .SetParameterValue("@BlocklistHash", hash); var index = 0L; foreach (var s in blocklisthashes) { - m_insertTempBlockListHash.SetParameterValue("@BlockHash", s) + await m_insertTempBlockListHash + .SetParameterValue("@BlockHash", s) .SetParameterValue("@Index", index++) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); } return true; } - - public IEnumerable GetBlockLists(long volumeid) + /// + /// Retrieves a list of block hashes associated with a specific volume ID from the database. + /// This method is used to obtain the block hashes for a given volume, allowing for efficient retrieval of block data. + /// + /// The ID of the volume for which to retrieve block hashes. + /// A cancellation token to observe while waiting for the task to complete. + /// An asynchronous enumerable collection of block hashes associated with the specified volume ID. + public async IAsyncEnumerable GetBlockLists(long volumeid, [EnumeratorCancellation] CancellationToken token) { - using (var cmd = m_connection.CreateCommand(@"SELECT DISTINCT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"", ""Block"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" AND ""Block"".""VolumeID"" = @VolumeId")) - { - cmd.SetParameterValue("@VolumeId", volumeid); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - yield return rd.ConvertValueToString(0) ?? ""; - } + await using var cmd = m_connection.CreateCommand(@" + SELECT DISTINCT ""BlocklistHash"".""Hash"" + FROM + ""BlocklistHash"", + ""Block"" + WHERE + ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" + AND ""Block"".""VolumeID"" = @VolumeId + ") + .SetTransaction(m_rtr) + .SetParameterValue("@VolumeId", volumeid); + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + yield return rd.ConvertValueToString(0) ?? ""; } - public IEnumerable GetMissingBlockListVolumes(int passNo, long blocksize, long hashsize, bool forceBlockUse) + /// + /// Retrieves a list of remote volumes that are missing blocks based on the specified parameters. + /// This method is used to identify volumes that need to be processed for missing blocks, allowing for efficient management of remote storage. + /// + /// The current pass number, used to determine the state of processing. + /// The size of the blocks in the blocklist. + /// The size of the hashes in the blocklist. + /// A boolean indicating whether to force the use of blocks, even if they are not missing. + /// A cancellation token to observe while waiting for the task to complete. + /// An asynchronous enumerable collection of remote volumes that are missing blocks. + public async IAsyncEnumerable GetMissingBlockListVolumes(int passNo, long blocksize, long hashsize, bool forceBlockUse, [EnumeratorCancellation] CancellationToken token) { - using (var cmd = m_connection.CreateCommand()) + await using var cmd = m_connection.CreateCommand(m_rtr); + var selectCommand = @" + SELECT DISTINCT + ""RemoteVolume"".""Name"", + ""RemoteVolume"".""Hash"", + ""RemoteVolume"".""Size"", + ""RemoteVolume"".""ID"" + FROM ""RemoteVolume"" + "; + + var missingBlocklistEntries = $@" + SELECT ""BlocklistHash"".""Hash"" + FROM ""BlocklistHash"" + LEFT OUTER JOIN ""BlocksetEntry"" + ON ""BlocksetEntry"".""Index"" = (""BlocklistHash"".""Index"" * {blocksize / hashsize}) + AND ""BlocksetEntry"".""BlocksetID"" = ""BlocklistHash"".""BlocksetID"" + WHERE ""BlocksetEntry"".""BlocksetID"" IS NULL + "; + + var missingBlockInfo = @" + SELECT ""VolumeID"" + FROM ""Block"" + WHERE + ""VolumeID"" < 0 + AND ""Size"" > 0 + "; + + var missingBlocklistVolumes = $@" + SELECT ""VolumeID"" + FROM + ""Block"", + ({missingBlocklistEntries}) ""A"" + WHERE ""A"".""Hash"" = ""Block"".""Hash"" + "; + + var countMissingInformation = $@" + SELECT COUNT(*) + FROM ( + SELECT DISTINCT ""VolumeID"" + FROM ( + {missingBlockInfo} + UNION {missingBlocklistVolumes} + ) + ) + "; + + if (passNo == 0) { - var selectCommand = @"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"", ""RemoteVolume"".""ID"" FROM ""RemoteVolume"""; + // On the first pass, we select all the volumes we know we need, + // which may be an empty list + cmd.SetCommandAndParameters($@" + {selectCommand} + WHERE ""ID"" IN ({missingBlocklistVolumes}) + "); - var missingBlocklistEntries = - FormatInvariant($@"SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"" LEFT OUTER JOIN ""BlocksetEntry"" ON ""BlocksetEntry"".""Index"" = (""BlocklistHash"".""Index"" * {blocksize / hashsize}) AND ""BlocksetEntry"".""BlocksetID"" = ""BlocklistHash"".""BlocksetID"" WHERE ""BlocksetEntry"".""BlocksetID"" IS NULL"); + // Reset the list + m_proccessedVolumes.Clear(); + } + else + { + //On anything but the first pass, we check if we are done + var r = await cmd + .SetCommandAndParameters(countMissingInformation) + .ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false); - var missingBlockInfo = - @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""VolumeID"" < 0 AND SIZE > 0"; + if (r == 0 && !forceBlockUse) + yield break; - var missingBlocklistVolumes = FormatInvariant( - $@"SELECT ""VolumeID"" FROM ""Block"", ({missingBlocklistEntries}) A WHERE ""A"".""Hash"" = ""Block"".""Hash"" " - ); - - var countMissingInformation = FormatInvariant($@"SELECT COUNT(*) FROM (SELECT DISTINCT ""VolumeID"" FROM ({missingBlockInfo} UNION {missingBlocklistVolumes}))"); - - if (passNo == 0) + if (passNo == 1) { - // On the first pass, we select all the volumes we know we need, - // which may be an empty list - cmd.SetCommandAndParameters(selectCommand + FormatInvariant($@" WHERE ""ID"" IN ({missingBlocklistVolumes})")); + // On the second pass, we select all volumes that are not mentioned in the db + + var mentionedVolumes = @" + SELECT DISTINCT ""VolumeID"" + FROM ""Block"" + "; + + cmd + .SetCommandAndParameters($@" + {selectCommand} + WHERE + ""ID"" NOT IN ({mentionedVolumes}) + AND ""Type"" = @Type + ") + .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); + - // Reset the list - m_proccessedVolumes.Clear(); } else { - //On anything but the first pass, we check if we are done - var r = cmd.ExecuteScalarInt64(countMissingInformation, 0); - if (r == 0 && !forceBlockUse) - yield break; - - if (passNo == 1) - { - // On the second pass, we select all volumes that are not mentioned in the db - - var mentionedVolumes = - @"SELECT DISTINCT ""VolumeID"" FROM ""Block"" "; - - cmd.SetCommandAndParameters(selectCommand + FormatInvariant($@" WHERE ""ID"" NOT IN ({mentionedVolumes}) AND ""Type"" = @Type ")) - .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); - - } - else - { - // On the final pass, we select all volumes - // the filter will ensure that we do not download anything twice - cmd.SetCommandAndParameters(selectCommand + @" WHERE ""Type"" = @Type ") - .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); - } + // On the final pass, we select all volumes + // the filter will ensure that we do not download anything twice + cmd + .SetCommandAndParameters($@" + {selectCommand} + WHERE ""Type"" = @Type + ") + .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()); } + } - using (var rd = cmd.ExecuteReader()) + await using var rd = await cmd + .ExecuteReaderAsync(token) + .ConfigureAwait(false); + + while (await rd.ReadAsync(token).ConfigureAwait(false)) + { + var volumeID = rd.ConvertValueToInt64(3); + + // Guard against multiple downloads of the same file + if (m_proccessedVolumes.TryAdd(volumeID, volumeID)) { - while (rd.Read()) - { - var volumeID = rd.ConvertValueToInt64(3); - - // Guard against multiple downloads of the same file - if (!m_proccessedVolumes.ContainsKey(volumeID)) - { - m_proccessedVolumes.Add(volumeID, volumeID); - - yield return new RemoteVolume( - rd.ConvertValueToString(0), - rd.ConvertValueToString(1), - rd.ConvertValueToInt64(2, -1) - ); - } - } + yield return new RemoteVolume( + rd.ConvertValueToString(0), + rd.ConvertValueToString(1), + rd.ConvertValueToInt64(2, -1) + ); } } } - public void CleanupMissingVolumes() + /// + /// Cleans up missing volumes by replacing blocks for non-present remote files with existing ones, + /// removing references to non-present remote files, and marking index files for later removal. + /// + /// A cancellation token to observe while waiting for the task to complete. + /// A task that completes when the cleanup operation is finished. + public async Task CleanupMissingVolumes(CancellationToken token) { var tablename = "SwapBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); @@ -585,110 +1268,226 @@ BE.BlocksetID IS NULL "); // existing ones (as recorded in the DuplicateBlock table) // The second part removes references to the non-present remote files, // and marks the index files that pointed to them, such that they will be removed later on - var sql = FormatInvariant($@" -CREATE TEMPORARY TABLE ""{tablename}"" AS -SELECT - ""A"".""ID"" AS ""BlockID"", ""A"".""VolumeID"" AS ""SourceVolumeID"", ""A"".""State"" AS ""SourceVolumeState"", ""B"".""VolumeID"" AS ""TargetVolumeID"", ""B"".""State"" AS ""TargetVolumeState"" -FROM - (SELECT - ""Block"".""ID"", ""Block"".""VolumeID"", ""Remotevolume"".""State"" - FROM ""Block"", ""Remotevolume"" - WHERE ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Remotevolume"".""State"" = '{RemoteVolumeState.Temporary}' - ) A, - (SELECT - ""DuplicateBlock"".""BlockID"", MIN(""DuplicateBlock"".""VolumeID"") AS ""VolumeID"", ""Remotevolume"".""State"" - FROM ""DuplicateBlock"", ""Remotevolume"" - WHERE ""DuplicateBlock"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Remotevolume"".""State"" = '{RemoteVolumeState.Verified}' - GROUP BY ""DuplicateBlock"".""BlockID"", ""Remotevolume"".""State"" - ) B -WHERE ""A"".""ID"" = ""B"".""BlockID""; + var sql = $@" + CREATE TEMPORARY TABLE ""{tablename}"" AS + SELECT + ""A"".""ID"" AS ""BlockID"", + ""A"".""VolumeID"" AS ""SourceVolumeID"", + ""A"".""State"" AS ""SourceVolumeState"", + ""B"".""VolumeID"" AS ""TargetVolumeID"", + ""B"".""State"" AS ""TargetVolumeState"" + FROM + ( + SELECT + ""Block"".""ID"", + ""Block"".""VolumeID"", + ""Remotevolume"".""State"" + FROM + ""Block"", + ""Remotevolume"" + WHERE + ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" + AND ""Remotevolume"".""State"" = '{RemoteVolumeState.Temporary}' + ) A, + ( + SELECT + ""DuplicateBlock"".""BlockID"", + MIN(""DuplicateBlock"".""VolumeID"") AS ""VolumeID"", + ""Remotevolume"".""State"" + FROM + ""DuplicateBlock"", + ""Remotevolume"" + WHERE + ""DuplicateBlock"".""VolumeID"" = ""Remotevolume"".""ID"" + AND ""Remotevolume"".""State"" = '{RemoteVolumeState.Verified}' + GROUP BY + ""DuplicateBlock"".""BlockID"", + ""Remotevolume"".""State"" + ) B + WHERE ""A"".""ID"" = ""B"".""BlockID""; -UPDATE ""Block"" - SET ""VolumeID"" = (SELECT - ""TargetVolumeID"" FROM ""{tablename}"" - WHERE ""Block"".""ID"" = ""{tablename}"".""BlockID"" - ) - WHERE ""Block"".""ID"" IN (SELECT ""BlockID"" FROM ""{tablename}""); + UPDATE ""Block"" + SET ""VolumeID"" = ( + SELECT ""TargetVolumeID"" + FROM ""{tablename}"" + WHERE ""Block"".""ID"" = ""{tablename}"".""BlockID"" + ) + WHERE ""Block"".""ID"" IN ( + SELECT ""BlockID"" + FROM ""{tablename}"" + ); -UPDATE ""DuplicateBlock"" - SET ""VolumeID"" = (SELECT - ""SourceVolumeID"" FROM ""{tablename}"" - WHERE ""DuplicateBlock"".""BlockID"" = ""{tablename}"".""BlockID"" - ) -WHERE - (""DuplicateBlock"".""BlockID"", ""DuplicateBlock"".""VolumeID"") - IN (SELECT ""BlockID"", ""TargetVolumeID"" FROM ""{tablename}""); + UPDATE ""DuplicateBlock"" + SET ""VolumeID"" = ( + SELECT ""SourceVolumeID"" + FROM ""{tablename}"" + WHERE ""DuplicateBlock"".""BlockID"" = ""{tablename}"".""BlockID"" + ) + WHERE ( + ""DuplicateBlock"".""BlockID"", + ""DuplicateBlock"".""VolumeID"" + ) IN ( + SELECT + ""BlockID"", + ""TargetVolumeID"" + FROM ""{tablename}"" + ); -DROP TABLE ""{tablename}""; + DROP TABLE ""{tablename}""; -DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND ""State"" = '{RemoteVolumeState.Temporary}' AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"")); -DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND ""State"" = '{RemoteVolumeState.Temporary}' AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"")); -DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND ""State"" = '{RemoteVolumeState.Temporary}' AND ""ID"" NOT IN (SELECT DISTINCT ""VolumeID"" FROM ""Block""); -"); + DELETE FROM ""IndexBlockLink"" + WHERE ""BlockVolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE + ""Type"" = '{RemoteVolumeType.Blocks}' + AND ""State"" = '{RemoteVolumeState.Temporary}' + AND ""ID"" NOT IN ( + SELECT DISTINCT ""VolumeID"" + FROM ""Block"" + ) + ); + + DELETE FROM ""DuplicateBlock"" + WHERE ""VolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE + ""Type"" = '{RemoteVolumeType.Blocks}' + AND ""State"" = '{RemoteVolumeState.Temporary}' + AND ""ID"" NOT IN ( + SELECT DISTINCT ""VolumeID"" + FROM ""Block"" + ) + ); + + DELETE FROM ""RemoteVolume"" + WHERE + ""Type"" = '{RemoteVolumeType.Blocks}' + AND ""State"" = '{RemoteVolumeState.Temporary}' + AND ""ID"" NOT IN ( + SELECT DISTINCT ""VolumeID"" + FROM ""Block"" + ); + "; // We could delete these, but we don't have to, so we keep them around until the next compact is done // UPDATE ""RemoteVolume"" SET ""State"" = ""{3}"" WHERE ""Type"" = ""{5}"" AND ""ID"" NOT IN (SELECT ""IndexVolumeID"" FROM ""IndexBlockLink""); - var countsql = FormatInvariant($@"SELECT COUNT(*) FROM ""RemoteVolume"" WHERE ""State"" = '{RemoteVolumeState.Temporary}' AND ""Type"" = '{RemoteVolumeType.Blocks}' "); + var countsql = $@" + SELECT COUNT(*) + FROM ""RemoteVolume"" + WHERE + ""State"" = '{RemoteVolumeState.Temporary}' + AND ""Type"" = '{RemoteVolumeType.Blocks}' + "; - using (var cmd = m_connection.CreateCommand()) + await using var cmd = m_connection.CreateCommand(m_rtr); + var cnt = await cmd.ExecuteScalarInt64Async(countsql, token) + .ConfigureAwait(false); + + if (cnt > 0) { - var cnt = cmd.ExecuteScalarInt64(countsql); - if (cnt > 0) + try { - try - { - cmd.ExecuteNonQuery(sql); - var cnt2 = cmd.ExecuteScalarInt64(countsql); - Logging.Log.WriteWarningMessage(LOGTAG, "MissingVolumesDetected", null, "Replaced blocks for {0} missing volumes; there are now {1} missing volumes", cnt, cnt2); - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "MissingVolumesDetected", ex, "Found {0} missing volumes; failed while attempting to replace blocks from existing volumes", cnt); - throw; - } + await cmd.ExecuteNonQueryAsync(sql, token) + .ConfigureAwait(false); + + var cnt2 = await cmd.ExecuteScalarInt64Async(countsql, token) + .ConfigureAwait(false); + + Logging.Log.WriteWarningMessage(LOGTAG, "MissingVolumesDetected", null, "Replaced blocks for {0} missing volumes; there are now {1} missing volumes", cnt, cnt2); + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "MissingVolumesDetected", ex, "Found {0} missing volumes; failed while attempting to replace blocks from existing volumes", cnt); + throw; } } } /// /// Move blocks that are not referenced by any files to DeletedBlock table. - /// /// Needs to be called after the last FindMissingBlocklistHashes, otherwise the tables are not up to date. - public void CleanupDeletedBlocks(IDbTransaction transaction) + /// + /// A cancellation token to observe while waiting for the task to complete. + /// A task that completes when the cleanup operation is finished. + public async Task CleanupDeletedBlocks(CancellationToken token) { // Find out which blocks are deleted and move them into DeletedBlock, so that compact notices these blocks are empty // Deleted blocks do not appear in the BlocksetEntry and not in the BlocklistHash table var tmptablename = "DeletedBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - using (var cmd = m_connection.CreateCommand(tr.Parent)) - { - // 1. Select blocks not used by any file and not as a blocklist into temporary table - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tmptablename}"" - AS SELECT ""Block"".""ID"", ""Block"".""Hash"", ""Block"".""Size"", ""Block"".""VolumeID"" FROM ""Block"" - WHERE ""Block"".""ID"" NOT IN (SELECT ""BlocksetEntry"".""BlockID"" FROM ""BlocksetEntry"") - AND ""Block"".""Hash"" NOT IN (SELECT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"")")); - // 2. Insert blocks into DeletedBlock table - cmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""DeletedBlock"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""{tmptablename}""")); - // 3. Remove blocks from Block table - cmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Block"" WHERE ""ID"" IN (SELECT ""ID"" FROM ""{tmptablename}"")")); - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tmptablename}""")); - tr.Commit(); - } + await using var cmd = m_connection.CreateCommand(m_rtr); + // 1. Select blocks not used by any file and not as a blocklist into temporary table + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{tmptablename}"" AS + SELECT + ""Block"".""ID"", + ""Block"".""Hash"", + ""Block"".""Size"", + ""Block"".""VolumeID"" + FROM ""Block"" + WHERE + ""Block"".""ID"" NOT IN ( + SELECT ""BlocksetEntry"".""BlockID"" + FROM ""BlocksetEntry"" + ) + AND ""Block"".""Hash"" NOT IN ( + SELECT ""BlocklistHash"".""Hash"" + FROM ""BlocklistHash"" + ) + ", token) + .ConfigureAwait(false); + // 2. Insert blocks into DeletedBlock table + await cmd.ExecuteNonQueryAsync($@" + INSERT INTO ""DeletedBlock"" ( + ""Hash"", + ""Size"", + ""VolumeID"" + ) + SELECT + ""Hash"", + ""Size"", + ""VolumeID"" + FROM ""{tmptablename}"" + ", token) + .ConfigureAwait(false); + // 3. Remove blocks from Block table + await cmd.ExecuteNonQueryAsync($@" + DELETE FROM ""Block"" + WHERE ""ID"" IN ( + SELECT ""ID"" + FROM ""{tmptablename}"" + ) + ", token) + .ConfigureAwait(false); + + await cmd + .ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmptablename}""", token) + .ConfigureAwait(false); + + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); } public override void Dispose() { - using (var cmd = m_connection.CreateCommand()) + DisposeAsync().AsTask().Await(); + + } + + public async override ValueTask DisposeAsync() + { + await using (var cmd = m_connection.CreateCommand(m_rtr)) { if (m_tempblocklist != null) try { - cmd.ExecuteNonQuery(FormatInvariant(@$"DROP TABLE IF EXISTS ""{m_tempblocklist}""")); + await cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tempblocklist}""", default) + .ConfigureAwait(false); } catch (Exception ex) { @@ -702,7 +1501,8 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" if (m_tempsmalllist != null) try { - cmd.ExecuteNonQuery(FormatInvariant(@$"DROP TABLE IF EXISTS ""{m_tempsmalllist}""")); + await cmd.ExecuteNonQueryAsync(@$"DROP TABLE IF EXISTS ""{m_tempsmalllist}""", default) + .ConfigureAwait(false); } catch (Exception ex) { @@ -715,7 +1515,7 @@ DELETE FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks}' AND "" } - base.Dispose(); + await base.DisposeAsync().ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index c184ecef5..74ec8b2a1 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -1,1088 +1,2042 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -#nullable enable - -using System; -using System.Data; -using System.Collections.Generic; -using Duplicati.Library.Utility; - -namespace Duplicati.Library.Main.Database -{ - internal class LocalRepairDatabase : LocalDatabase - { - /// - /// The tag used for logging - /// - private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalRepairDatabase)); - - /// - /// Creates a new local repair database - /// - /// The path to the database - /// The page cache size - public LocalRepairDatabase(string path, long pagecachesize) - : base(path, "Repair", true, pagecachesize) - { - } - - /// - /// Creates a new local repair database from an existing local database - /// - /// The existing local database - public LocalRepairDatabase(LocalDatabase db) - : base(db) - { - } - - /// - /// Gets the fileset ID from the remote name - /// - /// The remote name of the fileset - /// An optional transaction - /// >The fileset ID - public (long FilesetId, DateTime Time, bool IsFullBackup) GetFilesetFromRemotename(string filelist, IDbTransaction? transaction) - { - using (var cmd = m_connection.CreateCommand(transaction)) - { - var rd = cmd.SetCommandAndParameters(@"SELECT ""Fileset"".""ID"", ""Fileset"".""Timestamp"", ""Fileset"".""IsFullBackup"" FROM ""Fileset"",""RemoteVolume"" WHERE ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""RemoteVolume"".""Name"" = @Name") - .SetParameterValue("@Name", filelist) - .ExecuteReader(); - - if (!rd.Read()) - throw new Exception($"No such remote file: {filelist}"); - - return (rd.ConvertValueToInt64(0, -1), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime(), rd.GetInt32(2) == BackupType.FULL_BACKUP); - } - } - - /// - /// Moves entries in the FilesetEntry table from previous fileset to current fileset - /// - /// Current fileset ID - /// Source fileset ID - /// The transaction - public void MoveFilesFromFileset(long filesetid, long prevFilesetId, IDbTransaction transaction) - { - if (filesetid <= 0) - throw new ArgumentException("filesetid must be > 0"); - - if (prevFilesetId <= 0) - throw new ArgumentException("prevId must be > 0"); - - using var cmd = m_connection.CreateCommand(transaction); - cmd.SetCommandAndParameters(@"UPDATE ""FilesetEntry"" SET ""FilesetID"" = @CurrentFilesetId WHERE ""FilesetID"" = @PreviousFilesetId ") - .SetParameterValue("@CurrentFilesetId", filesetid) - .SetParameterValue("@PreviousFilesetId", prevFilesetId) - .ExecuteNonQuery(); - } - - /// - /// Gets the list of index files that reference a given block file - /// - /// The block file ID - /// >A list of index file IDs - public IEnumerable GetIndexFilesReferencingBlockFile(long blockfileid, IDbTransaction? transaction) - { - using var cmd = m_connection.CreateCommand(transaction, @"SELECT ""RemoteVolume"".""Name"" FROM ""RemoteVolume"", ""IndexBlockLink"" WHERE ""IndexBlockLink"".""BlockVolumeID"" = @BlockFileId AND ""RemoteVolume"".""ID"" = ""IndexBlockLink"".""IndexVolumeID"" AND ""RemoteVolume"".""Type"" = @Type") - .SetParameterValue("@BlockFileId", blockfileid) - .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()); - - foreach (var rd in cmd.ExecuteReaderEnumerable()) - yield return rd.ConvertValueToString(0) ?? throw new Exception("RemoteVolume name was null"); - } - - /// - /// Gets a list of filesets that are missing files - /// - /// An optional transaction - /// A list of fileset IDs and timestamps - public IEnumerable> GetFilesetsWithMissingFiles(IDbTransaction transaction) - { - using (var cmd = m_connection.CreateCommand(transaction)) - using (var rd = cmd.ExecuteReader(@"SELECT ID, Timestamp FROM Fileset WHERE ID IN (SELECT FilesetID FROM ""FilesetEntry"" WHERE ""FileID"" NOT IN (SELECT ""ID"" FROM ""FileLookup""))")) - while (rd.Read()) - { - yield return new KeyValuePair( - rd.ConvertValueToInt64(0), - ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime() - ); - } - } - - /// - /// Deletes all fileset entries for a given fileset - /// - /// The fileset ID - /// An optional transaction - /// The number of deleted entries - public int DeleteFilesetEntries(long filesetid, IDbTransaction? transaction = null) - { - using (var cmd = m_connection.CreateCommand(transaction)) - return cmd.SetCommandAndParameters(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId") - .SetParameterValue("@FilesetId", filesetid) - .ExecuteNonQuery(); - } - - private class RemoteVolume : IRemoteVolume - { - public string Name { get; private set; } - public string Hash { get; private set; } - public long Size { get; private set; } - - public RemoteVolume(string name, string hash, long size) - { - this.Name = name; - this.Hash = hash; - this.Size = size; - } - } - - public IEnumerable GetBlockVolumesFromIndexName(string indexName) - { - using var cmd = m_connection.CreateCommand(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"" WHERE ""IndexVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = @Name)) ") - .SetParameterValue("@Name", indexName); - - foreach (var rd in cmd.ExecuteReaderEnumerable()) - yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); - } - - /// - /// A single block with source data for repair - /// - /// The block hash - /// The block size - /// The file that contains the block - /// The offset of the block in the file - public sealed record BlockWithSourceData(string Hash, long Size, string File, long Offset); - - /// - /// A single block with metadata source data for repair - /// - /// The block hash - /// The block size - /// The path of the file or directory that contains the block - public sealed record BlockWithMetadataSourceData(string Hash, long Size, string Path); - - /// - /// A single blocklist hash entry - /// - /// The blockset id - /// The hash of the blocklist entry (when done) - /// The total length of the blockset - /// The index of the blocklist hash - /// The index of the block in the blockset - /// The hash of the entry - public sealed record BlocklistHashesEntry( - long BlocksetId, - string BlocklistHash, - long BlocklistHashLength, - long BlocklistHashIndex, - int Index, - string Hash); - - /// - /// Helper interface for the missing block list - /// - public interface IMissingBlockList : IDisposable - { - /// - /// Registers a block as restored - /// - /// The block hash - /// The block size - /// The volume ID of the new target volume - /// The restored blocks - bool SetBlockRestored(string hash, long size, long volumeId); - /// - /// Gets the list of files that contains missing blocks - /// - /// The blocksize setting - /// A list of files with missing blocks - IEnumerable GetSourceFilesWithBlocks(long blocksize); - /// - /// Gets a list for filesystem entries that contain missing blocks in metadata - /// - /// A list of filesystem entries with missing blocks - IEnumerable GetSourceItemsWithMetadataBlocks(); - /// - /// Gets missing blocklist hashes - /// - /// The number of hashes for each block - /// >A list of blocklist hashes - IEnumerable GetBlocklistHashes(long hashesPerBlock); - /// - /// Gets the number of missing blocks - /// - /// >The number of missing blocks - long GetMissingBlockCount(); - /// - /// Gets all the filesets that are affected by missing blocks - /// - /// >A list of filesets - IEnumerable GetFilesetsUsingMissingBlocks(); - /// - /// Gets a list of remote files that may contain missing blocks - /// - /// >A list of remote files - IEnumerable GetMissingBlockSources(); - } - - /// - /// Implementation of the missing block list - /// - private class MissingBlockList : IMissingBlockList - { - /// - /// The connection to the database - /// - private readonly IDbConnection m_connection; - /// - /// The transaction to use - /// - private readonly ReusableTransaction m_transaction; - /// - /// The insert command to use for restoring blocks - /// - private readonly IDbCommand m_insertCommand; - /// - /// The command to copy blocks into the duplicate block table - /// - private readonly IDbCommand m_copyIntoDuplicatedBlocks; - /// - /// The command to assign blocks to a new volume - /// - private readonly IDbCommand m_assignBlocksToNewVolume; - /// - /// The name of the temporary table - /// - private readonly string m_tablename; - /// - /// The name of the volume where blocks are missing - /// - private readonly string m_volumename; - /// - /// The query to use for getting missing blocks - /// - private readonly string m_missingBlocksQuery; - - /// - /// Whether the object has been disposed - /// - private bool m_isDisposed = false; - - /// - /// Creates a new missing block list - /// - /// The name of the volume with missing blocks - /// The connection to the database - /// The transaction to use - public MissingBlockList(string volumename, IDbConnection connection, ReusableTransaction transaction) - { - m_connection = connection; - m_transaction = transaction; - m_volumename = volumename; - var tablename = "MissingBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using (var cmd = m_connection.CreateCommand(m_transaction.Transaction)) - { - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" INTEGER NOT NULL) ")); - m_tablename = tablename; - - var blockCount = cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_tablename}"" (""Hash"", ""Size"", ""Restored"") SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"", 0 AS ""Restored"" FROM ""Block"",""Remotevolume"" WHERE ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Remotevolume"".""Name"" = @Name ")) - .SetParameterValue("@Name", volumename) - .ExecuteNonQuery(); - - if (blockCount == 0) - throw new Exception($"Unexpected empty block volume: {0}"); - - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE UNIQUE INDEX ""{tablename}-Ix"" ON ""{tablename}"" (""Hash"", ""Size"", ""Restored"")")); - } - - m_insertCommand = m_connection.CreateCommand(FormatInvariant($@"UPDATE ""{tablename}"" SET ""Restored"" = @NewRestoredValue WHERE ""Hash"" = @Hash AND ""Size"" = @Size AND ""Restored"" = @PreviousRestoredValue ")); - m_copyIntoDuplicatedBlocks = m_connection.CreateCommand(@"INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") SELECT ""Block"".""ID"", ""Block"".""VolumeID"" FROM ""Block"" WHERE ""Block"".""Hash"" = @Hash AND ""Block"".""Size"" = @Size"); - m_assignBlocksToNewVolume = m_connection.CreateCommand(@"UPDATE ""Block"" SET ""VolumeID"" = @TargetVolumeId WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); - m_missingBlocksQuery = FormatInvariant($@"SELECT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Restored"" = @Restored "); - } - - /// - public bool SetBlockRestored(string hash, long size, long targetVolumeId) - { - var restored = m_insertCommand.SetTransaction(m_transaction.Transaction).SetParameterValue("@NewRestoredValue", 1) - .SetParameterValue("@Hash", hash) - .SetParameterValue("@Size", size) - .SetParameterValue("@PreviousRestoredValue", 0) - .ExecuteNonQuery() == 1; - - if (restored) - { - m_copyIntoDuplicatedBlocks.SetTransaction(m_transaction.Transaction) - .SetParameterValue("@Hash", hash) - .SetParameterValue("@Size", size) - .ExecuteNonQuery(); - - var c = m_assignBlocksToNewVolume.SetTransaction(m_transaction.Transaction) - .SetParameterValue("@TargetVolumeId", targetVolumeId) - .SetParameterValue("@Hash", hash) - .SetParameterValue("@Size", size) - .ExecuteNonQuery(); - - if (c != 1) - throw new Exception($"Unexpected number of updated blocks: {c} != 1"); - } - - return restored; - } - - /// - public IEnumerable GetSourceFilesWithBlocks(long blocksize) - { - using var cmd = m_connection.CreateCommand(m_transaction.Transaction, FormatInvariant($@"SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {blocksize} FROM ""{m_tablename}"", ""Block"", ""BlocksetEntry"", ""File"" WHERE ""File"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" AND ""{m_tablename}"".""Restored"" = @Restored ORDER BY ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"", ""BlocksetEntry"".""Index"" * {blocksize}")) - .SetParameterValue("@Restored", 0); - - foreach (var rd in cmd.ExecuteReaderEnumerable()) - { - var hash = rd.ConvertValueToString(0) ?? throw new Exception("Hash value was null"); - var size = rd.ConvertValueToInt64(1); - var file = rd.ConvertValueToString(2) ?? throw new Exception("File value was null"); - var offset = rd.ConvertValueToInt64(3); - - yield return new BlockWithSourceData(hash, size, file, offset); - } - } - - /// - public IEnumerable GetSourceItemsWithMetadataBlocks() - { - using var cmd = m_connection.CreateCommand(m_transaction.Transaction, FormatInvariant($@"SELECT DISTINCT ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path"" FROM ""{m_tablename}"", ""Block"", ""BlocksetEntry"", ""Metadataset"", ""File"" WHERE ""File"".""MetadataID"" == ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" AND ""{m_tablename}"".""Restored"" = @Restored ORDER BY ""{m_tablename}"".""Hash"", ""{m_tablename}"".""Size"", ""File"".""Path""")) - .SetParameterValue("@Restored", 0); - - foreach (var rd in cmd.ExecuteReaderEnumerable()) - { - var hash = rd.ConvertValueToString(0) ?? throw new Exception("Hash value was null"); - var size = rd.ConvertValueToInt64(1); - var path = rd.ConvertValueToString(2) ?? throw new Exception("File value was null"); - - yield return new BlockWithMetadataSourceData(hash, size, path); - } - } - - /// - public IEnumerable GetBlocklistHashes(long hashesPerBlock) - { - using var cmd = m_connection.CreateCommand(m_transaction.Transaction); - var blocklistTableName = $"BlocklistHashList-{Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray())}"; - try - { - // We need to create a snapshot as we will be updating the m_tablename table during enumeration - cmd.SetCommandAndParameters(FormatInvariant($@" - CREATE TEMPORARY TABLE ""{blocklistTableName}"" AS - SELECT - b.""Hash"" AS ""BlockHash"", - bs.""Id"" AS ""BlocksetId"", - bs.""Length"" AS ""BlocksetLength"", - bse.""Index"" / @HashesPerBlock AS ""BlocklistHashIndex"", - ( - SELECT blh.""Hash"" - FROM ""BlocklistHash"" blh - WHERE blh.""BlocksetID"" = bs.""ID"" - AND blh.""Index"" == bse.""Index"" / @HashesPerBlock - LIMIT 1 - ) AS ""BlocklistHashHash"", - bse.""Index"" AS ""BlocksetEntryIndex"" - FROM - ""BlocksetEntry"" bse - JOIN ""Block"" b ON b.""ID"" = bse.""BlockID"" - JOIN ""Blockset"" bs ON bs.""ID"" = bse.""BlocksetID"" - WHERE - EXISTS ( - SELECT 1 - FROM ""BlocklistHash"" blh - JOIN ""{m_tablename}"" mt ON mt.""Hash"" = blh.""Hash"" - WHERE blh.""BlocksetID"" = bs.""ID"" - AND mt.""Restored"" = @Restored - ) - ")) - .SetParameterValue("@HashesPerBlock", hashesPerBlock) - .SetParameterValue("@Restored", 0) - .ExecuteNonQuery(); - - cmd.SetCommandAndParameters(FormatInvariant($@" - SELECT - ""BlockHash"", - ""BlocksetId"", - ""BlocksetLength"", - ""BlocklistHashIndex"", - ""BlocklistHashHash"", - ""BlocksetEntryIndex"" - FROM ""{blocklistTableName}"" - ORDER BY ""BlocksetId"", ""BlocklistHashIndex"", ""BlocksetEntryIndex""")); - - foreach (var rd in cmd.ExecuteReaderEnumerable()) - { - var hash = rd.ConvertValueToString(0) ?? throw new Exception("Block.Hash is null"); - var blocksetId = rd.ConvertValueToInt64(1); - var length = rd.ConvertValueToInt64(2); - var blocklistHashIndex = rd.ConvertValueToInt64(3); - var blocklistHash = rd.ConvertValueToString(4) ?? throw new Exception("BlocklistHash is null"); - var index = rd.ConvertValueToInt64(5); - - yield return new BlocklistHashesEntry(blocksetId, blocklistHash, length, blocklistHashIndex, (int)index, hash); - } - } - finally - { - try { cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{blocklistTableName}"" ")); } - catch { } - } - } - - /// - public long GetMissingBlockCount() - { - using var cmd = m_connection.CreateCommand(m_transaction.Transaction, FormatInvariant($@"SELECT COUNT(*) FROM ({m_missingBlocksQuery})")) - .SetParameterValue("@Restored", 0); - - return cmd.ExecuteScalarInt64(0); - } - - /// - /// Gets the list of missing blocks - /// - /// >A list of missing blocks - public IEnumerable<(string Hash, long Size)> GetMissingBlocks() - { - using var cmd = m_connection.CreateCommand(m_transaction.Transaction, m_missingBlocksQuery) - .SetParameterValue("@Restored", 0); - - foreach (var rd in cmd.ExecuteReaderEnumerable()) - yield return (rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1)); - } - - /// - public long MoveBlocksToNewVolume(long targetVolumeId, long sourceVolumeId, IDbTransaction? transaction = null) - { - if (targetVolumeId <= 0) - throw new ArgumentOutOfRangeException(nameof(targetVolumeId), "Target volume ID must be greater than 0"); - - // Move the source blocks into the DuplicateBlock table - using var cmd = m_connection.CreateCommand(transaction, FormatInvariant($@" - INSERT OR IGNORE INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") - SELECT b.""ID"", b.""VolumeID"" - FROM ""Block"" b - WHERE b.""VolumeID"" = @SourceVolumeId - AND (b.""Hash"", b.""Size"") IN ( - SELECT ""Hash"", ""Size"" - FROM ""{m_tablename}"" - WHERE ""Restored"" = @Restored - ) - ")) - .SetParameterValue("@SourceVolumeId", sourceVolumeId) - .SetParameterValue("@Restored", 1); - - var moved = cmd.ExecuteNonQuery(); - - // Then update the blocks table to point to the new volume - cmd.SetCommandAndParameters(FormatInvariant($@" - UPDATE ""Block"" - SET ""VolumeID"" = @TargetVolumeId - WHERE ""VolumeID"" = @SourceVolumeId - AND (""Hash"", ""Size"") IN ( - SELECT ""Hash"", ""Size"" - FROM ""{m_tablename}"" - WHERE ""Restored"" = @Restored - ) - ")) - .SetParameterValue("@TargetVolumeId", targetVolumeId) - .SetParameterValue("@SourceVolumeId", sourceVolumeId) - .SetParameterValue("@Restored", 1); - - var updated = cmd.ExecuteNonQuery(); - if (updated != moved) - throw new Exception($"Unexpected number of updated blocks: {updated} != {moved}"); - - return updated; - } - - public IEnumerable GetFilesetsUsingMissingBlocks() - { - var blocks = FormatInvariant($@"SELECT DISTINCT ""FileLookup"".""ID"" AS ID FROM ""{m_tablename}"", ""Block"", ""Blockset"", ""BlocksetEntry"", ""FileLookup"" WHERE ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" "); - var blocklists = FormatInvariant($@"SELECT DISTINCT ""FileLookup"".""ID"" AS ID FROM ""{m_tablename}"", ""Block"", ""Blockset"", ""BlocklistHash"", ""FileLookup"" WHERE ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""BlocklistHash"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" "); - var cmdtxt = FormatInvariant($@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""FilesetEntry"", ""Fileset"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""RemoteVolume"".""Type"" = @Type AND ""FilesetEntry"".""FileID"" IN (SELECT DISTINCT ""ID"" FROM ( {blocks} UNION {blocklists} ))"); - - using var cmd = m_connection.CreateCommand(m_transaction.Transaction, cmdtxt) - .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); - - foreach (var rd in cmd.ExecuteReaderEnumerable()) - yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); - } - - public IEnumerable GetMissingBlockSources() - { - using var cmd = m_connection.CreateCommand(m_transaction.Transaction, FormatInvariant($@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""Block"", ""{m_tablename}"" WHERE ""{m_tablename}"".""Restored"" = @Restored AND ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" AND ""Block"".""VolumeID"" = ""RemoteVolume"".""ID"" AND ""Remotevolume"".""Name"" != @Name ")) - .SetParameterValue("@Restored", 0) - .SetParameterValue("@Name", m_volumename); - - foreach (var rd in cmd.ExecuteReaderEnumerable()) - yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); - } - - public void Dispose() - { - if (m_isDisposed) - return; - - m_isDisposed = true; - try - { - if (m_tablename != null) - using (var cmd = m_connection.CreateCommand(m_transaction.Transaction)) - cmd.SetTransaction(m_transaction.Transaction).ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tablename}"" ")); - } - catch { } - - try { m_insertCommand?.Dispose(); } - catch { } - } - } - - public IMissingBlockList CreateBlockList(string volumename, ReusableTransaction transaction) - { - return new MissingBlockList(volumename, m_connection, transaction); - } - - public void FixDuplicateMetahash() - { - using (var tr = m_connection.BeginTransactionSafe()) - using (var cmd = m_connection.CreateCommand(tr)) - { - var sql_count = - @"SELECT COUNT(*) FROM (" + - @" SELECT DISTINCT c1 FROM (" + - @"SELECT COUNT(*) AS ""C1"" FROM (SELECT DISTINCT ""BlocksetID"" FROM ""Metadataset"") UNION SELECT COUNT(*) AS ""C1"" FROM ""Metadataset"" " + - @")" + - @")"; - - var x = cmd.ExecuteScalarInt64(sql_count, 0); - if (x > 1) - { - Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateMetadataHashes", "Found duplicate metadatahashes, repairing"); - - var tablename = "TmpFile-" + Guid.NewGuid().ToString("N"); - - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" AS SELECT * FROM ""File""")); - - var sql = @"SELECT ""A"".""ID"", ""B"".""BlocksetID"" FROM (SELECT MIN(""ID"") AS ""ID"", COUNT(""ID"") AS ""Duplicates"" FROM ""Metadataset"" GROUP BY ""BlocksetID"") ""A"", ""Metadataset"" ""B"" WHERE ""A"".""Duplicates"" > 1 AND ""A"".""ID"" = ""B"".""ID"""; - - using (var c2 = m_connection.CreateCommand(tr)) - { - c2.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{tablename}"" SET ""MetadataID"" = @MetadataId WHERE ""MetadataID"" IN (SELECT ""ID"" FROM ""Metadataset"" WHERE ""BlocksetID"" = @BlocksetId)") - + @"; DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" = @BlocksetId AND ""ID"" != @MetadataId"); - using (var rd = cmd.ExecuteReader(sql)) - while (rd.Read()) - c2.SetParameterValue("@MetadataId", rd.GetValue(0)) - .SetParameterValue("@BlocksetId", rd.GetValue(1)) - .ExecuteNonQuery(); - } - - sql = FormatInvariant($@"SELECT ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""Entries"" FROM ( - SELECT MIN(""ID"") AS ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""{tablename}"" GROUP BY ""Path"", ""BlocksetID"", ""MetadataID"") - WHERE ""Entries"" > 1 ORDER BY ""ID"""); - - using (var c2 = m_connection.CreateCommand(tr)) - { - c2.SetCommandAndParameters(FormatInvariant($@"UPDATE ""FilesetEntry"" SET ""FileID"" = @FileId WHERE ""FileID"" IN (SELECT ""ID"" FROM ""{tablename}"" WHERE ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId)") - + FormatInvariant($@"; DELETE FROM ""{tablename}"" WHERE ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId AND ""ID"" != @FileId")); - - foreach (var rd in cmd.ExecuteReaderEnumerable(sql)) - c2.SetParameterValue("@FileId", rd.GetValue(0)) - .SetParameterValue("@Path", rd.GetValue(1)) - .SetParameterValue("@BlocksetId", rd.GetValue(2)) - .SetParameterValue("@MetadataId", rd.GetValue(3)) - .ExecuteNonQuery(); - } - - cmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT ""ID"" FROM ""{tablename}"") ")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{tablename}-Ix"" ON ""{tablename}"" (""ID"", ""MetadataID"")")); - cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""FileLookup"" SET ""MetadataID"" = (SELECT ""MetadataID"" FROM ""{tablename}"" A WHERE ""A"".""ID"" = ""FileLookup"".""ID"") ")); - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE ""{tablename}"" ")); - - x = cmd.ExecuteScalarInt64(sql_count, 0); - if (x > 1) - throw new Interface.UserInformationException("Repair failed, there are still duplicate metadatahashes!", "DuplicateHashesRepairFailed"); - - Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateMetadataHashesFixed", "Duplicate metadatahashes repaired succesfully"); - tr.Commit(); - } - } - } - - public void FixDuplicateFileentries() - { - using (var tr = m_connection.BeginTransactionSafe()) - using (var cmd = m_connection.CreateCommand(tr)) - { - var sql_count = @"SELECT COUNT(*) FROM (SELECT ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Duplicates"" FROM ""FileLookup"" GROUP BY ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") WHERE ""Duplicates"" > 1"; - - var x = cmd.ExecuteScalarInt64(sql_count, 0); - if (x > 0) - { - Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateFileEntries", "Found duplicate file entries, repairing"); - - var sql = @"SELECT ""ID"", ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""Entries"" FROM ( - SELECT MIN(""ID"") AS ""ID"", ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""FileLookup"" GROUP BY ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") - WHERE ""Entries"" > 1 ORDER BY ""ID"""; - - using (var c2 = m_connection.CreateCommand(tr)) - { - c2.SetCommandAndParameters(@"UPDATE ""FilesetEntry"" SET ""FileID"" = @FileId WHERE ""FileID"" IN (SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetatadataId)" - + @"; DELETE FROM ""FileLookup"" WHERE ""PrefixID"" = @PrefixId AND ""Path"" = @Path AND ""BlocksetID"" = @BlocksetId AND ""MetadataID"" = @MetadataId AND ""ID"" != @FileId"); - - foreach (var rd in cmd.SetCommandAndParameters(sql).ExecuteReaderEnumerable()) - c2.SetParameterValue("@FileId", rd.GetValue(0)) - .SetParameterValue("@PrefixId", rd.GetValue(1)) - .SetParameterValue("@Path", rd.GetValue(2)) - .SetParameterValue("@BlocksetId", rd.GetValue(3)) - .SetParameterValue("@MetadataId", rd.GetValue(4)) - .ExecuteNonQuery(); - } - - x = cmd.ExecuteScalarInt64(sql_count, 0); - if (x > 1) - throw new Interface.UserInformationException("Repair failed, there are still duplicate file entries!", "DuplicateFilesRepairFailed"); - - Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateFileEntriesFixed", "Duplicate file entries repaired succesfully"); - tr.Commit(); - } - } - - } - - public void FixMissingBlocklistHashes(string blockhashalgorithm, long blocksize) - { - var blocklistbuffer = new byte[blocksize]; - - using (var tr = m_connection.BeginTransactionSafe()) - using (var cmd = m_connection.CreateCommand(tr)) - using (var blockhasher = HashFactory.CreateHasher(blockhashalgorithm)) - { - var hashsize = blockhasher.HashSize / 8; - - var sql = FormatInvariant($@"SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {blocksize / hashsize} - 1) / {blocksize / hashsize}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"""); - var countsql = @"SELECT COUNT(*) FROM (" + sql + @")"; - - var itemswithnoblocklisthash = cmd.ExecuteScalarInt64(countsql, 0); - if (itemswithnoblocklisthash != 0) - { - Logging.Log.WriteInformationMessage(LOGTAG, "MissingBlocklistHashes", "Found {0} missing blocklisthash entries, repairing", itemswithnoblocklisthash); - using (var c2 = m_connection.CreateCommand(tr)) - using (var c3 = m_connection.CreateCommand(tr)) - using (var c4 = m_connection.CreateCommand(tr)) - using (var c5 = m_connection.CreateCommand(tr)) - using (var c6 = m_connection.CreateCommand(tr)) - { - c3.SetCommandAndParameters(@"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (@BlocksetId, @Index, @Hash) "); - c4.SetCommandAndParameters(@"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size"); - c5.SetCommandAndParameters(@"SELECT ""ID"" FROM ""DeletedBlock"" WHERE ""Hash"" = @Hash AND ""Size"" = @Size AND ""VolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND (""State"" IN (@State1, @State2)))"); - c6.SetCommandAndParameters(@"INSERT INTO ""Block"" (""Hash"", ""Size"", ""VolumeID"") SELECT ""Hash"", ""Size"", ""VolumeID"" FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId LIMIT 1; DELETE FROM ""DeletedBlock"" WHERE ""ID"" = @DeletedBlockId;"); - - foreach (var e in cmd.ExecuteReaderEnumerable(sql)) - { - var blocksetid = e.ConvertValueToInt64(0); - var ix = 0L; - int blocklistoffset = 0; - - c2.SetCommandAndParameters(@"DELETE FROM ""BlocklistHash"" WHERE ""BlocksetID"" = @BlocksetId") - .SetParameterValue("@BlocksetId", blocksetid) - .ExecuteNonQuery(); - - c2.SetCommandAndParameters(@"SELECT ""A"".""Hash"" FROM ""Block"" ""A"", ""BlocksetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""BlockID"" AND ""B"".""BlocksetID"" = @BlocksetId ORDER BY ""B"".""Index""") - .SetParameterValue("@BlocksetId", blocksetid); - - foreach (var h in c2.ExecuteReaderEnumerable()) - { - var tmp = Convert.FromBase64String(h.ConvertValueToString(0) ?? throw new Exception("Hash value was null")); - if (blocklistbuffer.Length - blocklistoffset < tmp.Length) - { - var blkey = Convert.ToBase64String(blockhasher.ComputeHash(blocklistbuffer, 0, blocklistoffset)); - - // Check if the block exists in "blocks" - var existingBlockId = c4.SetParameterValue("@Hash", blkey) - .SetParameterValue("@Size", blocklistoffset) - .ExecuteScalarInt64(-1); - - if (existingBlockId <= 0) - { - var deletedBlockId = c5.SetParameterValue("@Hash", blkey) - .SetParameterValue("@Size", blocklistoffset) - .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) - .SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()) - .SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()) - .ExecuteScalarInt64(-1); - - if (deletedBlockId <= 0) - throw new Exception($"Missing block for blocklisthash: {blkey}"); - else - { - var rc = c6.SetParameterValue("@DeletedBlockId", deletedBlockId) - .ExecuteNonQuery(); - - if (rc != 2) - throw new Exception($"Unexpected update count: {rc}"); - } - } - - // Add to table - c3.SetParameterValue("@BlocksetId", blocksetid) - .SetParameterValue("@Index", ix) - .SetParameterValue("@Hash", blkey) - .ExecuteNonQuery(); - - ix++; - blocklistoffset = 0; - } - - Array.Copy(tmp, 0, blocklistbuffer, blocklistoffset, tmp.Length); - blocklistoffset += tmp.Length; - - } - - if (blocklistoffset != 0) - { - var blkeyfinal = Convert.ToBase64String(blockhasher.ComputeHash(blocklistbuffer, 0, blocklistoffset)); - - // Ensure that the block exists in "blocks" - var existingBlockId = c4.SetParameterValue("@Hash", blkeyfinal) - .SetParameterValue("@Size", blocklistoffset) - .ExecuteScalarInt64(-1); - - if (existingBlockId <= 0) - { - var deletedBlockId = c5.SetParameterValue("@Hash", blkeyfinal) - .SetParameterValue("@Size", blocklistoffset) - .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) - .SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()) - .SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()) - .ExecuteScalarInt64(-1); - - if (deletedBlockId == 0) - throw new Exception($"Missing block for blocklisthash: {blkeyfinal}"); - else - { - var rc = c6.SetParameterValue("@DeletedBlockId", deletedBlockId) - .ExecuteNonQuery(); - if (rc != 2) - throw new Exception($"Unexpected update count: {rc}"); - } - } - - // Add to table - c3.SetParameterValue("@BlocksetId", blocksetid) - .SetParameterValue("@Index", ix) - .SetParameterValue("@Hash", blkeyfinal) - .ExecuteNonQuery(); - } - } - } - - - itemswithnoblocklisthash = cmd.ExecuteScalarInt64(countsql, 0); - if (itemswithnoblocklisthash != 0) - throw new Interface.UserInformationException($"Failed to repair, after repair {itemswithnoblocklisthash} blocklisthashes were missing", "MissingBlocklistHashesRepairFailed"); - - Logging.Log.WriteInformationMessage(LOGTAG, "MissingBlocklisthashesRepaired", "Missing blocklisthashes repaired succesfully"); - tr.Commit(); - } - } - - } - - public void FixDuplicateBlocklistHashes(long blocksize, long hashsize) - { - using (var tr = m_connection.BeginTransactionSafe()) - using (var cmd = m_connection.CreateCommand(tr)) - { - var dup_sql = @"SELECT * FROM (SELECT ""BlocksetID"", ""Index"", COUNT(*) AS ""EC"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"", ""Index"") WHERE ""EC"" > 1"; - - var sql_count = @"SELECT COUNT(*) FROM (" + dup_sql + ")"; - - var x = cmd.ExecuteScalarInt64(sql_count, 0); - if (x > 0) - { - Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateBlocklistHashes", "Found duplicate blocklisthash entries, repairing"); - - var unique_count = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""BlocksetID"", ""Index"" FROM ""BlocklistHash"")", 0); - - using (var c2 = m_connection.CreateCommand(tr)) - { - c2.SetCommandAndParameters(@"DELETE FROM ""BlocklistHash"" WHERE rowid IN (SELECT rowid FROM ""BlocklistHash"" WHERE ""BlocksetID"" = @BlocksetId AND ""Index"" = @Index LIMIT @Limit)"); - foreach (var rd in cmd.ExecuteReaderEnumerable(dup_sql)) - { - var expected = rd.GetInt32(2) - 1; - var actual = c2.SetParameterValue("@BlocksetId", rd.GetValue(0)) - .SetParameterValue("@Index", rd.GetValue(1)) - .SetParameterValue("@Limit", expected) - .ExecuteNonQuery(); - - if (actual != expected) - throw new Exception($"Unexpected number of results after fix, got: {actual}, expected: {expected}"); - } - } - - x = cmd.ExecuteScalarInt64(sql_count); - if (x > 1) - throw new Exception("Repair failed, there are still duplicate file entries!"); - - var real_count = cmd.ExecuteScalarInt64(@"SELECT Count(*) FROM ""BlocklistHash""", 0); - - if (real_count != unique_count) - throw new Interface.UserInformationException($"Failed to repair, result should have been {unique_count} blocklist hashes, but result was {real_count} blocklist hashes", "DuplicateBlocklistHashesRepairFailed"); - - try - { - VerifyConsistency(blocksize, hashsize, true, tr); - } - catch (Exception ex) - { - throw new Interface.UserInformationException("Repaired blocklisthashes, but the database was broken afterwards, rolled back changes", "DuplicateBlocklistHashesRepairFailed", ex); - } - - Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateBlocklistHashesRepaired", "Duplicate blocklisthashes repaired succesfully"); - tr.Commit(); - } - } - } - - public void CheckAllBlocksAreInVolume(string filename, IEnumerable> blocks, IDbTransaction transaction) - { - using var cmd = m_connection.CreateCommand(transaction); - var tablename = "ProbeBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - try - { - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" (""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)")); - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{tablename}"" (""Hash"", ""Size"") VALUES (@Hash, @Size)")); - - foreach (var kp in blocks) - { - cmd.SetParameterValue("@Hash", kp.Key); - cmd.SetParameterValue("@Size", kp.Value); - cmd.ExecuteNonQuery(); - } - - var id = cmd.SetCommandAndParameters(@"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = @Name") - .SetParameterValue("@Name", filename) - .ExecuteScalarInt64(-1); - - var aliens = cmd.SetCommandAndParameters(FormatInvariant($@"SELECT COUNT(*) FROM (SELECT ""A"".""VolumeID"" FROM ""{tablename}"" B LEFT OUTER JOIN ""Block"" A ON ""A"".""Hash"" = ""B"".""Hash"" AND ""A"".""Size"" = ""B"".""Size"") WHERE ""VolumeID"" != @VolumeId ")) - .SetParameterValue("@VolumeId", id) - .ExecuteScalarInt64(0); - - if (aliens != 0) - throw new Exception($"Not all blocks were found in {filename}"); - } - finally - { - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tablename}"" ")); - } - } - - public void CheckBlocklistCorrect(string hash, long length, IEnumerable blocklist, long blocksize, long blockhashlength, IDbTransaction? transaction) - { - using (var cmd = m_connection.CreateCommand(transaction)) - { - var query = FormatInvariant($@" -SELECT - ""C"".""Hash"", - ""C"".""Size"" -FROM - ""BlocksetEntry"" A, - ( - SELECT - ""Y"".""BlocksetID"", - ""Y"".""Hash"" AS ""BlocklistHash"", - ""Y"".""Index"" AS ""BlocklistHashIndex"", - ""Z"".""Size"" AS ""BlocklistSize"", - ""Z"".""ID"" AS ""BlocklistHashBlockID"" - FROM - ""BlocklistHash"" Y, - ""Block"" Z - WHERE - ""Y"".""Hash"" = ""Z"".""Hash"" AND ""Y"".""Hash"" = @Hash AND ""Z"".""Size"" = @Size - LIMIT 1 - ) B, - ""Block"" C -WHERE - ""A"".""BlocksetID"" = ""B"".""BlocksetID"" - AND - ""A"".""BlockID"" = ""C"".""ID"" - AND - ""A"".""Index"" >= ""B"".""BlocklistHashIndex"" * ({blocksize} / {blockhashlength}) - AND - ""A"".""Index"" < (""B"".""BlocklistHashIndex"" + 1) * ({blocksize} / {blockhashlength}) -ORDER BY - ""A"".""Index"" - -"); - - using (var en = blocklist.GetEnumerator()) - { - cmd.SetCommandAndParameters(query) - .SetParameterValue("@Hash", hash) - .SetParameterValue("@Size", length); - foreach (var r in cmd.ExecuteReaderEnumerable()) - { - if (!en.MoveNext()) - throw new Exception($"Too few entries in source blocklist with hash {hash}"); - if (en.Current != r.ConvertValueToString(0)) - throw new Exception($"Mismatch in blocklist with hash {hash}"); - } - - if (en.MoveNext()) - throw new Exception($"Too many source blocklist entries in {hash}"); - } - } - } - - public IEnumerable MissingLocalFilesets() - { - using var cmd = m_connection.CreateCommand(@"SELECT ""Name"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" NOT IN (@States) AND ""ID"" NOT IN (SELECT ""VolumeID"" FROM ""Fileset"")") - .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .ExpandInClauseParameter("@States", [RemoteVolumeState.Deleting.ToString(), RemoteVolumeState.Deleted.ToString()]); - - foreach (var rd in cmd.ExecuteReaderEnumerable()) - yield return rd.ConvertValueToString(0) ?? ""; - } - - public IEnumerable<(long FilesetID, DateTime Timestamp, bool IsFull)> MissingRemoteFilesets() - { - using var cmd = m_connection.CreateCommand(@"SELECT ""ID"", ""Timestamp"", ""IsFullBackup"" FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" NOT IN (@States))") - .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) - .ExpandInClauseParameter("@States", [RemoteVolumeState.Deleting.ToString(), RemoteVolumeState.Deleted.ToString()]); - - foreach (var rd in cmd.ExecuteReaderEnumerable()) - yield return (rd.ConvertValueToInt64(0), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)), rd.ConvertValueToInt64(2) == BackupType.FULL_BACKUP); - } - - public IEnumerable EmptyIndexFiles() - { - using var cmd = m_connection.CreateCommand(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = @Type AND ""State"" IN (@States) AND ""ID"" NOT IN (SELECT ""IndexVolumeId"" FROM ""IndexBlockLink"")") - .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()) - .ExpandInClauseParameter("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); - - foreach (var rd in cmd.ExecuteReaderEnumerable()) - yield return new RemoteVolume(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToString(1) ?? "", rd.ConvertValueToInt64(2)); - } - - public void FixEmptyMetadatasets(Options options) - { - using var tr = m_connection.BeginTransactionSafe(); - using var cmd = m_connection.CreateCommand(tr, - @"SELECT COUNT(*) - FROM Metadataset - JOIN Blockset ON Metadataset.BlocksetID = Blockset.ID - WHERE Blockset.Length = 0"); - var emptyMetaCount = cmd.ExecuteScalarInt64(0); - if (emptyMetaCount <= 0) - return; - - Logging.Log.WriteInformationMessage(LOGTAG, "ZeroLengthMetadata", "Found {0} zero-length metadata entries", emptyMetaCount); - - // Create replacement metadata - var emptyMeta = Utility.WrapMetadata(new Dictionary(), options); - var emptyBlocksetId = GetEmptyMetadataBlocksetId(Array.Empty(), emptyMeta.FileHash, emptyMeta.Blob.Length, tr); - if (emptyBlocksetId < 0) - throw new Interface.UserInformationException( - "Failed to locate an empty metadata blockset to replace missing metadata. Set the option --disable-replace-missing-metadata=true to ignore this and drop files with missing metadata.", - "FailedToLocateEmptyMetadataBlockset"); - - // Step 1: Create temp table with Metadataset IDs referencing empty blocksets (excluding the one to keep) - var tablename = "FixMetadatasets-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - - try - { - cmd.SetCommandAndParameters( - @$"CREATE TEMP TABLE ""{tablename}"" AS - SELECT m.ID AS MetadataID, m.BlocksetID - FROM Metadataset m - JOIN Blockset b ON m.BlocksetID = b.ID - WHERE b.Length = 0 - AND m.BlocksetID != @KeepBlockset") - .SetParameterValue("@KeepBlockset", emptyBlocksetId) - .ExecuteNonQuery(); - - // Step 2: Update FileLookup to use a valid metadata ID - cmd.SetCommandAndParameters( - @$"UPDATE FileLookup - SET MetadataID = ( - SELECT ID FROM Metadataset - WHERE BlocksetID = @KeepBlockset - LIMIT 1 - ) - WHERE MetadataID IN (SELECT MetadataID FROM ""{tablename}"")") - .SetParameterValue("@KeepBlockset", emptyBlocksetId) - .ExecuteNonQuery(); - - // Step 3: Delete obsolete Metadataset entries - cmd.SetCommandAndParameters( - @$"DELETE FROM Metadataset - WHERE ID IN (SELECT MetadataID FROM ""{tablename}"")") - .ExecuteNonQuery(); - - // Step 4: Delete orphaned blocksets (affected only) - cmd.SetCommandAndParameters( - @$"DELETE FROM Blockset - WHERE ID IN (SELECT BlocksetID FROM ""{tablename}"") - AND NOT EXISTS (SELECT 1 FROM Metadataset WHERE BlocksetID = Blockset.ID) - AND NOT EXISTS (SELECT 1 FROM File WHERE BlocksetID = Blockset.ID)") - .ExecuteNonQuery(); - - // Step 5: Confirm all broken metadata entries are resolved - cmd.SetCommandAndParameters( - @"SELECT COUNT(*) - FROM Metadataset - JOIN Blockset ON Metadataset.BlocksetID = Blockset.ID - WHERE Blockset.Length = 0"); - - var remaining = cmd.ExecuteScalarInt64(0); - if (remaining > 0) - throw new Interface.UserInformationException( - "Some zero-length metadata entries could not be repaired.", - "MetadataRepairFailed"); - - Logging.Log.WriteInformationMessage(LOGTAG, "ZeroLengthMetadataRepaired", "Zero length metadata entries repaired successfully"); - tr.Commit(); - } - finally - { - try - { - cmd.SetCommandAndParameters(FormatInvariant($@"DROP TABLE IF EXISTS ""{tablename}"" ")) - .ExecuteNonQuery(); - } - catch (Exception ex) - { - Logging.Log.WriteVerboseMessage(LOGTAG, "ErrorDroppingTempTable", ex, "Failed to drop temporary table {0}: {1}", tablename, ex.Message); - } - } - } - } -} - +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.Utility; +using Microsoft.Data.Sqlite; + +namespace Duplicati.Library.Main.Database +{ + /// + /// Represents a specialized local database used for repair operations in Duplicati. + /// Provides methods for creating and managing repair databases, handling missing or duplicate blocks, + /// repairing metadata and file entries, and verifying consistency of backup data. + /// + /// + /// This class extends and provides additional functionality + /// for repairing and maintaining the integrity of backup databases, including block and fileset management, + /// duplicate detection and correction, and blocklist hash repairs. + /// + internal class LocalRepairDatabase : LocalDatabase + { + /// + /// The tag used for logging. + /// + private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalRepairDatabase)); + + /// + /// Creates a new local repair database. + /// + /// The path to the database. + /// An optional existing database instance to use. + /// A cancellation token to cancel the operation. + /// A task that when awaited returns a new instance of . + public static async Task CreateRepairDatabase(string path, LocalRepairDatabase? dbnew, CancellationToken token) + { + dbnew ??= new LocalRepairDatabase(); + + dbnew = (LocalRepairDatabase) + await CreateLocalDatabaseAsync(path, "Repair", true, dbnew, token) + .ConfigureAwait(false); + + return dbnew; + } + + /// + /// Creates a new local repair database from an existing local database. + /// + /// The parent local database to use. + /// An optional existing database instance to use. + /// A cancellation token to cancel the operation. + /// A task that when awaited returns a new instance of . + public static async Task CreateAsync(LocalDatabase dbparent, LocalRepairDatabase? dbnew, CancellationToken token) + { + dbnew ??= new LocalRepairDatabase(); + + return (LocalRepairDatabase) + await CreateLocalDatabaseAsync(dbparent, dbnew, token) + .ConfigureAwait(false); + } + + /// + /// Gets the fileset ID from the remote name. + /// + /// The remote name of the fileset. + /// A cancellation token to cancel the operation. + /// A task that when awaited returns a tuple containing the fileset ID, timestamp, and whether it is a full backup. + /// Thrown if the remote file does not exist. + public async Task<(long FilesetId, DateTime Time, bool IsFullBackup)> GetFilesetFromRemotename(string filelist, CancellationToken token) + { + await using var cmd = m_connection.CreateCommand(@" + SELECT + ""Fileset"".""ID"", + ""Fileset"".""Timestamp"", + ""Fileset"".""IsFullBackup"" + FROM + ""Fileset"", + ""RemoteVolume"" + WHERE + ""Fileset"".""VolumeID"" = ""RemoteVolume"".""ID"" + AND ""RemoteVolume"".""Name"" = @Name + ") + .SetTransaction(m_rtr) + .SetParameterValue("@Name", filelist); + + var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + + if (!await rd.ReadAsync(token).ConfigureAwait(false)) + throw new Exception($"No such remote file: {filelist}"); + + return ( + rd.ConvertValueToInt64(0, -1), + ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime(), + rd.GetInt32(2) == BackupType.FULL_BACKUP + ); + } + + /// + /// Moves entries in the FilesetEntry table from previous fileset to current fileset. + /// + /// Current fileset ID. + /// Source fileset ID. + /// A cancellation token to cancel the operation. + /// A task that represents the asynchronous operation. + /// Thrown if either fileset ID is less than or equal to zero. + public async Task MoveFilesFromFileset(long filesetid, long prevFilesetId, CancellationToken token) + { + if (filesetid <= 0) + throw new ArgumentException("filesetid must be > 0"); + + if (prevFilesetId <= 0) + throw new ArgumentException("prevId must be > 0"); + + await using var cmd = m_connection.CreateCommand(@" + UPDATE ""FilesetEntry"" + SET ""FilesetID"" = @CurrentFilesetId + WHERE ""FilesetID"" = @PreviousFilesetId + ") + .SetTransaction(m_rtr) + .SetParameterValue("@CurrentFilesetId", filesetid) + .SetParameterValue("@PreviousFilesetId", prevFilesetId); + + await cmd.ExecuteNonQueryAsync(token).ConfigureAwait(false); + } + + /// + /// Gets the list of index files that reference a given block file. + /// + /// The block file ID. + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of index file names that reference the specified block file. + public async IAsyncEnumerable GetIndexFilesReferencingBlockFile(long blockfileid, [EnumeratorCancellation] CancellationToken token) + { + await using var cmd = m_connection.CreateCommand(@" + SELECT ""RemoteVolume"".""Name"" + FROM + ""RemoteVolume"", + ""IndexBlockLink"" + WHERE + ""IndexBlockLink"".""BlockVolumeID"" = @BlockFileId + AND ""RemoteVolume"".""ID"" = ""IndexBlockLink"".""IndexVolumeID"" + AND ""RemoteVolume"".""Type"" = @Type + ") + .SetTransaction(m_rtr) + .SetParameterValue("@BlockFileId", blockfileid) + .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) + yield return rd.ConvertValueToString(0) ?? throw new Exception("RemoteVolume name was null"); + } + + /// + /// Gets a list of filesets that are missing files. + /// + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of key-value pairs where the key is the fileset ID and the value is the timestamp of the fileset. + public async IAsyncEnumerable> GetFilesetsWithMissingFiles([EnumeratorCancellation] CancellationToken token) + { + await using var cmd = m_connection.CreateCommand(@" + SELECT + ""ID"", + ""Timestamp"" + FROM ""Fileset"" + WHERE ""ID"" IN ( + SELECT ""FilesetID"" + FROM ""FilesetEntry"" + WHERE ""FileID"" NOT IN ( + SELECT ""ID"" + FROM ""FileLookup"" + ) + ) + ") + .SetTransaction(m_rtr); + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + { + yield return new KeyValuePair( + rd.ConvertValueToInt64(0), + ParseFromEpochSeconds(rd.ConvertValueToInt64(1)).ToLocalTime() + ); + } + } + + /// + /// Deletes all fileset entries for a given fileset. + /// + /// The fileset ID. + /// A cancellation token to cancel the operation. + /// A task that when awaited returns the number of deleted entries. + public async Task DeleteFilesetEntries(long filesetid, CancellationToken token) + { + await using var cmd = m_connection.CreateCommand(@" + DELETE FROM ""FilesetEntry"" + WHERE ""FilesetID"" = @FilesetId + ") + .SetTransaction(m_rtr) + .SetParameterValue("@FilesetId", filesetid); + + return await cmd.ExecuteNonQueryAsync(token).ConfigureAwait(false); + } + + /// + /// Represents a remote volume with its name, hash, and size. + /// + private class RemoteVolume : IRemoteVolume + { + /// + /// The name of the remote volume. + /// + public string Name { get; private set; } + /// + /// The hash of the remote volume. + /// + public string Hash { get; private set; } + /// + /// The size of the remote volume in bytes. + /// + public long Size { get; private set; } + + /// + /// Initializes a new instance of the class. + /// + /// The name of the remote volume. + /// The hash of the remote volume. + /// The size of the remote volume in bytes. + public RemoteVolume(string name, string hash, long size) + { + this.Name = name; + this.Hash = hash; + this.Size = size; + } + } + + /// + /// Gets a list of block volumes that are associated with a given index name. + /// + /// The name of the index volume. + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of representing the block volumes. + public async IAsyncEnumerable GetBlockVolumesFromIndexName(string indexName, [EnumeratorCancellation] CancellationToken token) + { + await using var cmd = m_connection.CreateCommand(@" + SELECT + ""Name"", + ""Hash"", + ""Size"" + FROM ""RemoteVolume"" + WHERE ""ID"" IN ( + SELECT ""BlockVolumeID"" + FROM ""IndexBlockLink"" + WHERE ""IndexVolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" = @Name + ) + ) + ") + .SetTransaction(m_rtr) + .SetParameterValue("@Name", indexName); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) + yield return new RemoteVolume( + rd.ConvertValueToString(0) ?? "", + rd.ConvertValueToString(1) ?? "", + rd.ConvertValueToInt64(2) + ); + } + + /// + /// A single block with source data for repair. + /// + /// The block hash. + /// The block size. + /// The file that contains the block. + /// The offset of the block in the file. + public sealed record BlockWithSourceData(string Hash, long Size, string File, long Offset); + + /// + /// A single block with metadata source data for repair. + /// + /// The block hash. + /// The block size. + /// The path of the file or directory that contains the block. + public sealed record BlockWithMetadataSourceData(string Hash, long Size, string Path); + + /// + /// A single blocklist hash entry. + /// + /// The blockset id. + /// The hash of the blocklist entry (when done). + /// The total length of the blockset. + /// The index of the blocklist hash. + /// The index of the block in the blockset. + /// The hash of the entry. + public sealed record BlocklistHashesEntry( + long BlocksetId, + string BlocklistHash, + long BlocklistHashLength, + long BlocklistHashIndex, + int Index, + string Hash); + + /// + /// Helper interface for the missing block list. + /// + public interface IMissingBlockList : IDisposable, IAsyncDisposable + { + /// + /// Registers a block as restored. + /// + /// The block hash. + /// The block size. + /// The volume ID of the new target volume. + /// A cancellation token to cancel the operation. + /// A task that when awaited returns true if the block was successfully marked as restored, false otherwise. + Task SetBlockRestored(string hash, long size, long volumeId, CancellationToken token); + /// + /// Gets the list of files that contains missing blocks. + /// + /// The blocksize setting. + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of representing the files with missing blocks. + IAsyncEnumerable GetSourceFilesWithBlocks(long blocksize, CancellationToken token); + /// + /// Gets a list for filesystem entries that contain missing blocks in metadata. + /// + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of representing the metadata blocks. + IAsyncEnumerable GetSourceItemsWithMetadataBlocks(CancellationToken token); + /// + /// Gets missing blocklist hashes. + /// + /// The number of hashes for each block. + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of representing the blocklist hashes. + IAsyncEnumerable GetBlocklistHashes(long hashesPerBlock, CancellationToken token); + /// + /// Gets the number of missing blocks. + /// + /// A cancellation token to cancel the operation. + /// A task that when awaited returns the count of missing blocks. + Task GetMissingBlockCount(CancellationToken token); + /// + /// Gets all the filesets that are affected by missing blocks. + /// + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of representing the filesets that contain missing blocks. + IAsyncEnumerable GetFilesetsUsingMissingBlocks(CancellationToken token); + /// + /// Gets a list of remote files that may contain missing blocks. + /// + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of representing the remote volumes that may contain missing blocks. + IAsyncEnumerable GetMissingBlockSources(CancellationToken token); + } + + /// + /// Implementation of the missing block list. + /// + private class MissingBlockList : IMissingBlockList + { + /// + /// The connection to the database + /// + private readonly SqliteConnection m_connection; + /// + /// The transaction to use + /// + private readonly ReusableTransaction m_rtr; + /// + /// Updates the "Restored" status of a block in the temporary missing blocks table for a given hash and size. + /// + private SqliteCommand m_insertCommand = null!; + /// + /// Inserts or ignores a block and its volume assignment into the "DuplicateBlock" table for a given hash and size. + /// + private SqliteCommand m_copyIntoDuplicatedBlocks = null!; + /// + /// Updates the "VolumeID" of a block in the "Block" table to assign it to a new volume for a given hash and size. + /// + private SqliteCommand m_assignBlocksToNewVolume = null!; + /// + /// Selects the hash and size of all missing blocks (where "Restored" is 0) from the temporary missing blocks table. + /// + private SqliteCommand m_missingBlocksCommand = null!; + /// + /// Counts the number of missing blocks (where "Restored" is 0) in the temporary missing blocks table. + /// + private SqliteCommand m_missingBlocksCountCommand = null!; + + /// + /// The name of the temporary table. + /// + private readonly string m_tablename; + /// + /// The name of the volume where blocks are missing. + /// + private readonly string m_volumename; + + /// + /// Whether the object has been disposed. + /// + private bool m_isDisposed = false; + + /// + /// Creates a new missing block list. + /// + /// The name of the volume with missing blocks. + /// The connection to the database. + /// The transaction to use. + private MissingBlockList(string volumename, SqliteConnection connection, ReusableTransaction rtr) + { + m_connection = connection; + m_rtr = rtr; + m_volumename = volumename; + var tablename = "MissingBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + m_tablename = tablename; + } + + /// + /// Creates a new instance of the missing block list. + /// + /// The name of the volume with missing blocks. + /// The connection to the database. + /// The transaction to use. + /// A cancellation token to cancel the operation. + /// A task that when awaited returns a new instance of . + public static async Task CreateMissingBlockList(string volumename, SqliteConnection connection, ReusableTransaction transaction, CancellationToken token) + { + var blocklist = new MissingBlockList(volumename, connection, transaction); + await using (var cmd = connection.CreateCommand(transaction.Transaction)) + { + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{blocklist.m_tablename}"" ( + ""Hash"" TEXT NOT NULL, + ""Size"" INTEGER NOT NULL, + ""Restored"" INTEGER NOT NULL + ) + ", token) + .ConfigureAwait(false); + + cmd.SetCommandAndParameters($@" + INSERT INTO ""{blocklist.m_tablename}"" ( + ""Hash"", + ""Size"", + ""Restored"" + ) + SELECT DISTINCT + ""Block"".""Hash"", + ""Block"".""Size"", + 0 AS ""Restored"" FROM ""Block"", + ""Remotevolume"" + WHERE + ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" + AND ""Remotevolume"".""Name"" = @Name + ") + .SetParameterValue("@Name", volumename); + + var blockCount = await cmd.ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + if (blockCount == 0) + throw new Exception($"Unexpected empty block volume: {0}"); + + await cmd.ExecuteNonQueryAsync($@" + CREATE UNIQUE INDEX ""{blocklist.m_tablename}-Ix"" + ON ""{blocklist.m_tablename}"" ( + ""Hash"", + ""Size"", + ""Restored"" + ) + ", token) + .ConfigureAwait(false); + } + + blocklist.m_insertCommand = await connection.CreateCommandAsync($@" + UPDATE ""{blocklist.m_tablename}"" + SET ""Restored"" = @NewRestoredValue + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size + AND ""Restored"" = @PreviousRestoredValue + ", token) + .ConfigureAwait(false); + + blocklist.m_copyIntoDuplicatedBlocks = await connection.CreateCommandAsync(@" + INSERT OR IGNORE INTO ""DuplicateBlock"" ( + ""BlockID"", + ""VolumeID"" + ) + SELECT + ""Block"".""ID"", + ""Block"".""VolumeID"" + FROM ""Block"" + WHERE + ""Block"".""Hash"" = @Hash + AND ""Block"".""Size"" = @Size + ", token) + .ConfigureAwait(false); + + blocklist.m_assignBlocksToNewVolume = await connection.CreateCommandAsync(@" + UPDATE ""Block"" + SET ""VolumeID"" = @TargetVolumeId + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size + ", token) + .ConfigureAwait(false); + + var m_missingBlocksQuery = $@" + SELECT + ""{blocklist.m_tablename}"".""Hash"", + ""{blocklist.m_tablename}"".""Size"" + FROM ""{blocklist.m_tablename}"" + WHERE ""{blocklist.m_tablename}"".""Restored"" = @Restored "; + + blocklist.m_missingBlocksCommand = + await connection.CreateCommandAsync(m_missingBlocksQuery, token) + .ConfigureAwait(false); + + blocklist.m_missingBlocksCountCommand = + await connection.CreateCommandAsync($@" + SELECT COUNT(*) + FROM ({m_missingBlocksQuery}) + ", token) + .ConfigureAwait(false); + + return blocklist; + } + + /// + public async Task SetBlockRestored(string hash, long size, long targetVolumeId, CancellationToken token) + { + var restored = await m_insertCommand + .SetTransaction(m_rtr) + .SetParameterValue("@NewRestoredValue", 1) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .SetParameterValue("@PreviousRestoredValue", 0) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false) == 1; + + if (restored) + { + await m_copyIntoDuplicatedBlocks.SetTransaction(m_rtr) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + var c = await m_assignBlocksToNewVolume.SetTransaction(m_rtr) + .SetParameterValue("@TargetVolumeId", targetVolumeId) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + if (c != 1) + throw new Exception($"Unexpected number of updated blocks: {c} != 1"); + } + + return restored; + } + + /// + public async IAsyncEnumerable GetSourceFilesWithBlocks(long blocksize, [EnumeratorCancellation] CancellationToken token) + { + await using var cmd = m_connection.CreateCommand($@" + SELECT DISTINCT + ""{m_tablename}"".""Hash"", + ""{m_tablename}"".""Size"", + ""File"".""Path"", + ""BlocksetEntry"".""Index"" * {blocksize} + FROM + ""{m_tablename}"", + ""Block"", + ""BlocksetEntry"", + ""File"" + WHERE + ""File"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" + AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" + AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" + AND ""{m_tablename}"".""Restored"" = @Restored + ORDER BY + ""{m_tablename}"".""Hash"", + ""{m_tablename}"".""Size"", + ""File"".""Path"", + ""BlocksetEntry"".""Index"" * {blocksize} + ") + .SetTransaction(m_rtr) + .SetParameterValue("@Restored", 0); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) + { + var hash = rd.ConvertValueToString(0) ?? throw new Exception("Hash value was null"); + var size = rd.ConvertValueToInt64(1); + var file = rd.ConvertValueToString(2) ?? throw new Exception("File value was null"); + var offset = rd.ConvertValueToInt64(3); + + yield return new BlockWithSourceData(hash, size, file, offset); + } + } + + /// + public async IAsyncEnumerable GetSourceItemsWithMetadataBlocks([EnumeratorCancellation] CancellationToken token) + { + await using var cmd = m_connection.CreateCommand($@" + SELECT DISTINCT + ""{m_tablename}"".""Hash"", + ""{m_tablename}"".""Size"", + ""File"".""Path"" + FROM + ""{m_tablename}"", + ""Block"", + ""BlocksetEntry"", + ""Metadataset"", + ""File"" + WHERE + ""File"".""MetadataID"" == ""Metadataset"".""ID"" + AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" + AND ""{m_tablename}"".""Hash"" = ""Block"".""Hash"" + AND ""{m_tablename}"".""Size"" = ""Block"".""Size"" + AND ""{m_tablename}"".""Restored"" = @Restored + ORDER BY + ""{m_tablename}"".""Hash"", + ""{m_tablename}"".""Size"", + ""File"".""Path"" + ") + .SetTransaction(m_rtr) + .SetParameterValue("@Restored", 0); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) + { + var hash = rd.ConvertValueToString(0) ?? throw new Exception("Hash value was null"); + var size = rd.ConvertValueToInt64(1); + var path = rd.ConvertValueToString(2) ?? throw new Exception("File value was null"); + + yield return new BlockWithMetadataSourceData(hash, size, path); + } + } + + /// + public async IAsyncEnumerable GetBlocklistHashes(long hashesPerBlock, [EnumeratorCancellation] CancellationToken token) + { + await using var cmd = m_connection.CreateCommand(m_rtr); + + var blocklistTableName = $"BlocklistHashList-{Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray())}"; + + try + { + // We need to create a snapshot as we will be updating the m_tablename table during enumeration + await cmd.SetCommandAndParameters($@" + CREATE TEMPORARY TABLE ""{blocklistTableName}"" AS + SELECT + ""b"".""Hash"" AS ""BlockHash"", + ""bs"".""Id"" AS ""BlocksetId"", + ""bs"".""Length"" AS ""BlocksetLength"", + ""bse"".""Index"" / @HashesPerBlock AS ""BlocklistHashIndex"", + ( + SELECT ""blh"".""Hash"" + FROM ""BlocklistHash"" ""blh"" + WHERE + ""blh"".""BlocksetID"" = ""bs"".""ID"" + AND ""blh"".""Index"" == ""bse"".""Index"" / @HashesPerBlock + LIMIT 1 + ) AS ""BlocklistHashHash"", + ""bse"".""Index"" AS ""BlocksetEntryIndex"" + FROM ""BlocksetEntry"" ""bse"" + JOIN ""Block"" ""b"" + ON ""b"".""ID"" = ""bse"".""BlockID"" + JOIN ""Blockset"" ""bs"" + ON ""bs"".""ID"" = ""bse"".""BlocksetID"" + WHERE EXISTS ( + SELECT 1 + FROM ""BlocklistHash"" ""blh"" + JOIN ""{m_tablename}"" ""mt"" + ON ""mt"".""Hash"" = ""blh"".""Hash"" + WHERE + ""blh"".""BlocksetID"" = ""bs"".""ID"" + AND ""mt"".""Restored"" = @Restored + ) + ") + .SetParameterValue("@HashesPerBlock", hashesPerBlock) + .SetParameterValue("@Restored", 0) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + cmd.SetCommandAndParameters($@" + SELECT + ""BlockHash"", + ""BlocksetId"", + ""BlocksetLength"", + ""BlocklistHashIndex"", + ""BlocklistHashHash"", + ""BlocksetEntryIndex"" + FROM ""{blocklistTableName}"" + ORDER BY + ""BlocksetId"", + ""BlocklistHashIndex"", + ""BlocksetEntryIndex"" + "); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) + { + var hash = rd.ConvertValueToString(0) ?? throw new Exception("Block.Hash is null"); + var blocksetId = rd.ConvertValueToInt64(1); + var length = rd.ConvertValueToInt64(2); + var blocklistHashIndex = rd.ConvertValueToInt64(3); + var blocklistHash = rd.ConvertValueToString(4) ?? throw new Exception("BlocklistHash is null"); + var index = rd.ConvertValueToInt64(5); + + yield return new BlocklistHashesEntry( + blocksetId, + blocklistHash, + length, + blocklistHashIndex, + (int)index, + hash + ); + } + } + finally + { + try + { + await cmd.ExecuteNonQueryAsync($@" + DROP TABLE IF EXISTS ""{blocklistTableName}"" + ", token) + .ConfigureAwait(false); + } + catch { } + } + } + + /// + public async Task GetMissingBlockCount(CancellationToken token) + { + return await m_missingBlocksCountCommand + .SetTransaction(m_rtr) + .SetParameterValue("@Restored", 0) + .ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false); + } + + /// + /// Gets the list of missing blocks. + /// + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of tuples containing the block hash and size. + public async IAsyncEnumerable<(string Hash, long Size)> GetMissingBlocks([EnumeratorCancellation] CancellationToken token) + { + m_missingBlocksCommand + .SetTransaction(m_rtr) + .SetParameterValue("@Restored", 0); + + await foreach (var rd in m_missingBlocksCommand.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) + yield return ( + rd.ConvertValueToString(0) ?? "", + rd.ConvertValueToInt64(1) + ); + } + + /// + public async Task MoveBlocksToNewVolume(long targetVolumeId, long sourceVolumeId, CancellationToken token) + { + if (targetVolumeId <= 0) + throw new ArgumentOutOfRangeException(nameof(targetVolumeId), "Target volume ID must be greater than 0"); + + // Move the source blocks into the DuplicateBlock table + await using var cmd = m_connection.CreateCommand($@" + INSERT OR IGNORE INTO ""DuplicateBlock"" ( + ""BlockID"", + ""VolumeID"" + ) + SELECT + ""b"".""ID"", + ""b"".""VolumeID"" + FROM ""Block"" ""b"" + WHERE + ""b"".""VolumeID"" = @SourceVolumeId + AND (""b"".""Hash"", ""b"".""Size"") IN ( + SELECT + ""Hash"", + ""Size"" + FROM ""{m_tablename}"" + WHERE ""Restored"" = @Restored + ) + ") + .SetParameterValue("@SourceVolumeId", sourceVolumeId) + .SetParameterValue("@Restored", 1); + + var moved = await cmd.ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + // Then update the blocks table to point to the new volume + var updated = await cmd.SetCommandAndParameters($@" + UPDATE ""Block"" + SET ""VolumeID"" = @TargetVolumeId + WHERE + ""VolumeID"" = @SourceVolumeId + AND (""Hash"", ""Size"") IN ( + SELECT + ""Hash"", + ""Size"" + FROM ""{m_tablename}"" + WHERE ""Restored"" = @Restored + ) + ") + .SetParameterValue("@TargetVolumeId", targetVolumeId) + .SetParameterValue("@SourceVolumeId", sourceVolumeId) + .SetParameterValue("@Restored", 1) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + if (updated != moved) + throw new Exception($"Unexpected number of updated blocks: {updated} != {moved}"); + + return updated; + } + + /// + public async IAsyncEnumerable GetFilesetsUsingMissingBlocks([EnumeratorCancellation] CancellationToken token) + { + var blocks = $@" + SELECT DISTINCT ""FileLookup"".""ID"" AS ID + FROM + ""{m_tablename}"", + ""Block"", + ""Blockset"", + ""BlocksetEntry"", + ""FileLookup"" + WHERE + ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" + AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" + AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" + AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" + "; + var blocklists = $@" + SELECT DISTINCT ""FileLookup"".""ID"" AS ID + FROM + ""{m_tablename}"", + ""Block"", + ""Blockset"", + ""BlocklistHash"", + ""FileLookup"" + WHERE + ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" + AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" + AND ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" + AND ""BlocklistHash"".""BlocksetID"" = ""Blockset"".""ID"" + AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" + "; + var cmdtxt = $@" + SELECT DISTINCT + ""RemoteVolume"".""Name"", + ""RemoteVolume"".""Hash"", + ""RemoteVolume"".""Size"" + FROM + ""RemoteVolume"", + ""FilesetEntry"", + ""Fileset"" + WHERE + ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" + AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" + AND ""RemoteVolume"".""Type"" = @Type + AND ""FilesetEntry"".""FileID"" IN ( + SELECT DISTINCT ""ID"" + FROM ( + {blocks} UNION {blocklists} + ) + ) + "; + + await using var cmd = m_connection.CreateCommand(cmdtxt) + .SetTransaction(m_rtr) + .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) + yield return new RemoteVolume( + rd.ConvertValueToString(0) ?? "", + rd.ConvertValueToString(1) ?? "", + rd.ConvertValueToInt64(2) + ); + } + + /// + public async IAsyncEnumerable GetMissingBlockSources([EnumeratorCancellation] CancellationToken token) + { + await using var cmd = m_connection.CreateCommand($@" + SELECT DISTINCT + ""RemoteVolume"".""Name"", + ""RemoteVolume"".""Hash"", + ""RemoteVolume"".""Size"" + FROM + ""RemoteVolume"", + ""Block"", + ""{m_tablename}"" + WHERE + ""{m_tablename}"".""Restored"" = @Restored + AND ""Block"".""Hash"" = ""{m_tablename}"".""Hash"" + AND ""Block"".""Size"" = ""{m_tablename}"".""Size"" + AND ""Block"".""VolumeID"" = ""RemoteVolume"".""ID"" + AND ""Remotevolume"".""Name"" != @Name + ") + .SetTransaction(m_rtr) + .SetParameterValue("@Restored", 0) + .SetParameterValue("@Name", m_volumename); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) + yield return new RemoteVolume( + rd.ConvertValueToString(0) ?? "", + rd.ConvertValueToString(1) ?? "", + rd.ConvertValueToInt64(2) + ); + } + + public void Dispose() + { + if (m_isDisposed) + return; + + DisposeAsync().AsTask().Await(); + } + + public async ValueTask DisposeAsync() + { + if (m_isDisposed) + return; + + m_isDisposed = true; + try + { + if (m_tablename != null) + { + + await using var cmd = await m_connection.CreateCommandAsync($@"DROP TABLE IF EXISTS ""{m_tablename}""", default) + .ConfigureAwait(false); + + await cmd.SetTransaction(m_rtr) + .ExecuteNonQueryAsync() + .ConfigureAwait(false); + } + } + catch { } + + try { m_insertCommand?.Dispose(); } + catch { } + } + } + + /// + /// Creates a new missing block list for the specified volume. + /// + /// The name of the volume with missing blocks. + /// A cancellation token to cancel the operation. + /// A task that when awaited returns an instance of . + public async Task CreateBlockList(string volumename, CancellationToken token) + { + return await MissingBlockList.CreateMissingBlockList(volumename, m_connection, m_rtr, token) + .ConfigureAwait(false); + } + + /// + /// Fixes duplicate metadata hashes in the database. + /// + /// A cancellation token to cancel the operation. + /// A task that when completed indicates that the repair has been attempted. + public async Task FixDuplicateMetahash(CancellationToken token) + { + await using var cmd = m_connection.CreateCommand(m_rtr.Transaction); + + var sql_count = @" + SELECT COUNT(*) + FROM ( + SELECT DISTINCT ""C1"" + FROM ( + SELECT COUNT(*) AS ""C1"" + FROM ( + SELECT DISTINCT ""BlocksetID"" + FROM ""Metadataset"" + ) + UNION SELECT COUNT(*) AS ""C1"" + FROM ""Metadataset"" + ) + ) + "; + + var x = await cmd.ExecuteScalarInt64Async(sql_count, 0, token) + .ConfigureAwait(false); + + if (x > 1) + { + Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateMetadataHashes", "Found duplicate metadatahashes, repairing"); + + var tablename = "TmpFile-" + Guid.NewGuid().ToString("N"); + + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{tablename}"" + AS SELECT * + FROM ""File"" + ", token) + .ConfigureAwait(false); + + var sql = @" + SELECT + ""A"".""ID"", + ""B"".""BlocksetID"" + FROM ( + SELECT + MIN(""ID"") AS ""ID"", + COUNT(""ID"") AS ""Duplicates"" + FROM ""Metadataset"" + GROUP BY ""BlocksetID"" + ) ""A"", + ""Metadataset"" ""B"" + WHERE + ""A"".""Duplicates"" > 1 + AND ""A"".""ID"" = ""B"".""ID"" + "; + + await using (var c2 = m_connection.CreateCommand(m_rtr.Transaction)) + { + c2.SetCommandAndParameters($@" + UPDATE ""{tablename}"" + SET ""MetadataID"" = @MetadataId + WHERE ""MetadataID"" IN ( + SELECT ""ID"" + FROM ""Metadataset"" + WHERE ""BlocksetID"" = @BlocksetId + ); + DELETE FROM ""Metadataset"" + WHERE + ""BlocksetID"" = @BlocksetId + AND ""ID"" != @MetadataId + "); + + await using var rd = await cmd.ExecuteReaderAsync(sql, token) + .ConfigureAwait(false); + + while (await rd.ReadAsync(token).ConfigureAwait(false)) + { + await c2 + .SetParameterValue("@MetadataId", rd.GetValue(0)) + .SetParameterValue("@BlocksetId", rd.GetValue(1)) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } + } + + sql = $@" + SELECT + ""ID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"", + ""Entries"" + FROM ( + SELECT + MIN(""ID"") AS ""ID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"", + COUNT(*) as ""Entries"" + FROM ""{tablename}"" + GROUP BY + ""Path"", + ""BlocksetID"", + ""MetadataID"" + ) + WHERE ""Entries"" > 1 + ORDER BY ""ID"" + "; + + await using (var c2 = m_connection.CreateCommand(m_rtr.Transaction)) + { + c2.SetCommandAndParameters($@" + UPDATE ""FilesetEntry"" + SET ""FileID"" = @FileId + WHERE ""FileID"" IN ( + SELECT ""ID"" + FROM ""{tablename}"" + WHERE + ""Path"" = @Path + AND ""BlocksetID"" = @BlocksetId + AND ""MetadataID"" = @MetadataId + ); + DELETE FROM ""{tablename}"" + WHERE + ""Path"" = @Path + AND ""BlocksetID"" = @BlocksetId + AND ""MetadataID"" = @MetadataId + AND ""ID"" != @FileId + "); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(sql, token).ConfigureAwait(false)) + { + await c2 + .SetParameterValue("@FileId", rd.GetValue(0)) + .SetParameterValue("@Path", rd.GetValue(1)) + .SetParameterValue("@BlocksetId", rd.GetValue(2)) + .SetParameterValue("@MetadataId", rd.GetValue(3)) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } + } + + await cmd.ExecuteNonQueryAsync($@" + DELETE FROM ""FileLookup"" + WHERE ""ID"" NOT IN ( + SELECT ""ID"" + FROM ""{tablename}"" + ) + ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{tablename}-Ix"" + ON ""{tablename}"" ( + ""ID"", + ""MetadataID"" + ) + ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""FileLookup"" + SET ""MetadataID"" = ( + SELECT ""MetadataID"" + FROM ""{tablename}"" ""A"" + WHERE ""A"".""ID"" = ""FileLookup"".""ID"" + ) + ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{tablename}"" ", token) + .ConfigureAwait(false); + + x = await cmd.ExecuteScalarInt64Async(sql_count, 0, token) + .ConfigureAwait(false); + + if (x > 1) + throw new Interface.UserInformationException("Repair failed, there are still duplicate metadatahashes!", "DuplicateHashesRepairFailed"); + + Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateMetadataHashesFixed", "Duplicate metadatahashes repaired succesfully"); + + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); + } + } + + /// + /// Fixes duplicate file entries in the database. + /// + /// A cancellation token to cancel the operation. + /// A task that when completed indicates that the repair has been attempted. + public async Task FixDuplicateFileentries(CancellationToken token) + { + await using var cmd = m_connection.CreateCommand(m_rtr.Transaction); + + var sql_count = @" + SELECT COUNT(*) + FROM ( + SELECT + ""PrefixID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"", + COUNT(*) as ""Duplicates"" + FROM ""FileLookup"" + GROUP BY + ""PrefixID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"" + ) + WHERE ""Duplicates"" > 1 + "; + + var x = await cmd.ExecuteScalarInt64Async(sql_count, 0, token) + .ConfigureAwait(false); + + if (x > 0) + { + Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateFileEntries", "Found duplicate file entries, repairing"); + + var sql = @" + SELECT + ""ID"", + ""PrefixID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"", + ""Entries"" + FROM ( + SELECT + MIN(""ID"") AS ""ID"", + ""PrefixID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"", + COUNT(*) as ""Entries"" + FROM ""FileLookup"" + GROUP BY + ""PrefixID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"" + ) + WHERE ""Entries"" > 1 + ORDER BY ""ID"" + "; + + await using (var c2 = m_connection.CreateCommand(m_rtr.Transaction)) + { + c2.SetCommandAndParameters(@" + UPDATE ""FilesetEntry"" + SET ""FileID"" = @FileId + WHERE ""FileID"" IN ( + SELECT ""ID"" + FROM ""FileLookup"" + WHERE + ""PrefixID"" = @PrefixId + AND ""Path"" = @Path + AND ""BlocksetID"" = @BlocksetId + AND ""MetadataID"" = @MetatadataId + ); + DELETE FROM ""FileLookup"" + WHERE + ""PrefixID"" = @PrefixId + AND ""Path"" = @Path + AND ""BlocksetID"" = @BlocksetId + AND ""MetadataID"" = @MetadataId + AND ""ID"" != @FileId + "); + + cmd.SetCommandAndParameters(sql); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) + { + await c2 + .SetParameterValue("@FileId", rd.GetValue(0)) + .SetParameterValue("@PrefixId", rd.GetValue(1)) + .SetParameterValue("@Path", rd.GetValue(2)) + .SetParameterValue("@BlocksetId", rd.GetValue(3)) + .SetParameterValue("@MetadataId", rd.GetValue(4)) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } + } + + x = await cmd.ExecuteScalarInt64Async(sql_count, 0, token) + .ConfigureAwait(false); + + if (x > 1) + throw new Interface.UserInformationException("Repair failed, there are still duplicate file entries!", "DuplicateFilesRepairFailed"); + + Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateFileEntriesFixed", "Duplicate file entries repaired succesfully"); + + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); + } + + } + + /// + /// Fixes missing blocklist hashes in the database. + /// + /// The hash algorithm used for the blocklist hashes. + /// The size of each block in bytes. + /// A cancellation token to cancel the operation. + /// A task that when completed indicates that the repair has been attempted. + public async Task FixMissingBlocklistHashes(string blockhashalgorithm, long blocksize, CancellationToken token) + { + var blocklistbuffer = new byte[blocksize]; + + await using var cmd = m_connection.CreateCommand(m_rtr.Transaction); + using var blockhasher = HashFactory.CreateHasher(blockhashalgorithm); + + var hashsize = blockhasher.HashSize / 8; + + var sql = $@" + SELECT * + FROM ( + SELECT + ""N"".""BlocksetID"", + ((""N"".""BlockCount"" + {blocksize / hashsize} - 1) / {blocksize / hashsize}) AS ""BlocklistHashCountExpected"", + CASE + WHEN ""G"".""BlocklistHashCount"" IS NULL + THEN 0 + ELSE ""G"".""BlocklistHashCount"" + END AS ""BlocklistHashCountActual"" + FROM ( + SELECT + ""BlocksetID"", + COUNT(*) AS ""BlockCount"" + FROM ""BlocksetEntry"" + GROUP BY ""BlocksetID"" + ) ""N"" + LEFT OUTER JOIN ( + SELECT + ""BlocksetID"", + COUNT(*) AS ""BlocklistHashCount"" + FROM ""BlocklistHash"" + GROUP BY ""BlocksetID"" + ) ""G"" + ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" + WHERE ""N"".""BlockCount"" > 1 + ) + WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"" + "; + + var countsql = @$" + SELECT COUNT(*) + FROM ({sql}) + "; + + var itemswithnoblocklisthash = await cmd + .ExecuteScalarInt64Async(countsql, 0, token) + .ConfigureAwait(false); + + if (itemswithnoblocklisthash != 0) + { + Logging.Log.WriteInformationMessage(LOGTAG, "MissingBlocklistHashes", "Found {0} missing blocklisthash entries, repairing", itemswithnoblocklisthash); + await using (var c2 = m_connection.CreateCommand(m_rtr.Transaction)) + await using (var c3 = m_connection.CreateCommand(m_rtr.Transaction)) + await using (var c4 = m_connection.CreateCommand(m_rtr.Transaction)) + await using (var c5 = m_connection.CreateCommand(m_rtr.Transaction)) + await using (var c6 = m_connection.CreateCommand(m_rtr.Transaction)) + { + c3.SetCommandAndParameters(@" + INSERT INTO ""BlocklistHash"" ( + ""BlocksetID"", + ""Index"", + ""Hash"" + ) + VALUES ( + @BlocksetId, + @Index, + @Hash + ) + "); + + c4.SetCommandAndParameters(@" + SELECT ""ID"" + FROM ""Block"" + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size + "); + + c5.SetCommandAndParameters(@" + SELECT ""ID"" + FROM ""DeletedBlock"" + WHERE + ""Hash"" = @Hash + AND ""Size"" = @Size + AND ""VolumeID"" IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE + ""Type"" = @Type + AND ( + ""State"" IN ( + @State1, + @State2 + ) + ) + ) + "); + + c6.SetCommandAndParameters(@" + INSERT INTO ""Block"" ( + ""Hash"", + ""Size"", + ""VolumeID"" + ) + SELECT + ""Hash"", + ""Size"", + ""VolumeID"" + FROM ""DeletedBlock"" + WHERE ""ID"" = @DeletedBlockId + LIMIT 1; + DELETE FROM ""DeletedBlock"" + WHERE ""ID"" = @DeletedBlockId; + "); + + await foreach (var e in cmd.ExecuteReaderEnumerableAsync(sql, token).ConfigureAwait(false)) + { + var blocksetid = e.ConvertValueToInt64(0); + var ix = 0L; + int blocklistoffset = 0; + + await c2.SetCommandAndParameters(@" + DELETE FROM ""BlocklistHash"" + WHERE ""BlocksetID"" = @BlocksetId + ") + .SetParameterValue("@BlocksetId", blocksetid) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + c2.SetCommandAndParameters(@" + SELECT ""A"".""Hash"" + FROM + ""Block"" ""A"", + ""BlocksetEntry"" ""B"" + WHERE + ""A"".""ID"" = ""B"".""BlockID"" + AND ""B"".""BlocksetID"" = @BlocksetId + ORDER BY ""B"".""Index"" + ") + .SetParameterValue("@BlocksetId", blocksetid); + + await foreach (var h in c2.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) + { + var tmp = Convert.FromBase64String(h.ConvertValueToString(0) ?? throw new Exception("Hash value was null")); + if (blocklistbuffer.Length - blocklistoffset < tmp.Length) + { + var blkey = Convert.ToBase64String(blockhasher.ComputeHash(blocklistbuffer, 0, blocklistoffset)); + + // Check if the block exists in "blocks" + var existingBlockId = await c4 + .SetParameterValue("@Hash", blkey) + .SetParameterValue("@Size", blocklistoffset) + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); + + if (existingBlockId <= 0) + { + var deletedBlockId = await c5 + .SetParameterValue("@Hash", blkey) + .SetParameterValue("@Size", blocklistoffset) + .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) + .SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()) + .SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()) + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); + + if (deletedBlockId <= 0) + throw new Exception($"Missing block for blocklisthash: {blkey}"); + else + { + var rc = await c6 + .SetParameterValue("@DeletedBlockId", deletedBlockId) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + if (rc != 2) + throw new Exception($"Unexpected update count: {rc}"); + } + } + + // Add to table + await c3.SetParameterValue("@BlocksetId", blocksetid) + .SetParameterValue("@Index", ix) + .SetParameterValue("@Hash", blkey) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + ix++; + blocklistoffset = 0; + } + + Array.Copy(tmp, 0, blocklistbuffer, blocklistoffset, tmp.Length); + blocklistoffset += tmp.Length; + + } + + if (blocklistoffset != 0) + { + var blkeyfinal = Convert.ToBase64String(blockhasher.ComputeHash(blocklistbuffer, 0, blocklistoffset)); + + // Ensure that the block exists in "blocks" + var existingBlockId = await c4 + .SetParameterValue("@Hash", blkeyfinal) + .SetParameterValue("@Size", blocklistoffset) + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); + + if (existingBlockId <= 0) + { + var deletedBlockId = await c5 + .SetParameterValue("@Hash", blkeyfinal) + .SetParameterValue("@Size", blocklistoffset) + .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) + .SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()) + .SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()) + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); + + if (deletedBlockId == 0) + throw new Exception($"Missing block for blocklisthash: {blkeyfinal}"); + else + { + var rc = await c6 + .SetParameterValue("@DeletedBlockId", deletedBlockId) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + if (rc != 2) + throw new Exception($"Unexpected update count: {rc}"); + } + } + + // Add to table + await c3 + .SetParameterValue("@BlocksetId", blocksetid) + .SetParameterValue("@Index", ix) + .SetParameterValue("@Hash", blkeyfinal) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } + } + } + + itemswithnoblocklisthash = await cmd + .ExecuteScalarInt64Async(countsql, 0, token) + .ConfigureAwait(false); + + if (itemswithnoblocklisthash != 0) + throw new Interface.UserInformationException($"Failed to repair, after repair {itemswithnoblocklisthash} blocklisthashes were missing", "MissingBlocklistHashesRepairFailed"); + + Logging.Log.WriteInformationMessage(LOGTAG, "MissingBlocklisthashesRepaired", "Missing blocklisthashes repaired succesfully"); + + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); + } + + } + + /// + /// Fixes duplicate blocklist hashes in the database. + /// + /// The size of each block in bytes. + /// The size of each hash in bytes. + /// A cancellation token to cancel the operation. + /// A task that when completed indicates that the repair has been attempted. + public async Task FixDuplicateBlocklistHashes(long blocksize, long hashsize, CancellationToken token) + { + await using var cmd = m_connection.CreateCommand(m_rtr.Transaction); + + var dup_sql = @" + SELECT * + FROM ( + SELECT + ""BlocksetID"", + ""Index"", + COUNT(*) AS ""EC"" + FROM ""BlocklistHash"" + GROUP BY + ""BlocksetID"", + ""Index"" + ) + WHERE ""EC"" > 1 + "; + + var sql_count = @$" + SELECT COUNT(*) + FROM ({dup_sql}) + "; + + var x = await cmd.ExecuteScalarInt64Async(sql_count, 0, token) + .ConfigureAwait(false); + + if (x > 0) + { + Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateBlocklistHashes", "Found duplicate blocklisthash entries, repairing"); + + var unique_count = await cmd.ExecuteScalarInt64Async(@" + SELECT COUNT(*) + FROM ( + SELECT DISTINCT + ""BlocksetID"", + ""Index"" + FROM ""BlocklistHash"" + ) + ", 0, token) + .ConfigureAwait(false); + + await using (var c2 = m_connection.CreateCommand(m_rtr.Transaction)) + { + c2.SetCommandAndParameters(@" + DELETE FROM ""BlocklistHash"" + WHERE rowid IN ( + SELECT rowid + FROM ""BlocklistHash"" + WHERE + ""BlocksetID"" = @BlocksetId + AND ""Index"" = @Index + LIMIT @Limit + ) + "); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(dup_sql, token).ConfigureAwait(false)) + { + var expected = rd.GetInt32(2) - 1; + var actual = await c2 + .SetParameterValue("@BlocksetId", rd.GetValue(0)) + .SetParameterValue("@Index", rd.GetValue(1)) + .SetParameterValue("@Limit", expected) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + if (actual != expected) + throw new Exception($"Unexpected number of results after fix, got: {actual}, expected: {expected}"); + } + } + + x = await cmd.ExecuteScalarInt64Async(sql_count, token) + .ConfigureAwait(false); + + if (x > 1) + throw new Exception("Repair failed, there are still duplicate file entries!"); + + var real_count = await cmd.ExecuteScalarInt64Async(@" + SELECT Count(*) + FROM ""BlocklistHash"" + ", 0, token) + .ConfigureAwait(false); + + if (real_count != unique_count) + throw new Interface.UserInformationException($"Failed to repair, result should have been {unique_count} blocklist hashes, but result was {real_count} blocklist hashes", "DuplicateBlocklistHashesRepairFailed"); + + try + { + await VerifyConsistency(blocksize, hashsize, true, token) + .ConfigureAwait(false); + } + catch (Exception ex) + { + throw new Interface.UserInformationException("Repaired blocklisthashes, but the database was broken afterwards, rolled back changes", "DuplicateBlocklistHashesRepairFailed", ex); + } + + Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateBlocklistHashesRepaired", "Duplicate blocklisthashes repaired succesfully"); + + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); + } + } + + /// + /// Checks if all blocks in the specified volume are present in the database. + /// + /// The name of the volume to check. + /// A collection of blocks to check, represented as key-value pairs where the key is the block hash and the value is the block size. + /// A cancellation token to cancel the operation. + /// Thrown if not all blocks are found in the specified volume. + /// A task that when awaited indicates the completion of the check. + public async Task CheckAllBlocksAreInVolume(string filename, IEnumerable> blocks, CancellationToken token) + { + await using var cmd = m_connection.CreateCommand(m_rtr.Transaction); + var tablename = "ProbeBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + + try + { + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{tablename}"" ( + ""Hash"" TEXT NOT NULL, + ""Size"" INTEGER NOT NULL + ) + ", token) + .ConfigureAwait(false); + + cmd.SetCommandAndParameters($@" + INSERT INTO ""{tablename}"" ( + ""Hash"", + ""Size"" + ) + VALUES ( + @Hash, + @Size + ) + "); + + foreach (var kp in blocks) + { + await cmd + .SetParameterValue("@Hash", kp.Key) + .SetParameterValue("@Size", kp.Value) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } + + var id = await cmd.SetCommandAndParameters(@" + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" = @Name + ") + .SetParameterValue("@Name", filename) + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); + + var aliens = await cmd.SetCommandAndParameters($@" + SELECT COUNT(*) + FROM ( + SELECT ""A"".""VolumeID"" + FROM ""{tablename}"" ""B"" + LEFT OUTER JOIN ""Block"" ""A"" + ON ""A"".""Hash"" = ""B"".""Hash"" + AND ""A"".""Size"" = ""B"".""Size"" + ) + WHERE ""VolumeID"" != @VolumeId + ") + .SetParameterValue("@VolumeId", id) + .ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false); + + if (aliens != 0) + throw new Exception($"Not all blocks were found in {filename}"); + } + finally + { + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tablename}"" ", token) + .ConfigureAwait(false); + } + } + + /// + /// Checks if the provided blocklist matches the expected entries in the database for a given block hash and size. + /// + /// The hash of the blocklist to check. + /// The size of the blocklist in bytes. + /// The expected blocklist entries to compare against. + /// The size of each block in bytes. + /// The length of each block hash in bytes. + /// A cancellation token to cancel the operation. + /// A task that when awaited indicates the completion of the check. + /// Thrown if the blocklist does not match the expected entries. + public async Task CheckBlocklistCorrect(string hash, long length, IEnumerable blocklist, long blocksize, long blockhashlength, CancellationToken token) + { + await using var cmd = m_connection.CreateCommand(m_rtr.Transaction); + + var query = $@" + SELECT + ""C"".""Hash"", + ""C"".""Size"" + FROM + ""BlocksetEntry"" ""A"", + ( + SELECT + ""Y"".""BlocksetID"", + ""Y"".""Hash"" AS ""BlocklistHash"", + ""Y"".""Index"" AS ""BlocklistHashIndex"", + ""Z"".""Size"" AS ""BlocklistSize"", + ""Z"".""ID"" AS ""BlocklistHashBlockID"" + FROM + ""BlocklistHash"" ""Y"", + ""Block"" ""Z"" + WHERE + ""Y"".""Hash"" = ""Z"".""Hash"" + AND ""Y"".""Hash"" = @Hash + AND ""Z"".""Size"" = @Size + LIMIT 1 + ) ""B"", + ""Block"" ""C"" + WHERE + ""A"".""BlocksetID"" = ""B"".""BlocksetID"" + AND ""A"".""BlockID"" = ""C"".""ID"" + AND ""A"".""Index"" >= ""B"".""BlocklistHashIndex"" * ({blocksize} / {blockhashlength}) + AND ""A"".""Index"" < (""B"".""BlocklistHashIndex"" + 1) * ({blocksize} / {blockhashlength}) + ORDER BY ""A"".""Index"" + "; + + using var en = blocklist.GetEnumerator(); + + cmd.SetCommandAndParameters(query) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", length); + + await foreach (var r in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) + { + if (!en.MoveNext()) + throw new Exception($"Too few entries in source blocklist with hash {hash}"); + if (en.Current != r.ConvertValueToString(0)) + throw new Exception($"Mismatch in blocklist with hash {hash}"); + } + + if (en.MoveNext()) + throw new Exception($"Too many source blocklist entries in {hash}"); + } + + /// + /// Checks if there are any missing local filesets in the database. + /// + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of missing local fileset names. + public async IAsyncEnumerable MissingLocalFilesets([EnumeratorCancellation] CancellationToken token) + { + await using var cmd = m_connection.CreateCommand(@" + SELECT ""Name"" + FROM ""RemoteVolume"" + WHERE + ""Type"" = @Type + AND ""State"" NOT IN (@States) + AND ""ID"" NOT IN ( + SELECT ""VolumeID"" + FROM ""Fileset"" + ) + ") + .SetTransaction(m_rtr) + .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) + .ExpandInClauseParameterMssqlite("@States", [ + RemoteVolumeState.Deleting.ToString(), + RemoteVolumeState.Deleted.ToString() + ]); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) + yield return rd.ConvertValueToString(0) ?? ""; + } + + /// + /// Checks if there are any missing remote filesets in the database. + /// + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of tuples containing the fileset ID, timestamp, and whether it is a full backup. + public async IAsyncEnumerable<(long FilesetID, DateTime Timestamp, bool IsFull)> MissingRemoteFilesets([EnumeratorCancellation] CancellationToken token) + { + await using var cmd = m_connection.CreateCommand(@" + SELECT + ""ID"", + ""Timestamp"", + ""IsFullBackup"" + FROM ""Fileset"" + WHERE ""VolumeID"" NOT IN ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE + ""Type"" = @Type + AND ""State"" NOT IN (@States) + ) + ") + .SetTransaction(m_rtr) + .SetParameterValue("@Type", RemoteVolumeType.Files.ToString()) + .ExpandInClauseParameterMssqlite("@States", [ + RemoteVolumeState.Deleting.ToString(), + RemoteVolumeState.Deleted.ToString() + ]); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) + yield return ( + rd.ConvertValueToInt64(0), + ParseFromEpochSeconds(rd.ConvertValueToInt64(1)), + rd.ConvertValueToInt64(2) == BackupType.FULL_BACKUP + ); + } + + /// + /// Checks if there are any empty index files in the database. + /// + /// A cancellation token to cancel the operation. + /// An asynchronous enumerable of remote volumes that are empty index files. + public async IAsyncEnumerable EmptyIndexFiles([EnumeratorCancellation] CancellationToken token) + { + await using var cmd = m_connection.CreateCommand(@" + SELECT + ""Name"", + ""Hash"", + ""Size"" + FROM ""RemoteVolume"" + WHERE + ""Type"" = @Type + AND ""State"" IN (@States) + AND ""ID"" NOT IN ( + SELECT ""IndexVolumeId"" + FROM ""IndexBlockLink"" + ) + ") + .SetTransaction(m_rtr) + .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()) + .ExpandInClauseParameterMssqlite("@States", [ + RemoteVolumeState.Uploaded.ToString(), + RemoteVolumeState.Verified.ToString() + ]); + + await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false)) + yield return new RemoteVolume( + rd.ConvertValueToString(0) ?? "", + rd.ConvertValueToString(1) ?? "", + rd.ConvertValueToInt64(2) + ); + } + + public async Task FixEmptyMetadatasets(Options options, CancellationToken token) + { + using var cmd = m_connection.CreateCommand(@" + SELECT COUNT(*) + FROM Metadataset + JOIN Blockset + ON Metadataset.BlocksetID = Blockset.ID + WHERE Blockset.Length = 0 + ") + .SetTransaction(m_rtr); + + var emptyMetaCount = await cmd.ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false); + if (emptyMetaCount <= 0) + return; + + Logging.Log.WriteInformationMessage(LOGTAG, "ZeroLengthMetadata", "Found {0} zero-length metadata entries", emptyMetaCount); + + // Create replacement metadata + var emptyMeta = Utility.WrapMetadata(new Dictionary(), options); + var emptyBlocksetId = await GetEmptyMetadataBlocksetId(Array.Empty(), emptyMeta.FileHash, emptyMeta.Blob.Length, token) + .ConfigureAwait(false); + + if (emptyBlocksetId < 0) + throw new Interface.UserInformationException( + "Failed to locate an empty metadata blockset to replace missing metadata. Set the option --disable-replace-missing-metadata=true to ignore this and drop files with missing metadata.", + "FailedToLocateEmptyMetadataBlockset"); + + // Step 1: Create temp table with Metadataset IDs referencing empty blocksets (excluding the one to keep) + var tablename = "FixMetadatasets-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + + try + { + // TODO quotes + await cmd.SetCommandAndParameters(@$" + CREATE TEMP TABLE ""{tablename}"" AS + SELECT + ""m"".""ID"" AS ""MetadataID"", + ""m"".""BlocksetID"" + FROM Metadataset ""m"" + JOIN Blockset ""b"" + ON ""m"".""BlocksetID"" = ""b"".""ID"" + WHERE + ""b"".""Length"" = 0 + AND ""m"".""BlocksetID"" != @KeepBlockset + ") + .SetParameterValue("@KeepBlockset", emptyBlocksetId) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + // Step 2: Update FileLookup to use a valid metadata ID + await cmd.SetCommandAndParameters(@$" + UPDATE ""FileLookup"" + SET ""MetadataID"" = ( + SELECT ""ID"" + FROM ""Metadataset"" + WHERE ""BlocksetID"" = @KeepBlockset + LIMIT 1 + ) + WHERE ""MetadataID"" IN ( + SELECT ""MetadataID"" + FROM ""{tablename}"" + ) + ") + .SetParameterValue("@KeepBlockset", emptyBlocksetId) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + // Step 3: Delete obsolete Metadataset entries + await cmd.SetCommandAndParameters(@$" + DELETE FROM ""Metadataset"" + WHERE ID IN ( + SELECT ""MetadataID"" + FROM ""{tablename}"" + ) + ") + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + // Step 4: Delete orphaned blocksets (affected only) + await cmd.SetCommandAndParameters(@$" + DELETE FROM ""Blockset"" + WHERE + ID IN ( + SELECT ""BlocksetID"" + FROM ""{tablename}"" + ) + AND NOT EXISTS ( + SELECT 1 + FROM ""Metadataset"" + WHERE ""BlocksetID"" = ""Blockset"".""ID"" + ) + AND NOT EXISTS ( + SELECT 1 + FROM ""File"" + WHERE ""BlocksetID"" = ""Blockset"".""ID"" + ) + ") + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + // Step 5: Confirm all broken metadata entries are resolved + cmd.SetCommandAndParameters(@" + SELECT COUNT(*) + FROM ""Metadataset"" + JOIN ""Blockset"" + ON ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" + WHERE ""Blockset"".""Length"" = 0 + "); + + var remaining = await cmd.ExecuteScalarInt64Async(0, token); + if (remaining > 0) + throw new Interface.UserInformationException( + "Some zero-length metadata entries could not be repaired.", + "MetadataRepairFailed"); + + Logging.Log.WriteInformationMessage(LOGTAG, "ZeroLengthMetadataRepaired", "Zero length metadata entries repaired successfully"); + + await m_rtr.CommitAsync(token: token) + .ConfigureAwait(false); + } + finally + { + try + { + await cmd.SetCommandAndParameters($@"DROP TABLE IF EXISTS ""{tablename}"" ") + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } + catch (Exception ex) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "ErrorDroppingTempTable", ex, "Failed to drop temporary table {0}: {1}", tablename, ex.Message); + } + } + } + } + +} diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 268ce2e04..6c108f483 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -1,1474 +1,3001 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -#nullable enable - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Data; -using System.Linq; -using System.Text; -using Duplicati.Library.Common.IO; -using Duplicati.Library.Main.Operation.Restore; -using Duplicati.Library.Main.Volumes; -using Duplicati.Library.Utility; - -namespace Duplicati.Library.Main.Database -{ - internal class LocalRestoreDatabase : LocalDatabase - { - /// - /// The tag used for logging - /// - private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalRestoreDatabase)); - - protected readonly string m_temptabsetguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - /// - /// The name of the temporary table in the database, which is used to store the list of files to restore. - /// - protected string? m_tempfiletable; - protected string? m_tempblocktable; - protected ConcurrentBag m_connection_pool = []; - protected string? m_latestblocktable; - protected string? m_fileprogtable; - protected string? m_totalprogtable; - protected string? m_filesnewlydonetable; - - protected DateTime m_restoreTime; - - public DateTime RestoreTime { get { return m_restoreTime; } } - - public LocalRestoreDatabase(string path, long pagecachesize) - : this(new LocalDatabase(path, "Restore", false, pagecachesize)) - { - ShouldCloseConnection = true; - } - - public LocalRestoreDatabase(LocalDatabase dbparent) - : base(dbparent) - { - } - - /// - /// Create tables and triggers for automatic tracking of progress during a restore operation. - /// This replaces continuous requerying of block progress by iterating over blocks table. - /// SQLite is much faster keeping information up to date with internal triggers. - /// - /// This allows to create another table that keeps track - /// of all files that are done (all data blocks restored). - /// - /// The method is prepared to create a table that keeps track of all files being done completely. - /// That means, it fires for files where the number of restored blocks equals the number of all blocks. - /// It is intended to be used for fast identification of fully restored files to trigger their verification. - /// It should be read after a commit and truncated after putting the files to a verification queue. - /// Note: If a file is done once and then set back to a none restored state, the file is not automatically removed. - /// But if it reaches a restored state later, it will be re-added (trigger will fire) - /// - public void CreateProgressTracker(bool createFilesNewlyDoneTracker) - { - m_fileprogtable = "FileProgress-" + m_temptabsetguid; - m_totalprogtable = "TotalProgress-" + m_temptabsetguid; - m_filesnewlydonetable = createFilesNewlyDoneTracker ? "FilesNewlyDone-" + m_temptabsetguid : null; - - using (var cmd = m_connection.CreateCommand()) - { - // How to handle METADATA? - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_fileprogtable}"" ")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_fileprogtable}"" ( -""FileId"" INTEGER PRIMARY KEY, -""TotalBlocks"" INTEGER NOT NULL, ""TotalSize"" INTEGER NOT NULL, -""BlocksRestored"" INTEGER NOT NULL, ""SizeRestored"" INTEGER NOT NULL -)")); - - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_totalprogtable}"" ")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_totalprogtable}"" ( -""TotalFiles"" INTEGER NOT NULL, ""TotalBlocks"" INTEGER NOT NULL, ""TotalSize"" INTEGER NOT NULL, -""FilesFullyRestored"" INTEGER NOT NULL, ""FilesPartiallyRestored"" INTEGER NOT NULL, -""BlocksRestored"" INTEGER NOT NULL, ""SizeRestored"" INTEGER NOT NULL -)")); - - if (createFilesNewlyDoneTracker) - { - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}"" ")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_filesnewlydonetable}"" ( -""ID"" INTEGER PRIMARY KEY -)")); - } - - try - { - // Initialize statistics with File- and Block-Data (it is valid to already have restored blocks in files) - // A rebuild with this function should be valid anytime. - // Note: FilesNewlyDone is NOT initialized, as in initialization nothing is really new. - string sql; - - // We use a LEFT JOIN to allow for empty files (no data Blocks) - sql = FormatInvariant($@" INSERT INTO ""{m_fileprogtable}"" (""FileId"", ""TotalBlocks"", ""TotalSize"", ""BlocksRestored"", ""SizeRestored"") -SELECT ""F"".""ID"", IFNULL(COUNT(""B"".""ID""), 0), IFNULL(SUM(""B"".""Size""), 0) - , IFNULL(COUNT(CASE ""B"".""Restored"" WHEN 1 THEN ""B"".""ID"" ELSE NULL END), 0) - , IFNULL(SUM(CASE ""B"".""Restored"" WHEN 1 THEN ""B"".""Size"" ELSE 0 END), 0) - FROM ""{m_tempfiletable}"" ""F"" LEFT JOIN ""{m_tempblocktable}"" ""B"" - ON ""B"".""FileID"" = ""F"".""ID"" - WHERE ""B"".""Metadata"" IS NOT 1 - GROUP BY ""F"".""ID"" "); - - // Will be one row per file. - cmd.ExecuteNonQuery(sql); - - sql = FormatInvariant($@"INSERT INTO ""{m_totalprogtable}"" ( - ""TotalFiles"", ""TotalBlocks"", ""TotalSize"" -, ""FilesFullyRestored"", ""FilesPartiallyRestored"", ""BlocksRestored"", ""SizeRestored"" - ) - SELECT IFNULL(COUNT(""P"".""FileId""), 0), IFNULL(SUM(""P"".""TotalBlocks""), 0), IFNULL(SUM(""P"".""TotalSize""), 0) - , IFNULL(COUNT(CASE WHEN ""P"".""BlocksRestored"" = ""P"".""TotalBlocks"" THEN 1 ELSE NULL END), 0) - , IFNULL(COUNT(CASE WHEN ""P"".""BlocksRestored"" BETWEEN 1 AND ""P"".""TotalBlocks"" - 1 THEN 1 ELSE NULL END), 0) - , IFNULL(SUM(""P"".""BlocksRestored""), 0), IFNULL(SUM(""P"".""SizeRestored""), 0) - FROM ""{m_fileprogtable}"" ""P"" "); - - // Will result in a single line (no support to also track metadata) - cmd.ExecuteNonQuery(sql); - - // Finally we create TRIGGERs to keep all our statistics up to date. - // This is lightning fast, as SQLite uses internal hooks and our indices to do the update magic. - // Note: We do assume that neither files nor blocks will be added or deleted during restore process - // and that the size of each block stays constant so there is no need to track that information - // with additional INSERT and DELETE triggers. - - // A trigger to update the file-stat entry each time a block changes restoration state. - sql = FormatInvariant($@"CREATE TEMPORARY TRIGGER ""TrackRestoredBlocks_{m_tempblocktable}"" AFTER UPDATE OF ""Restored"" ON ""{m_tempblocktable}"" -WHEN OLD.""Restored"" != NEW.""Restored"" AND NEW.""Metadata"" = 0 -BEGIN UPDATE ""{m_fileprogtable}"" - SET ""BlocksRestored"" = ""{m_fileprogtable}"".""BlocksRestored"" + (NEW.""Restored"" - OLD.""Restored"") - , ""SizeRestored"" = ""{m_fileprogtable}"".""SizeRestored"" + ((NEW.""Restored"" - OLD.""Restored"") * NEW.Size) - WHERE ""{m_fileprogtable}"".""FileId"" = NEW.""FileID"" -; END "); - cmd.ExecuteNonQuery(sql); - - // A trigger to update total stats each time a file stat changed (nested triggering by file-stats) - sql = FormatInvariant($@"CREATE TEMPORARY TRIGGER ""UpdateTotalStats_{m_fileprogtable}"" AFTER UPDATE ON ""{m_fileprogtable}"" -BEGIN UPDATE ""{m_totalprogtable}"" - SET ""FilesFullyRestored"" = ""{m_totalprogtable}"".""FilesFullyRestored"" - + (CASE WHEN NEW.""BlocksRestored"" = NEW.""TotalBlocks"" THEN 1 ELSE 0 END) - - (CASE WHEN OLD.""BlocksRestored"" = OLD.""TotalBlocks"" THEN 1 ELSE 0 END) - , ""FilesPartiallyRestored"" = ""{m_totalprogtable}"".""FilesPartiallyRestored"" - + (CASE WHEN NEW.""BlocksRestored"" BETWEEN 1 AND NEW.""TotalBlocks"" - 1 THEN 1 ELSE 0 END) - - (CASE WHEN OLD.""BlocksRestored"" BETWEEN 1 AND OLD.""TotalBlocks"" - 1 THEN 1 ELSE 0 END) - , ""BlocksRestored"" = ""{m_totalprogtable}"".""BlocksRestored"" + NEW.""BlocksRestored"" - OLD.""BlocksRestored"" - , ""SizeRestored"" = ""{m_totalprogtable}"".""SizeRestored"" + NEW.""SizeRestored"" - OLD.""SizeRestored"" -; END"); - cmd.ExecuteNonQuery(sql); - - - if (createFilesNewlyDoneTracker) - { - // A trigger checking if a file is done (all blocks restored in file-stat) (nested triggering by file-stats) - sql = FormatInvariant($@"CREATE TEMPORARY TRIGGER ""UpdateFilesNewlyDone_{m_fileprogtable}"" AFTER UPDATE OF ""BlocksRestored"", ""TotalBlocks"" ON ""{m_fileprogtable}"" -WHEN NEW.""BlocksRestored"" = NEW.""TotalBlocks"" -BEGIN - INSERT OR IGNORE INTO ""{m_filesnewlydonetable}"" (""ID"") VALUES (NEW.""FileId""); -END "); - cmd.ExecuteNonQuery(sql); - } - - } - catch (Exception ex) - { - m_fileprogtable = null; - m_totalprogtable = null; - Logging.Log.WriteWarningMessage(LOGTAG, "ProgressTrackerSetupError", ex, "Failed to set up progress tracking tables"); - throw; - } - } - } - - public Tuple PrepareRestoreFilelist(DateTime restoretime, long[] versions, IFilter filter) - { - m_tempfiletable = "Fileset-" + m_temptabsetguid; - m_tempblocktable = "Blocks-" + m_temptabsetguid; - - using (var cmd = m_connection.CreateCommand()) - { - var filesetIds = GetFilesetIDs(Library.Utility.Utility.NormalizeDateTime(restoretime), versions).ToList(); - while (filesetIds.Count > 0) - { - var filesetId = filesetIds[0]; - filesetIds.RemoveAt(0); - - m_restoreTime = ParseFromEpochSeconds(cmd - .SetCommandAndParameters(@"SELECT ""Timestamp"" FROM ""Fileset"" WHERE ""ID"" = @FilesetId") - .SetParameterValue("@FilesetId", filesetId) - .ExecuteScalarInt64(0)); - - var ix = FilesetTimes.Select((value, index) => new { value.Key, index }) - .Where(n => n.Key == filesetId) - .Select(pair => pair.index + 1) - .FirstOrDefault() - 1; - - Logging.Log.WriteInformationMessage(LOGTAG, "SearchingBackup", "Searching backup {0} ({1}) ...", ix, m_restoreTime); - - cmd.Parameters.Clear(); - - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tempfiletable}"" ")); - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tempblocktable}"" ")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tempfiletable}"" (""ID"" INTEGER PRIMARY KEY, ""Path"" TEXT NOT NULL, ""BlocksetID"" INTEGER NOT NULL, ""MetadataID"" INTEGER NOT NULL, ""TargetPath"" TEXT NULL, ""DataVerified"" BOOLEAN NOT NULL, ""LatestBlocksetId"" INTEGER, ""LocalSourceExists"" BOOLEAN) ")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tempblocktable}"" (""ID"" INTEGER PRIMARY KEY, ""FileID"" INTEGER NOT NULL, ""Index"" INTEGER NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" BOOLEAN NOT NULL, ""Metadata"" BOOLEAN NOT NULL, ""VolumeID"" INTEGER NOT NULL, ""BlockID"" INTEGER NOT NULL)")); - - // TODO: Optimize to use the path prefix - - if (filter == null || filter.Empty) - { - // Simple case, restore everything - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_tempfiletable}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") SELECT ""File"".""ID"", ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"", 0 FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = @FilesetId")) - .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQuery(); - } - else if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == FilterType.Simple) - { - // If we get a list of filenames, the lookup table is faster - // unfortunately we cannot do this if the filesystem is case sensitive as - // SQLite only supports ASCII compares - using (var tr = m_connection.BeginTransactionSafe()) - { - var p = expression.GetSimpleList(); - var m_filenamestable = "Filenames-" + m_temptabsetguid; - cmd.Transaction = tr; - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_filenamestable}"" (""Path"" TEXT NOT NULL) ")); - cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_filenamestable}"" (""Path"") VALUES (@Path)")); - - foreach (var s in p) - { - cmd.SetParameterValue("@Path", s) - .ExecuteNonQuery(); - } - - var c = cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_tempfiletable}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") SELECT ""File"".""ID"", ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"", 0 FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetEntry"".""FilesetID"" = @FilesetId AND ""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{m_filenamestable}"") ")) - .SetParameterValue("@FilesetId", filesetId) - .ExecuteNonQuery(); - - cmd.Parameters.Clear(); - - if (c != p.Length && c != 0) - { - var sb = new StringBuilder(); - sb.AppendLine(); - - using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT ""Path"" FROM ""{m_filenamestable}"" WHERE ""Path"" NOT IN (SELECT ""Path"" FROM ""{m_tempfiletable}"")"))) - while (rd.Read()) - sb.AppendLine(rd.ConvertValueToString(0)); - - var actualrestoretime = ParseFromEpochSeconds( - cmd.SetCommandAndParameters(@"SELECT ""Timestamp"" FROM ""Fileset"" WHERE ""ID"" = @FilesetId") - .SetParameterValue("@FilesetId", filesetId) - .ExecuteScalarInt64(0)); - - Logging.Log.WriteWarningMessage(LOGTAG, "FilesNotFoundInBackupList", null, "{0} File(s) were not found in list of files for backup at {1}, will not be restored: {2}", p.Length - c, actualrestoretime.ToLocalTime(), sb); - cmd.Parameters.Clear(); - } - - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_filenamestable}"" ")); - - using (new Logging.Timer(LOGTAG, "CommitPrepareFileset", "CommitPrepareFileset")) - tr.Commit(); - } - } - else - { - // Restore but filter elements based on the filter expression - // If this is too slow, we could add a special handler for wildcard searches too - cmd.SetCommandAndParameters(@"SELECT ""File"".""ID"", ""File"".""Path"", ""File"".""BlocksetID"", ""File"".""MetadataID"" FROM ""File"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FilesetID"" = @FilesetId") - .SetParameterValue("@FilesetId", filesetId); - - object[] values = new object[4]; - using (var cmd2 = m_connection.CreateCommand(FormatInvariant($@"INSERT INTO ""{m_tempfiletable}"" (""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""DataVerified"") VALUES (@ID, @Path, @BlocksetID, @MetadataID, 0)"))) - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - { - rd.GetValues(values); - var path = values[1] as string; - if (path != null && FilterExpression.Matches(filter, path.ToString()!)) - { - cmd2.SetParameterValue("@ID", values[0]) - .SetParameterValue("@Path", values[1]) - .SetParameterValue("@BlocksetID", values[2]) - .SetParameterValue("@MetadataID", values[3]) - .ExecuteNonQuery(); - } - } - } - - //creating indexes after insertion is much faster - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tempfiletable}_ID"" ON ""{m_tempfiletable}"" (""ID"")")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tempfiletable}_TargetPath"" ON ""{m_tempfiletable}"" (""TargetPath"")")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tempfiletable}_Path"" ON ""{m_tempfiletable}"" (""Path"")")); - - using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT COUNT(DISTINCT ""{m_tempfiletable}"".""Path""), SUM(""Blockset"".""Length"") FROM ""{m_tempfiletable}"", ""Blockset"" WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" "))) - { - var filecount = 0L; - var filesize = 0L; - - if (rd.Read()) - { - filecount = rd.ConvertValueToInt64(0, 0); - filesize = rd.ConvertValueToInt64(1, 0); - } - - if (filecount > 0) - { - Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreTargetFileCount", "Needs to restore {0} files ({1})", filecount, Library.Utility.Utility.FormatSizeString(filesize)); - return new Tuple(filecount, filesize); - } - } - } - } - - return new Tuple(0, 0); - } - - public string? GetFirstPath() - { - using (var cmd = m_connection.CreateCommand()) - { - var v0 = cmd.ExecuteScalar(FormatInvariant($@"SELECT ""Path"" FROM ""{m_tempfiletable}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1")); - if (v0 == null || v0 == DBNull.Value) - return null; - - return v0.ToString(); - } - } - - public string GetLargestPrefix() - { - using (var cmd = m_connection.CreateCommand()) - { - var v0 = cmd.ExecuteScalar(FormatInvariant($@"SELECT ""Path"" FROM ""{m_tempfiletable}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1")); - var maxpath = ""; - if (v0 != null && v0 != DBNull.Value) - maxpath = v0.ToString()!; - - var dirsep = Util.GuessDirSeparator(maxpath); - - var filecount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""{m_tempfiletable}"""), -1); - var foundfiles = -1L; - - //TODO: Handle FS case-sensitive? - cmd.SetCommandAndParameters(FormatInvariant($@"SELECT COUNT(*) FROM ""{m_tempfiletable}"" WHERE SUBSTR(""Path"", 1, @PrefixLength) = @Prefix")); - - while (filecount != foundfiles && maxpath.Length > 0) - { - var mp = Util.AppendDirSeparator(maxpath, dirsep); - foundfiles = cmd.SetParameterValue("@PrefixLength", mp.Length) - .SetParameterValue("@Prefix", mp) - .ExecuteScalarInt64(-1); - - if (filecount != foundfiles) - { - var oldlen = maxpath.Length; - - var lix = maxpath.LastIndexOf(dirsep, maxpath.Length - 2, StringComparison.Ordinal); - maxpath = maxpath.Substring(0, lix + 1); - if (string.IsNullOrWhiteSpace(maxpath) || maxpath.Length == oldlen) - maxpath = ""; - } - } - - return maxpath == "" ? "" : Util.AppendDirSeparator(maxpath, dirsep); - } - } - - public void SetTargetPaths(string largest_prefix, string destination) - { - var dirsep = Util.GuessDirSeparator(string.IsNullOrWhiteSpace(largest_prefix) ? GetFirstPath() : largest_prefix); - - using (var cmd = m_connection.CreateCommand()) - { - if (string.IsNullOrEmpty(destination)) - { - //The string fixing here is meant to provide some non-random - // defaults when restoring cross OS, e.g. backup on Linux, restore on Windows - //This is mostly meaningless, and the user really should use --restore-path - - if ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) && dirsep == "\\") - { - // For Win -> Linux, we remove the colon from the drive letter, and use the drive letter as root folder - cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 2, 1) == ':' THEN '\\' || SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) ELSE ""Path"" END")); - cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 1, 2) == '\\' THEN '\\' || SUBSTR(""Path"", 2) ELSE ""Path"" END")); - - } - else if (OperatingSystem.IsWindows() && dirsep == "/") - { - // For Linux -> Win, we use the temporary folder's drive as the root path - cmd.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""Targetpath"" = CASE WHEN SUBSTR(""Path"", 1, 1) == '/' THEN @Path || SUBSTR(""Path"", 2) ELSE ""Path"" END")) - .SetParameterValue("@Path", Util.AppendDirSeparator(System.IO.Path.GetPathRoot(Library.Utility.TempFolder.SystemTempPath)).Replace("\\", "/")) - .ExecuteNonQuery(); - } - else - { - // Same OS, just use the path directly - cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""Targetpath"" = ""Path"" ")); - } - } - else - { - if (string.IsNullOrEmpty(largest_prefix)) - { - //Special case, restoring to new folder, but files are from different drives (no shared root on Windows) - - // We use the format / / - cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = CASE WHEN SUBSTR(""Path"", 2, 1) == ':' THEN SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) ELSE ""Path"" END")); - - // For UNC paths, we use \\server\folder -> / / - cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = CASE WHEN SUBSTR(""Path"", 1, 2) == '\\' THEN SUBSTR(""Path"", 2) ELSE ""TargetPath"" END")); - } - else - { - largest_prefix = Util.AppendDirSeparator(largest_prefix, dirsep); - cmd.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = SUBSTR(""Path"", @PrefixLength)")) - .SetParameterValue("@PrefixLength", largest_prefix.Length + 1) - .ExecuteNonQuery(); - } - } - - // Cross-os path remapping support - if ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) && dirsep == "\\") - // For Win paths on Linux - cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = REPLACE(""TargetPath"", '\', '/')")); - else if (OperatingSystem.IsWindows() && dirsep == "/") - // For Linux paths on Windows - cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = REPLACE(REPLACE(""TargetPath"", '\', '_'), '/', '\')")); - - if (!string.IsNullOrEmpty(destination)) - { - // Paths are now relative with target-os naming system - // so we prefix them with the target path - cmd.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = @Destination || ""TargetPath"" ")) - .SetParameterValue("@Destination", Util.AppendDirSeparator(destination)) - .ExecuteNonQuery(); - } - } - } - - public void FindMissingBlocks(bool skipMetadata) - { - using (var cmd = m_connection.CreateCommand()) - { - var p1 = cmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""{m_tempblocktable}"" (""FileID"", ""Index"", ""Hash"", ""Size"", ""Restored"", ""Metadata"", ""VolumeId"", ""BlockId"") SELECT DISTINCT ""{m_tempfiletable}"".""ID"", ""BlocksetEntry"".""Index"", ""Block"".""Hash"", ""Block"".""Size"", 0, 0, ""Block"".""VolumeID"", ""Block"".""ID"" FROM ""{m_tempfiletable}"", ""BlocksetEntry"", ""Block"" WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ")); - - var p2 = 0; - if (!skipMetadata) - p2 = cmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""{m_tempblocktable}"" (""FileID"", ""Index"", ""Hash"", ""Size"", ""Restored"", ""Metadata"", ""VolumeId"", ""BlockId"") SELECT DISTINCT ""{m_tempfiletable}"".""ID"", ""BlocksetEntry"".""Index"", ""Block"".""Hash"", ""Block"".""Size"", 0, 1, ""Block"".""VolumeID"", ""Block"".""ID"" FROM ""{m_tempfiletable}"", ""BlocksetEntry"", ""Block"", ""Metadataset"" WHERE ""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ")); - - //creating indexes after insertion is much faster - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tempblocktable}_HashSizeIndex"" ON ""{m_tempblocktable}"" (""Hash"", ""Size"")")); - // better suited to speed up commit on UpdateBlocks - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tempblocktable}_FileIdIndexIndex"" ON ""{m_tempblocktable}"" (""FileId"", ""Index"")")); - - var size = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT SUM(""Size"") FROM ""{m_tempblocktable}"" "), 0); - Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreSourceSize", "Restore list contains {0} blocks with a total size of {1}", p1 + p2, Library.Utility.Utility.FormatSizeString(size)); - } - } - - public void UpdateTargetPath(long ID, string newname) - { - using var cmd = m_connection.CreateCommand(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""TargetPath"" = @TargetPath WHERE ""ID"" = @ID")) - .SetParameterValue("@TargetPath", newname) - .SetParameterValue("@ID", ID); - - cmd.ExecuteNonQuery(); - } - - public interface IExistingFileBlock - { - string Hash { get; } - long Index { get; } - long Size { get; } - } - - public interface IExistingFile - { - string TargetPath { get; } - string TargetHash { get; } - long TargetFileID { get; } - long Length { get; } - IEnumerable Blocks { get; } - } - - public interface IBlockSource - { - string Path { get; } - long Offset { get; } - bool IsMetadata { get; } - } - - public interface IBlockDescriptor - { - string Hash { get; } - long Size { get; } - long Offset { get; } - long Index { get; } - bool IsMetadata { get; } - IEnumerable Blocksources { get; } - } - - public interface ILocalBlockSource - { - string TargetPath { get; } - long TargetFileID { get; } - IEnumerable Blocks { get; } - } - - /// - /// Interface for an object describing a file to restore. - /// - public interface IFileToRestore - { - string Path { get; } - string Hash { get; } - long Length { get; } - } - - public interface IPatchBlock - { - long Offset { get; } - long Size { get; } - string Key { get; } - } - - public interface IVolumePatch - { - string Path { get; } - long FileID { get; } - IEnumerable Blocks { get; } - } - - private class ExistingFile : IExistingFile - { - private readonly IDataReader m_reader; - - public ExistingFile(IDataReader rd) { m_reader = rd; HasMore = true; } - - public string TargetPath { get { return m_reader.ConvertValueToString(0) ?? ""; } } - public string TargetHash { get { return m_reader.ConvertValueToString(1) ?? ""; } } - public long TargetFileID { get { return m_reader.ConvertValueToInt64(2); } } - public long Length { get { return m_reader.ConvertValueToInt64(3); } } - - public bool HasMore { get; private set; } - - private class ExistingFileBlock : IExistingFileBlock - { - private readonly IDataReader m_reader; - - public ExistingFileBlock(IDataReader rd) { m_reader = rd; } - - public string Hash { get { return m_reader.ConvertValueToString(4) ?? ""; } } - public long Index { get { return m_reader.ConvertValueToInt64(5); } } - public long Size { get { return m_reader.ConvertValueToInt64(6); } } - } - - public IEnumerable Blocks - { - get - { - string p = TargetPath; - while (HasMore && p == TargetPath) - { - yield return new ExistingFileBlock(m_reader); - HasMore = m_reader.Read(); - } - } - } - - public static IEnumerable GetExistingFilesWithBlocks(IDbConnection connection, string tablename) - { - using (var cmd = connection.CreateCommand(FormatInvariant($@"SELECT ""{tablename}"".""TargetPath"", ""Blockset"".""FullHash"", ""{tablename}"".""ID"", ""Blockset"".""Length"", ""Block"".""Hash"", ""BlocksetEntry"".""Index"", ""Block"".""Size"" FROM ""{tablename}"", ""Blockset"", ""BlocksetEntry"", ""Block"" WHERE ""{tablename}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""{tablename}"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ORDER BY ""{tablename}"".""TargetPath"", ""BlocksetEntry"".""Index"""))) - using (var rd = cmd.ExecuteReader()) - if (rd.Read()) - { - var more = true; - while (more) - { - var f = new ExistingFile(rd); - string current = f.TargetPath; - yield return f; - - more = f.HasMore; - while (more && current == f.TargetPath) - more = rd.Read(); - } - } - } - } - - public IEnumerable GetExistingFilesWithBlocks() - { - if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) - throw new InvalidOperationException("No temporary file table set up for this restore."); - return ExistingFile.GetExistingFilesWithBlocks(m_connection, m_tempfiletable); - } - - private class LocalBlockSource : ILocalBlockSource - { - private class BlockDescriptor : IBlockDescriptor - { - private class BlockSource : IBlockSource - { - private readonly IDataReader m_reader; - public BlockSource(IDataReader rd) { m_reader = rd; } - - public string Path { get { return m_reader.ConvertValueToString(6) ?? ""; } } - public long Offset { get { return m_reader.ConvertValueToInt64(7); } } - public bool IsMetadata { get { return false; } } - } - - private readonly IDataReader m_reader; - public BlockDescriptor(IDataReader rd) { m_reader = rd; HasMore = true; } - - private string TargetPath { get { return m_reader.ConvertValueToString(0) ?? ""; } } - - public string Hash { get { return m_reader.ConvertValueToString(2) ?? ""; } } - public long Offset { get { return m_reader.ConvertValueToInt64(3); } } - public long Index { get { return m_reader.ConvertValueToInt64(4); } } - public long Size { get { return m_reader.ConvertValueToInt64(5); } } - public bool IsMetadata { get { return !(m_reader.ConvertValueToInt64(9) == 0); } } - - public bool HasMore { get; private set; } - - public IEnumerable Blocksources - { - get - { - var p = TargetPath; - var h = Hash; - var s = Size; - - while (HasMore && p == TargetPath && h == Hash && s == Size) - { - yield return new BlockSource(m_reader); - HasMore = m_reader.Read(); - } - } - } - } - - private readonly IDataReader m_reader; - public LocalBlockSource(IDataReader rd) { m_reader = rd; HasMore = true; } - - public string TargetPath { get { return m_reader.ConvertValueToString(0) ?? ""; } } - public long TargetFileID { get { return m_reader.ConvertValueToInt64(1); } } - - public bool HasMore { get; private set; } - - public IEnumerable Blocks - { - get - { - var p = TargetPath; - while (HasMore && p == TargetPath) - { - var c = new BlockDescriptor(m_reader); - var h = c.Hash; - var s = c.Size; - - yield return c; - - HasMore = c.HasMore; - while (HasMore && c.Hash == h && c.Size == s && TargetPath == p) - HasMore = m_reader.Read(); - } - } - } - - public static IEnumerable GetFilesAndSourceBlocks(IDbConnection connection, string filetablename, string blocktablename, long blocksize, bool skipMetadata) - { - // TODO: Skip metadata as required - // Have to order by target path and hash, to ensure BlockDescriptor and BlockSource match adjacent rows - using (var cmd = connection.CreateCommand(FormatInvariant($@"SELECT DISTINCT ""A"".""TargetPath"", ""A"".""ID"", ""B"".""Hash"", (""B"".""Index"" * {blocksize}), ""B"".""Index"", ""B"".""Size"", ""C"".""Path"", (""D"".""Index"" * {blocksize}), ""E"".""Size"", ""B"".""Metadata"" FROM ""{filetablename}"" ""A"", ""{blocktablename}"" ""B"", ""File"" ""C"", ""BlocksetEntry"" ""D"", ""Block"" E WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""C"".""BlocksetID"" = ""D"".""BlocksetID"" AND ""D"".""BlockID"" = ""E"".""ID"" AND ""B"".""Hash"" = ""E"".""Hash"" AND ""B"".""Size"" = ""E"".""Size"" AND ""B"".""Restored"" = 0 ORDER BY ""A"".""TargetPath"", ""B"".""Index"""))) - using (var rd = cmd.ExecuteReader()) - { - var more = rd.Read(); - while (more) - { - var f = new LocalBlockSource(rd); - string current = f.TargetPath; - yield return f; - - more = f.HasMore; - while (more && current == f.TargetPath) - more = rd.Read(); - } - } - } - } - - public IEnumerable GetFilesAndSourceBlocks(bool skipMetadata, long blocksize) - { - if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) - throw new InvalidOperationException("No temporary file table set up for this restore."); - return LocalBlockSource.GetFilesAndSourceBlocks(m_connection, m_tempfiletable, m_tempblocktable, blocksize, skipMetadata); - } - - public IEnumerable GetMissingVolumes() - { - using (var cmd = m_connection.CreateCommand()) - { - // Return order from SQLite-DISTINCT is likely to be sorted by Name, which is bad for restore. - // If the end of very large files (e.g. iso's) is restored before the beginning, most OS write out zeros to fill the file. - // If we manage to get the volumes in an order restoring front blocks first, this can save time. - // An optimal algorithm would build a dependency net with cycle resolution to find the best near topological - // order of volumes, but this is a bit too fancy here. - // We will just put a very simple heuristic to work, that will try to prefer volumes containing lower block indexes: - // We just order all volumes by the maximum block index they contain. This query is slow, but should be worth the effort. - // Now it is likely to restore all files from front to back. Large files will always be done last. - // One could also use like the average block number in a volume, that needs to be measured. - - cmd.SetCommandAndParameters(FormatInvariant($@"SELECT ""RV"".""Name"", ""RV"".""Hash"", ""RV"".""Size"", ""BB"".""MaxIndex"" -FROM ""RemoteVolume"" ""RV"" INNER JOIN - (SELECT ""TB"".""VolumeID"", MAX(""TB"".""Index"") as ""MaxIndex"" - FROM ""{m_tempblocktable}"" ""TB"" - WHERE ""TB"".""Restored"" = 0 - GROUP BY ""TB"".""VolumeID"" - ) as ""BB"" ON ""RV"".""ID"" = ""BB"".""VolumeID"" -ORDER BY ""BB"".""MaxIndex"" ")); - - using (var rd = cmd.ExecuteReader()) - { - object[] r = new object[3]; - while (rd.Read()) - { - rd.GetValues(r); - yield return new RemoteVolume( - rd.ConvertValueToString(0), - rd.ConvertValueToString(1), - rd.ConvertValueToInt64(2, -1) - ); - } - } - } - } - - public interface IFilesAndMetadata : IDisposable - { - IEnumerable FilesWithMissingBlocks { get; } - IEnumerable MetadataWithMissingBlocks { get; } - } - - private class FilesAndMetadata : IFilesAndMetadata - { - private readonly string m_tmptable; - private readonly string m_filetablename; - private readonly string m_blocktablename; - private readonly long m_blocksize; - - private readonly IDbConnection m_connection; - - public FilesAndMetadata(IDbConnection connection, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume) - { - m_filetablename = filetablename; - m_blocktablename = blocktablename; - m_blocksize = blocksize; - m_connection = connection; - - using (var c = m_connection.CreateCommand()) - { - m_tmptable = "VolumeFiles-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - c.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tmptable}"" ( ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL )")); - - c.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_tmptable}"" (""Hash"", ""Size"") VALUES (@Hash, @Size)")); - foreach (var s in curvolume.Blocks) - { - c.SetParameterValue("@Hash", s.Key); - c.SetParameterValue("@Size", s.Value); - c.ExecuteNonQuery(); - } - - // The index _HashSizeIndex is not needed anymore. Index on "Blocks-..." is used on Join in GetMissingBlocks - } - } - - public void Dispose() - { - if (m_tmptable != null) - using (var c = m_connection.CreateCommand()) - c.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tmptable}""")); - } - - private class VolumePatch : IVolumePatch - { - private class PatchBlock : IPatchBlock - { - private readonly IDataReader m_reader; - public PatchBlock(IDataReader rd) { m_reader = rd; } - - public long Offset { get { return m_reader.ConvertValueToInt64(2); } } - public long Size { get { return m_reader.ConvertValueToInt64(3); } } - public string Key { get { return m_reader.ConvertValueToString(4) ?? ""; } } - } - - private readonly IDataReader m_reader; - public VolumePatch(IDataReader rd) { m_reader = rd; HasMore = true; } - - public string Path { get { return m_reader.ConvertValueToString(0) ?? ""; } } - public long FileID { get { return m_reader.ConvertValueToInt64(1); } } - public bool HasMore { get; private set; } - - public IEnumerable Blocks - { - get - { - string p = Path; - while (HasMore && p == Path) - { - yield return new PatchBlock(m_reader); - HasMore = m_reader.Read(); - } - } - } - } - - public IEnumerable FilesWithMissingBlocks - { - get - { - using (var cmd = m_connection.CreateCommand()) - { - // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! - cmd.SetCommandAndParameters(FormatInvariant($@" SELECT DISTINCT ""A"".""TargetPath"", ""BB"".""FileID"", (""BB"".""Index"" * {m_blocksize}), ""BB"".""Size"", ""BB"".""Hash"" -FROM ""{m_filetablename}"" ""A"", ""{m_blocktablename}"" ""BB"" -WHERE ""A"".""ID"" = ""BB"".""FileID"" AND ""BB"".""Restored"" = 0 AND ""BB"".""Metadata"" = {"0"} -AND ""BB"".""ID"" IN (SELECT ""B"".""ID"" FROM ""{m_blocktablename}"" ""B"", ""{m_tmptable}"" ""C"" WHERE ""B"".""Hash"" = ""C"".""Hash"" AND ""B"".""Size"" = ""C"".""Size"") -ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); - using (var rd = cmd.ExecuteReader()) - { - if (rd.Read()) - { - var more = true; - while (more) - { - var f = new VolumePatch(rd); - var current = f.Path; - yield return f; - - more = f.HasMore; - while (more && current == f.Path) - more = rd.Read(); - } - } - } - } - } - } - - public IEnumerable MetadataWithMissingBlocks - { - get - { - using (var cmd = m_connection.CreateCommand()) - { - // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! - cmd.SetCommandAndParameters(FormatInvariant($@" SELECT DISTINCT ""A"".""TargetPath"", ""BB"".""FileID"", (""BB"".""Index"" * {m_blocksize}), ""BB"".""Size"", ""BB"".""Hash"" - FROM ""{m_filetablename}"" ""A"", ""{m_blocktablename}"" ""BB"" -WHERE ""A"".""ID"" = ""BB"".""FileID"" AND ""BB"".""Restored"" = 0 AND ""BB"".""Metadata"" = {"1"} - AND ""BB"".""ID"" IN (SELECT ""B"".""ID"" FROM ""{m_blocktablename}"" ""B"", ""{m_tmptable}"" ""C"" WHERE ""B"".""Hash"" = ""C"".""Hash"" AND ""B"".""Size"" = ""C"".""Size"") -ORDER BY ""A"".""TargetPath"", ""BB"".""Index""")); - using (var rd = cmd.ExecuteReader()) - { - if (rd.Read()) - { - var more = true; - while (more) - { - var f = new VolumePatch(rd); - string current = f.Path; - yield return f; - - more = f.HasMore; - while (more && current == f.Path) - more = rd.Read(); - } - } - } - } - } - } - } - - public IFilesAndMetadata GetMissingBlockData(BlockVolumeReader curvolume, long blocksize) - { - if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) - throw new InvalidOperationException("No temporary file table set up for this restore."); - return new FilesAndMetadata(m_connection, m_tempfiletable, m_tempblocktable, blocksize, curvolume); - } - - /// - /// Returns a connection from the connection pool. - /// - /// A connection from the connection pool. - public IDbConnection GetConnectionFromPool() - { - if (!m_connection_pool.TryTake(out var connection)) - { - connection = SQLiteHelper.SQLiteLoader.LoadConnection(); - connection.ConnectionString = m_connection.ConnectionString + ";Cache=Shared;"; - connection.Open(); - - SQLiteHelper.SQLiteLoader.ApplyCustomPragmas(connection, m_pagecachesize); - - using var cmd = connection.CreateCommand(); - cmd.ExecuteNonQuery("PRAGMA journal_mode = WAL;"); - cmd.ExecuteNonQuery("PRAGMA read_uncommitted = true;"); - } - - return connection; - } - - private class FileToRestore : IFileToRestore - { - public string Path { get; private set; } - public string Hash { get; private set; } - public long Length { get; private set; } - - public FileToRestore(long id, string path, string hash, long length) - { - Path = path; - Hash = hash; - Length = length; - } - } - - public IEnumerable GetFilesToRestore(bool onlyNonVerified) - { - using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT ""{m_tempfiletable}"".""ID"", ""{m_tempfiletable}"".""TargetPath"", ""Blockset"".""FullHash"", ""Blockset"".""Length"" FROM ""{m_tempfiletable}"",""Blockset"" WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""{m_tempfiletable}"".""DataVerified"" <= @Verified")) - .SetParameterValue("@Verified", !onlyNonVerified); - - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - yield return new FileToRestore( - rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? "", rd.ConvertValueToString(2) ?? "", rd.ConvertValueToInt64(3)); - } - - /// - /// Returns a list of files and symlinks to restore. - /// - /// A list of files and symlinks to restore. - public IEnumerable GetFilesAndSymlinksToRestore() - { - // Order by length descending, so that larger files are restored first. - using var cmd = m_connection.CreateCommand(FormatInvariant($@" - SELECT F.ID, F.Path, F.TargetPath, IFNULL(B.FullHash, ''), IFNULL(B.Length, 0) AS ""Length"", F.BlocksetID - FROM ""{m_tempfiletable}"" F - LEFT JOIN Blockset B ON F.BlocksetID = B.ID - WHERE F.BlocksetID != {FOLDER_BLOCKSET_ID} - ORDER BY ""Length"" DESC")); - using var rd = cmd.ExecuteReader(); - while (rd.Read()) - yield return new FileRequest(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), rd.ConvertValueToString(2), rd.ConvertValueToString(3), rd.ConvertValueToInt64(4), rd.ConvertValueToInt64(5)); - } - - /// - /// Returns a list of folders to restore. Used to restore folder metadata. - /// - /// A list of folders to restore. - public IEnumerable GetFolderMetadataToRestore() - { - using var cmd = m_connection.CreateCommand(); - using var rd = cmd.ExecuteReader(FormatInvariant($@" - SELECT F.ID, '', F.TargetPath, '', 0, {FOLDER_BLOCKSET_ID} - FROM ""{m_tempfiletable}"" F - WHERE F.BlocksetID = {FOLDER_BLOCKSET_ID} AND F.MetadataID IS NOT NULL AND F.MetadataID >= 0")); - - while (rd.Read()) - yield return new FileRequest(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1), rd.ConvertValueToString(2), rd.ConvertValueToString(3), rd.ConvertValueToInt64(4), rd.ConvertValueToInt64(5)); - } - - /// - /// Returns a list of blocks and their volume IDs. Used by the to keep track of blocks and volumes to automatically evict them from the respective caches. - /// - /// Flag indicating whether the returned blocks should exclude the metadata blocks. - /// A list of tuples containing the block ID and the volume ID of the block. - public IEnumerable<(long, long)> GetBlocksAndVolumeIDs(bool skipMetadata) - { - using var cmd = Connection.CreateCommand(); - using var reader = cmd.ExecuteReader(FormatInvariant($@" - SELECT ""Block"".""ID"", ""Block"".""VolumeID"" - FROM ""BlocksetEntry"" - INNER JOIN ""{m_tempfiletable}"" ON ""BlocksetEntry"".""BlocksetID"" = ""{m_tempfiletable}"".""BlocksetID"" - INNER JOIN ""Block"" ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - ") - + (skipMetadata ? "" : FormatInvariant($@" - UNION ALL - SELECT ""Block"".""ID"", ""Block"".""VolumeID"" - FROM ""{m_tempfiletable}"" - INNER JOIN ""Metadataset"" ON ""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID"" - INNER JOIN ""BlocksetEntry"" ON ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" - INNER JOIN ""Block"" ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - "))); - while (reader.Read()) - yield return (reader.ConvertValueToInt64(0), reader.ConvertValueToInt64(1)); - } - - /// - /// Returns a list of for the given blockset ID. It is used by the to restore the blocks of a file. - /// - /// The BlocksetID of the file. - /// A list of needed to restore the given file. - public IEnumerable GetBlocksFromFile(long blocksetID) - { - var connection = GetConnectionFromPool(); - using var cmd = connection.CreateCommand(FormatInvariant(@$" - SELECT ""Block"".""ID"", ""Block"".""Hash"", ""Block"".""Size"", ""Block"".""VolumeID"" - FROM ""BlocksetEntry"" INNER JOIN ""Block"" - ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - WHERE ""BlocksetEntry"".""BlocksetID"" = @BlocksetID")) - .SetParameterValue("@BlocksetID", blocksetID); - - using var reader = cmd.ExecuteReader(); - for (long i = 0; reader.Read(); i++) - yield return new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false); - - // Return the connection to the pool - m_connection_pool.Add(connection); - } - - /// - /// Returns a list of for the metadata blocks of the given file. It is used by the to restore the metadata of a file. - /// - /// The ID of the file. - /// A list of needed to restore the metadata of the given file. - public IEnumerable GetMetadataBlocksFromFile(long fileID) - { - var connection = GetConnectionFromPool(); - using var cmd = connection.CreateCommand(FormatInvariant($@" - SELECT ""Block"".""ID"", ""Block"".""Hash"", ""Block"".""Size"", ""Block"".""VolumeID"" - FROM ""File"" - INNER JOIN ""Metadataset"" ON ""File"".""MetadataID"" = ""Metadataset"".""ID"" - INNER JOIN ""BlocksetEntry"" ON ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" - INNER JOIN ""Block"" ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" - WHERE ""File"".""ID"" = @FileID - ")) - .SetParameterValue("@FileID", fileID); - - using var reader = cmd.ExecuteReader(); - for (long i = 0; reader.Read(); i++) - { - yield return new BlockRequest(reader.ConvertValueToInt64(0), i, reader.ConvertValueToString(1), reader.ConvertValueToInt64(2), reader.ConvertValueToInt64(3), false); - } - - // Return the connection to the pool - m_connection_pool.Add(connection); - } - - /// - /// Returns the volume information for the given volume ID. It is used by the to get the volume information for the given volume ID. - /// - /// The ID of the volume. - /// A tuple containing the name, size, and hash of the volume. - public IEnumerable<(string, long, string)> GetVolumeInfo(long VolumeID) - { - using var cmd = m_connection.CreateCommand("SELECT Name, Size, Hash FROM RemoteVolume WHERE ID = @VolumeID") - .SetParameterValue("@VolumeID", VolumeID); - using var reader = cmd.ExecuteReader(); - while (reader.Read()) - yield return (reader.ConvertValueToString(0) ?? "", reader.ConvertValueToInt64(1), reader.ConvertValueToString(2) ?? ""); - } - - public void DropRestoreTable() - { - using (var cmd = m_connection.CreateCommand()) - { - if (m_tempfiletable != null) - try - { - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tempfiletable}""")); - } - catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } - finally { m_tempfiletable = null; } - - if (m_tempblocktable != null) - try - { - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tempblocktable}""")); - } - catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } - finally { m_tempblocktable = null; } - - if (m_latestblocktable != null) - try - { - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_latestblocktable}""")); - } - catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } - finally { m_latestblocktable = null; } - - if (m_fileprogtable != null) - try - { - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_fileprogtable}""")); - } - catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } - finally { m_fileprogtable = null; } - - if (m_totalprogtable != null) - try - { - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_totalprogtable}""")); - } - catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } - finally { m_totalprogtable = null; } - - if (m_filesnewlydonetable != null) - try - { - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}""")); - } - catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } - finally { m_filesnewlydonetable = null; } - - } - } - - public interface IBlockMarker : IDisposable - { - void SetBlockRestored(long targetfileid, long index, string hash, long blocksize, bool metadata); - void SetAllBlocksMissing(long targetfileid); - void SetAllBlocksRestored(long targetfileid, bool includeMetadata); - void SetFileDataVerified(long targetfileid); - void Commit(); - void UpdateProcessed(IOperationProgressUpdater writer); - } - - /// - /// A new implementation of IBlockMarker, marking the blocks directly in the blocks table as restored - /// and reading statistics about progress from DB (kept up-to-date by triggers). - /// There is no negative influence on performance, esp. since the block table is temporary anyway. - /// - private class DirectBlockMarker : IBlockMarker - { - private IDbCommand m_insertblockCommand; - private IDbCommand m_resetfileCommand; - private IDbCommand m_updateAsRestoredCommand; - private IDbCommand m_updateFileAsDataVerifiedCommand; - private IDbCommand m_statUpdateCommand; - private IDbTransaction m_transaction; - private bool m_hasUpdates = false; - - private readonly string m_blocktablename; - private readonly string m_filetablename; - - public DirectBlockMarker(IDbConnection connection, string blocktablename, string filetablename, string statstablename) - { - m_transaction = connection.BeginTransactionSafe(); - m_blocktablename = blocktablename; - m_filetablename = filetablename; - - m_insertblockCommand = connection.CreateCommand(m_transaction, FormatInvariant($@"UPDATE ""{m_blocktablename}"" SET ""Restored"" = 1 -WHERE ""FileID"" = @TargetFileId AND ""Index"" = @Index AND ""Hash"" = @Hash AND ""Size"" = @Size AND ""Metadata"" = @Metadata AND ""Restored"" = 0 ")); - - m_resetfileCommand = connection.CreateCommand(m_transaction, FormatInvariant($@"UPDATE ""{m_blocktablename}"" SET ""Restored"" = 0 WHERE ""FileID"" = @TargetFileId ")); - m_updateAsRestoredCommand = connection.CreateCommand(m_transaction, FormatInvariant($@"UPDATE ""{m_blocktablename}"" SET ""Restored"" = 1 WHERE ""FileID"" = @TargetFileId AND ""Metadata"" <= @Metadata ")); - m_updateFileAsDataVerifiedCommand = connection.CreateCommand(m_transaction, FormatInvariant($@"UPDATE ""{m_filetablename}"" SET ""DataVerified"" = 1 WHERE ""ID"" = @TargetFileId")); - - if (statstablename != null) - { - // Fields in Stats: TotalFiles, TotalBlocks, TotalSize - // FilesFullyRestored, FilesPartiallyRestored, BlocksRestored, SizeRestored - m_statUpdateCommand = connection.CreateCommand(m_transaction, FormatInvariant($@"SELECT SUM(""FilesFullyRestored""), SUM(""SizeRestored"") FROM ""{statstablename}"" ")); - } - else // very slow fallback if stats tables were not created - m_statUpdateCommand = connection.CreateCommand(m_transaction, FormatInvariant($@"SELECT COUNT(DISTINCT ""FileID""), SUM(""Size"") FROM ""{m_blocktablename}"" WHERE ""Restored"" = 1 ")); - - } - - public void UpdateProcessed(IOperationProgressUpdater updater) - { - if (!m_hasUpdates) - return; - - m_hasUpdates = false; - using (var rd = m_statUpdateCommand.ExecuteReader()) - { - var filesprocessed = 0L; - var processedsize = 0L; - - if (rd.Read()) - { - filesprocessed += rd.ConvertValueToInt64(0, 0); - processedsize += rd.ConvertValueToInt64(1, 0); - } - - updater.UpdatefilesProcessed(filesprocessed, processedsize); - } - } - - public void SetAllBlocksMissing(long targetfileid) - { - m_hasUpdates = true; - var r = m_resetfileCommand.SetParameterValue("@TargetFileId", targetfileid) - .ExecuteNonQuery(); - if (r <= 0) - throw new Exception("Unexpected reset result"); - } - - public void SetAllBlocksRestored(long targetfileid, bool includeMetadata) - { - m_hasUpdates = true; - var r = m_updateAsRestoredCommand.SetParameterValue("@TargetFileId", targetfileid) - .SetParameterValue("@Metadata", includeMetadata ? 1 : 0) - .ExecuteNonQuery(); - if (r <= 0) - throw new Exception("Unexpected reset result"); - } - - public void SetFileDataVerified(long targetfileid) - { - m_hasUpdates = true; - var r = m_updateFileAsDataVerifiedCommand.SetParameterValue("@TargetFileId", targetfileid) - .ExecuteNonQuery(); - if (r != 1) - throw new Exception("Unexpected result when marking file as verified."); - } - - public void SetBlockRestored(long targetfileid, long index, string hash, long size, bool metadata) - { - m_hasUpdates = true; - var r = m_insertblockCommand.SetParameterValue("@TargetFileId", targetfileid) - .SetParameterValue("@Index", index) - .SetParameterValue("@Hash", hash) - .SetParameterValue("@Size", size) - .SetParameterValue("@Metadata", metadata) - .ExecuteNonQuery(); - if (r != 1) - throw new Exception("Unexpected result when marking block."); - } - - public void Commit() - { - m_insertblockCommand.Dispose(); - m_insertblockCommand = null!; - using (new Logging.Timer(LOGTAG, "CommitBlockMarker", "CommitBlockMarker")) - m_transaction.Commit(); - m_transaction.Dispose(); - m_transaction = null!; - } - - public void Dispose() - { - m_insertblockCommand?.Dispose(); - m_resetfileCommand?.Dispose(); - m_updateAsRestoredCommand?.Dispose(); - m_updateFileAsDataVerifiedCommand?.Dispose(); - m_statUpdateCommand?.Dispose(); - m_transaction?.Dispose(); - } - } - - public IBlockMarker CreateBlockMarker() - { - if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) - throw new InvalidOperationException("No temporary file table set up for this restore."); - if (string.IsNullOrWhiteSpace(m_totalprogtable)) - throw new InvalidOperationException("No progress table set up for this restore."); - return new DirectBlockMarker(m_connection, m_tempblocktable, m_tempfiletable, m_totalprogtable); - } - - public override void Dispose() - { - foreach (var connection in m_connection_pool) - { - connection.Close(); - connection.Dispose(); - } - m_connection_pool.Clear(); - DropRestoreTable(); - base.Dispose(); - } - - public IEnumerable GetTargetFolders() - { - using var cmd = m_connection.CreateCommand(FormatInvariant($@"SELECT ""TargetPath"" FROM ""{m_tempfiletable}"" WHERE ""BlocksetID"" == @BlocksetID")) - .SetParameterValue("@BlocksetID", FOLDER_BLOCKSET_ID); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - yield return rd.ConvertValueToString(0) ?? ""; - } - - public interface IFastSource - { - string TargetPath { get; } - long TargetFileID { get; } - string SourcePath { get; } - IEnumerable Blocks { get; } - } - - public interface IBlockEntry - { - long Offset { get; } - long Size { get; } - long Index { get; } - string Hash { get; } - } - - private class FastSource : IFastSource - { - private class BlockEntry : IBlockEntry - { - private readonly IDataReader m_rd; - private readonly long m_blocksize; - public BlockEntry(IDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; } - public long Offset { get { return m_rd.ConvertValueToInt64(3) * m_blocksize; } } - public long Index { get { return m_rd.ConvertValueToInt64(3); } } - public long Size { get { return m_rd.ConvertValueToInt64(5); } } - public string Hash { get { return m_rd.ConvertValueToString(4) ?? ""; } } - } - - private readonly IDataReader m_rd; - private readonly long m_blocksize; - public FastSource(IDataReader rd, long blocksize) { m_rd = rd; m_blocksize = blocksize; MoreData = true; } - public bool MoreData { get; private set; } - public string TargetPath { get { return m_rd.ConvertValueToString(0) ?? ""; } } - public long TargetFileID { get { return m_rd.ConvertValueToInt64(2); } } - public string SourcePath { get { return m_rd.ConvertValueToString(1) ?? ""; } } - - public IEnumerable Blocks - { - get - { - var tid = TargetFileID; - - do - { - yield return new BlockEntry(m_rd, m_blocksize); - } while ((MoreData = m_rd.Read()) && tid == TargetFileID); - - } - } - } - - public IEnumerable GetFilesAndSourceBlocksFast(long blocksize) - { - using (var transaction = m_connection.BeginTransactionSafe()) - using (var cmdReader = m_connection.CreateCommand()) - using (var cmd = m_connection.CreateCommand(transaction)) - { - cmd.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET ""LocalSourceExists"" = 1 WHERE Path = @Path")); - using (var rd = cmdReader.ExecuteReader(FormatInvariant($@"SELECT DISTINCT ""{m_tempfiletable}"".""Path"" FROM ""{m_tempfiletable}"""))) - { - while (rd.Read()) - { - var sourcepath = rd.ConvertValueToString(0); - if (SystemIO.IO_OS.FileExists(sourcepath)) - { - cmd.SetParameterValue("@Path", sourcepath); - cmd.ExecuteNonQuery(); - } - else - { - Logging.Log.WriteVerboseMessage(LOGTAG, "LocalSourceMissing", "Local source file not found: {0}", sourcepath); - } - } - } - - //This localSourceExists index will make the query engine to start by searching FileSet table. As the result is ordered by FileSet.ID, we will get the cursor "instantly" - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_tempfiletable}_LocalSourceExists"" ON ""{m_tempfiletable}"" (""LocalSourceExists"")")); - transaction.Commit(); - } - - m_latestblocktable = "LatestBlocksetIds-" + m_temptabsetguid; - - var whereclause = FormatInvariant($@" - ""{m_tempfiletable}"".""LocalSourceExists"" = 1 AND - ""{m_tempblocktable}"".""Restored"" = 0 AND ""{m_tempblocktable}"".""Metadata"" = 0 AND - ""{m_tempfiletable}"".""TargetPath"" != ""{m_tempfiletable}"".""Path"""); - - var latestBlocksetIds = FormatInvariant($@" - SELECT - ""File"".""Path"" AS ""PATH"", - ""File"".""BlocksetID"" AS ""BlocksetID"", - MAX(""Fileset"".""Timestamp"") AS ""Timestamp"" - FROM - ""File"", - ""FilesetEntry"", - ""Fileset"" - WHERE - ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND - ""FilesetEntry"".""FilesetID"" = ""Fileset"".""ID"" AND - ""File"".""Path"" IN - (SELECT DISTINCT - ""{m_tempfiletable}"".""Path"" - FROM - ""{m_tempfiletable}"", - ""{m_tempblocktable}"" - WHERE - ""{m_tempfiletable}"".""ID"" = ""{m_tempblocktable}"".""FileID"" AND - {whereclause}) - GROUP BY ""File"".""Path"""); - - using (var cmd = m_connection.CreateCommand()) - { - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_latestblocktable}"" ")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_latestblocktable}"" AS {latestBlocksetIds}")); - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE INDEX ""{m_latestblocktable}_path"" ON ""{m_latestblocktable}"" (""Path"")")); - - cmd.ExecuteNonQuery(FormatInvariant($@"UPDATE ""{m_tempfiletable}"" SET LatestBlocksetId = (SELECT BlocksetId FROM ""{m_latestblocktable}"" WHERE Path = ""{m_tempfiletable}"".Path)")); - } - - var sources = FormatInvariant($@" -SELECT DISTINCT - ""{m_tempfiletable}"".""TargetPath"", - ""{m_tempfiletable}"".""Path"", - ""{m_tempfiletable}"".""ID"", - ""{m_tempblocktable}"".""Index"", - ""{m_tempblocktable}"".""Hash"", - ""{m_tempblocktable}"".""Size"" -FROM - ""{m_tempfiletable}"", - ""{m_tempblocktable}"", - ""BlocksetEntry"" -WHERE - ""{m_tempfiletable}"".""ID"" = ""{m_tempblocktable}"".""FileID"" AND - ""BlocksetEntry"".""BlocksetID"" = ""{m_tempfiletable}"".""LatestBlocksetID"" AND - ""BlocksetEntry"".""BlockID"" = ""{m_tempblocktable}"".""BlockID"" AND - ""BlocksetEntry"".""Index"" = ""{m_tempblocktable}"".""Index"" AND - {whereclause} -ORDER BY ""{m_tempfiletable}"".""ID"", ""{m_tempblocktable}"".""Index"" "); - - using (var cmd = m_connection.CreateCommand()) - using (var rd = cmd.ExecuteReader(sources)) - { - if (rd.Read()) - { - bool more; - do - { - var n = new FastSource(rd, blocksize); - var tid = n.TargetFileID; - yield return n; - - more = n.MoreData; - while (more && n.TargetFileID == tid) - more = rd.Read(); - - } while (more); - } - } - } - - } -} +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.Common.IO; +using Duplicati.Library.Main.Operation.Restore; +using Duplicati.Library.Main.Volumes; +using Duplicati.Library.SQLiteHelper; +using Duplicati.Library.Utility; +using Microsoft.Data.Sqlite; + +namespace Duplicati.Library.Main.Database +{ + /// + /// Represents a local database used during restore. + /// Provides methods for preparing and tracking restore file lists, managing temporary tables for files and blocks, + /// tracking restore progress, and efficiently querying and updating restore-related metadata. + /// This class supports advanced restore scenarios, including filtering, progress tracking via triggers, + /// cross-platform path mapping, and efficient block and file restoration workflows. + /// + internal class LocalRestoreDatabase : LocalDatabase + { + /// + /// The tag used for logging. + /// + private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalRestoreDatabase)); + + /// + /// A unique identifier for the temporary table set, used to ensure that temporary tables do not conflict with others. + /// + protected readonly string m_temptabsetguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + /// + /// The name of the temporary table in the database, which is used to store the list of files to restore. + /// + protected string? m_tempfiletable; + /// + /// The name of the temporary table in the database, which is used to store the blocks associated with the files to restore. + /// + protected string? m_tempblocktable; + /// + /// A pool of connections and transactions that are used for concurrent access to the database. + /// + protected ConcurrentBag<(SqliteConnection, ReusableTransaction)> m_connection_pool = []; + /// + /// The latest block table used to track the most recent blocks processed during a restore operation. + /// + protected string? m_latestblocktable; + /// + /// The name of the temporary table used to track progress of file restoration. + /// + protected string? m_fileprogtable; + /// + /// The name of the temporary table used to track overall restoration progress across all files. + /// + protected string? m_totalprogtable; + /// + /// The name of the temporary table used to track files that have been fully restored (all blocks restored). + /// + protected string? m_filesnewlydonetable; + + /// + /// A semaphore used to ensure that only one thread can access the database at a time. + /// + private SemaphoreSlim m_dbLock = new(1, 1); + + /// + /// The time at which the restore operation begins. + /// + protected DateTime m_restoreTime; + + /// + /// Gets the time that the restore operation was initiated. + /// + public DateTime RestoreTime { get { return m_restoreTime; } } + + /// + /// Initializes a new instance of the class. + /// + /// The path to the database file. + /// An optional existing instance of to use, or null to create a new one. Used to mimic constructor chaining. + /// A cancellation token to monitor for cancellation requests. + /// A task that when awaited on, returns a new instance of . + public static async Task CreateAsync(string path, LocalRestoreDatabase? dbnew, CancellationToken token) + { + dbnew ??= new LocalRestoreDatabase(); + + dbnew = (LocalRestoreDatabase) + await CreateLocalDatabaseAsync(path, "Restore", false, dbnew, token) + .ConfigureAwait(false); + dbnew.ShouldCloseConnection = true; + + return dbnew; + } + + /// + /// Initializes a new instance of the class using an existing parent database. + /// + /// The parent database from which to create the restore database. + /// An optional existing instance of to use, or null to create a new one. Used to mimic constructor chaining. + /// A cancellation token to monitor for cancellation requests. + /// A task that when awaited on, returns a new instance of . + public static async Task CreateAsync(LocalDatabase dbparent, LocalRestoreDatabase? dbnew, CancellationToken token) + { + dbnew ??= new LocalRestoreDatabase(); + + return (LocalRestoreDatabase) + await CreateLocalDatabaseAsync(dbparent, dbnew, token) + .ConfigureAwait(false); + } + + /// + /// Create tables and triggers for automatic tracking of progress during a restore operation. + /// This replaces continuous requerying of block progress by iterating over blocks table. + /// SQLite is much faster keeping information up to date with internal triggers. + /// + /// This allows to create another table that keeps track + /// of all files that are done (all data blocks restored). + /// A cancellation token to monitor for cancellation requests. + /// + /// The method is prepared to create a table that keeps track of all files being done completely. + /// That means, it fires for files where the number of restored blocks equals the number of all blocks. + /// It is intended to be used for fast identification of fully restored files to trigger their verification. + /// It should be read after a commit and truncated after putting the files to a verification queue. + /// Note: If a file is done once and then set back to a none restored state, the file is not automatically removed. + /// But if it reaches a restored state later, it will be re-added (trigger will fire) + /// + /// A task that completes when the progress tracker is created. + public async Task CreateProgressTracker(bool createFilesNewlyDoneTracker, CancellationToken token) + { + m_fileprogtable = "FileProgress-" + m_temptabsetguid; + m_totalprogtable = "TotalProgress-" + m_temptabsetguid; + m_filesnewlydonetable = createFilesNewlyDoneTracker ? "FilesNewlyDone-" + m_temptabsetguid : null; + + await using var cmd = m_connection.CreateCommand() + .SetTransaction(m_rtr); + + // How to handle METADATA? + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_fileprogtable}"" ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{m_fileprogtable}"" ( + ""FileId"" INTEGER PRIMARY KEY, + ""TotalBlocks"" INTEGER NOT NULL, + ""TotalSize"" INTEGER NOT NULL, + ""BlocksRestored"" INTEGER NOT NULL, + ""SizeRestored"" INTEGER NOT NULL + ) + ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_totalprogtable}"" ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{m_totalprogtable}"" ( + ""TotalFiles"" INTEGER NOT NULL, + ""TotalBlocks"" INTEGER NOT NULL, + ""TotalSize"" INTEGER NOT NULL, + ""FilesFullyRestored"" INTEGER NOT NULL, + ""FilesPartiallyRestored"" INTEGER NOT NULL, + ""BlocksRestored"" INTEGER NOT NULL, + ""SizeRestored"" INTEGER NOT NULL + ) + ", token) + .ConfigureAwait(false); + + if (createFilesNewlyDoneTracker) + { + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}"" ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{m_filesnewlydonetable}"" ( + ""ID"" INTEGER PRIMARY KEY + ) + ", token) + .ConfigureAwait(false); + } + + try + { + // Initialize statistics with File- and Block-Data (it is valid to already have restored blocks in files) + // A rebuild with this function should be valid anytime. + // Note: FilesNewlyDone is NOT initialized, as in initialization nothing is really new. + + // We use a LEFT JOIN to allow for empty files (no data Blocks) + + // Will be one row per file. + await cmd.ExecuteNonQueryAsync($@" + INSERT INTO ""{m_fileprogtable}"" ( + ""FileId"", + ""TotalBlocks"", + ""TotalSize"", + ""BlocksRestored"", + ""SizeRestored"" + ) + SELECT + ""F"".""ID"", + IFNULL(COUNT(""B"".""ID""), 0), + IFNULL(SUM(""B"".""Size""), 0), + IFNULL( + COUNT( + CASE ""B"".""Restored"" + WHEN 1 + THEN ""B"".""ID"" + ELSE NULL + END + ), + 0 + ), + IFNULL( + SUM( + CASE ""B"".""Restored"" + WHEN 1 + THEN ""B"".""Size"" + ELSE 0 + END + ), + 0 + ) + FROM ""{m_tempfiletable}"" ""F"" + LEFT JOIN ""{m_tempblocktable}"" ""B"" + ON ""B"".""FileID"" = ""F"".""ID"" + WHERE ""B"".""Metadata"" IS NOT 1 + GROUP BY ""F"".""ID"" + ", token) + .ConfigureAwait(false); + + // Will result in a single line (no support to also track metadata) + await cmd.ExecuteNonQueryAsync($@" + INSERT INTO ""{m_totalprogtable}"" ( + ""TotalFiles"", + ""TotalBlocks"", + ""TotalSize"", + ""FilesFullyRestored"", + ""FilesPartiallyRestored"", + ""BlocksRestored"", + ""SizeRestored"" + ) + SELECT + IFNULL(COUNT(""P"".""FileId""), 0), + IFNULL(SUM(""P"".""TotalBlocks""), 0), + IFNULL(SUM(""P"".""TotalSize""), 0), + IFNULL( + COUNT( + CASE + WHEN ""P"".""BlocksRestored"" = ""P"".""TotalBlocks"" + THEN 1 + ELSE NULL + END + ), + 0 + ), + IFNULL( + COUNT( + CASE + WHEN + ""P"".""BlocksRestored"" BETWEEN 1 + AND ""P"".""TotalBlocks"" - 1 + THEN 1 + ELSE NULL + END + ), + 0 + ), + IFNULL(SUM(""P"".""BlocksRestored""), 0), + IFNULL(SUM(""P"".""SizeRestored""), 0) + FROM ""{m_fileprogtable}"" ""P"" + ", token) + .ConfigureAwait(false); + + // Finally we create TRIGGERs to keep all our statistics up to date. + // This is lightning fast, as SQLite uses internal hooks and our indices to do the update magic. + // Note: We do assume that neither files nor blocks will be added or deleted during restore process + // and that the size of each block stays constant so there is no need to track that information + // with additional INSERT and DELETE triggers. + + // A trigger to update the file-stat entry each time a block changes restoration state. + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TRIGGER ""TrackRestoredBlocks_{m_tempblocktable}"" + AFTER UPDATE OF ""Restored"" + ON ""{m_tempblocktable}"" + WHEN OLD.""Restored"" != NEW.""Restored"" + AND NEW.""Metadata"" = 0 + BEGIN UPDATE ""{m_fileprogtable}"" + SET + ""BlocksRestored"" = + ""{m_fileprogtable}"".""BlocksRestored"" + + (NEW.""Restored"" - OLD.""Restored""), + ""SizeRestored"" = + ""{m_fileprogtable}"".""SizeRestored"" + + ((NEW.""Restored"" - OLD.""Restored"") * NEW.Size) + WHERE ""{m_fileprogtable}"".""FileId"" = NEW.""FileID"" + ; END + ", token) + .ConfigureAwait(false); + + // A trigger to update total stats each time a file stat changed (nested triggering by file-stats) + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TRIGGER ""UpdateTotalStats_{m_fileprogtable}"" + AFTER UPDATE ON ""{m_fileprogtable}"" + BEGIN UPDATE ""{m_totalprogtable}"" + SET + ""FilesFullyRestored"" = + ""{m_totalprogtable}"".""FilesFullyRestored"" + + (CASE + WHEN NEW.""BlocksRestored"" = NEW.""TotalBlocks"" + THEN 1 + ELSE 0 + END) + - (CASE + WHEN OLD.""BlocksRestored"" = OLD.""TotalBlocks"" + THEN 1 + ELSE 0 + END), + ""FilesPartiallyRestored"" = + ""{m_totalprogtable}"".""FilesPartiallyRestored"" + + (CASE + WHEN + NEW.""BlocksRestored"" BETWEEN 1 + AND NEW.""TotalBlocks"" - 1 + THEN 1 + ELSE 0 + END) + - (CASE + WHEN + OLD.""BlocksRestored"" BETWEEN 1 + AND OLD.""TotalBlocks"" - 1 + THEN 1 + ELSE 0 + END), + ""BlocksRestored"" = + ""{m_totalprogtable}"".""BlocksRestored"" + + NEW.""BlocksRestored"" + - OLD.""BlocksRestored"", + ""SizeRestored"" = + ""{m_totalprogtable}"".""SizeRestored"" + + NEW.""SizeRestored"" + - OLD.""SizeRestored"" + ; END + ", token) + .ConfigureAwait(false); + + if (createFilesNewlyDoneTracker) + { + // A trigger checking if a file is done (all blocks restored in file-stat) (nested triggering by file-stats) + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TRIGGER ""UpdateFilesNewlyDone_{m_fileprogtable}"" + AFTER UPDATE OF + ""BlocksRestored"", + ""TotalBlocks"" + ON ""{m_fileprogtable}"" + WHEN NEW.""BlocksRestored"" = NEW.""TotalBlocks"" + BEGIN + INSERT OR IGNORE INTO ""{m_filesnewlydonetable}"" (""ID"") + VALUES (NEW.""FileId""); + END + ", token) + .ConfigureAwait(false); + } + + } + catch (Exception ex) + { + m_fileprogtable = null; + m_totalprogtable = null; + Logging.Log.WriteWarningMessage(LOGTAG, "ProgressTrackerSetupError", ex, "Failed to set up progress tracking tables"); + throw; + } + finally + { + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); + } + } + + /// + /// Prepares the restore file list by querying the database for filesets and their associated files. + /// + /// The time at which the restore operation is being performed. + /// An array of version identifiers to filter the filesets. + /// An optional filter to apply to the files being restored. + /// A cancellation token to monitor for cancellation requests. + /// A task that, when awaited, returns a tuple containing the count of files to restore and the total size of those files. + public async Task> PrepareRestoreFilelist(DateTime restoretime, long[] versions, IFilter filter, CancellationToken token) + { + m_tempfiletable = "Fileset-" + m_temptabsetguid; + m_tempblocktable = "Blocks-" + m_temptabsetguid; + + await using (var cmd = m_connection.CreateCommand()) + { + cmd.SetTransaction(m_rtr); + var filesetIds = + await GetFilesetIDs(Library.Utility.Utility.NormalizeDateTime(restoretime), versions, false, token) + .ToListAsync(cancellationToken: token) + .ConfigureAwait(false); + + while (filesetIds.Count > 0) + { + var filesetId = filesetIds[0]; + filesetIds.RemoveAt(0); + + cmd.SetCommandAndParameters(@" + SELECT ""Timestamp"" + FROM ""Fileset"" + WHERE ""ID"" = @FilesetId + ") + .SetParameterValue("@FilesetId", filesetId); + m_restoreTime = ParseFromEpochSeconds( + await cmd.ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false) + ); + + var ix = await FilesetTimes(token) + .Select((value, index) => new { value.Key, index }) + .Where(n => n.Key == filesetId) + .Select(pair => pair.index + 1) + .FirstOrDefaultAsync(cancellationToken: token) + .ConfigureAwait(false) - 1; + + Logging.Log.WriteInformationMessage(LOGTAG, "SearchingBackup", "Searching backup {0} ({1}) ...", ix, m_restoreTime); + + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempfiletable}"" ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempblocktable}"" ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{m_tempfiletable}"" ( + ""ID"" INTEGER PRIMARY KEY, + ""Path"" TEXT NOT NULL, + ""BlocksetID"" INTEGER NOT NULL, + ""MetadataID"" INTEGER NOT NULL, + ""TargetPath"" TEXT NULL, + ""DataVerified"" BOOLEAN NOT NULL, + ""LatestBlocksetId"" INTEGER, + ""LocalSourceExists"" BOOLEAN + ) + ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{m_tempblocktable}"" ( + ""ID"" INTEGER PRIMARY KEY, + ""FileID"" INTEGER NOT NULL, + ""Index"" INTEGER NOT NULL, + ""Hash"" TEXT NOT NULL, + ""Size"" INTEGER NOT NULL, + ""Restored"" BOOLEAN NOT NULL, + ""Metadata"" BOOLEAN NOT NULL, + ""VolumeID"" INTEGER NOT NULL, + ""BlockID"" INTEGER NOT NULL + ) + ", token) + .ConfigureAwait(false); + + // TODO: Optimize to use the path prefix + + if (filter == null || filter.Empty) + { + // Simple case, restore everything + await cmd.SetCommandAndParameters($@" + INSERT INTO ""{m_tempfiletable}"" ( + ""ID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"", + ""DataVerified"" + ) + SELECT + ""File"".""ID"", + ""File"".""Path"", + ""File"".""BlocksetID"", + ""File"".""MetadataID"", + 0 + FROM + ""File"", + ""FilesetEntry"" + WHERE + ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""FilesetEntry"".""FilesetID"" = @FilesetId + ") + .SetParameterValue("@FilesetId", filesetId) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } + else if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == FilterType.Simple) + { + using (new Logging.Timer(LOGTAG, "CommitBeforePrepareFileset", "CommitBeforePrepareFileset")) + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); + + cmd.SetTransaction(m_rtr); + // If we get a list of filenames, the lookup table is faster + // unfortunately we cannot do this if the filesystem is case sensitive as + // SQLite only supports ASCII compares + var p = expression.GetSimpleList(); + var m_filenamestable = "Filenames-" + m_temptabsetguid; + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{m_filenamestable}"" ( + ""Path"" TEXT NOT NULL + ) + ", token) + .ConfigureAwait(false); + + cmd.SetCommandAndParameters($@" + INSERT INTO ""{m_filenamestable}"" (""Path"") + VALUES (@Path) + ") + .ConfigureAwait(false); + + foreach (var s in p) + { + await cmd + .SetParameterValue("@Path", s) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } + + var c = await cmd.SetCommandAndParameters($@" + INSERT INTO ""{m_tempfiletable}"" ( + ""ID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"", + ""DataVerified"" + ) + SELECT + ""File"".""ID"", + ""File"".""Path"", + ""File"".""BlocksetID"", + ""File"".""MetadataID"", + 0 + FROM + ""File"", + ""FilesetEntry"" + WHERE + ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""FilesetEntry"".""FilesetID"" = @FilesetId + AND ""Path"" IN ( + SELECT DISTINCT ""Path"" + FROM ""{m_filenamestable}"" + ) + ") + .SetParameterValue("@FilesetId", filesetId) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + if (c != p.Length && c != 0) + { + var sb = new StringBuilder(); + sb.AppendLine(); + + cmd.SetCommandAndParameters($@" + SELECT ""Path"" + FROM ""{m_filenamestable}"" + WHERE ""Path"" NOT IN ( + SELECT ""Path"" + FROM ""{m_tempfiletable}"" + ) + "); + await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) + sb.AppendLine(rd.ConvertValueToString(0)); + + cmd.SetCommandAndParameters(@" + SELECT ""Timestamp"" + FROM ""Fileset"" + WHERE ""ID"" = @FilesetId + ") + .SetParameterValue("@FilesetId", filesetId); + var actualrestoretime = ParseFromEpochSeconds( + await cmd.ExecuteScalarInt64Async(0, token) + .ConfigureAwait(false) + ); + + Logging.Log.WriteWarningMessage(LOGTAG, "FilesNotFoundInBackupList", null, "{0} File(s) were not found in list of files for backup at {1}, will not be restored: {2}", p.Length - c, actualrestoretime.ToLocalTime(), sb); + } + + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filenamestable}"" ", token) + .ConfigureAwait(false); + + using (new Logging.Timer(LOGTAG, "CommitAfterPrepareFileset", "CommitAfterPrepareFileset")) + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); + cmd.SetTransaction(m_rtr); + } + else + { + // Restore but filter elements based on the filter expression + // If this is too slow, we could add a special handler for wildcard searches too + cmd.SetCommandAndParameters(@" + SELECT + ""File"".""ID"", + ""File"".""Path"", + ""File"".""BlocksetID"", + ""File"".""MetadataID"" + FROM + ""File"", + ""FilesetEntry"" + WHERE + ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""FilesetID"" = @FilesetId + ") + .SetParameterValue("@FilesetId", filesetId); + + object[] values = new object[4]; + await using var cmd2 = m_connection.CreateCommand($@" + INSERT INTO ""{m_tempfiletable}"" ( + ""ID"", + ""Path"", + ""BlocksetID"", + ""MetadataID"", + ""DataVerified"" + ) + VALUES ( + @ID, + @Path, + @BlocksetID, + @MetadataID, + 0 + ) + "); + await using var rd = await cmd.ExecuteReaderAsync(token) + .ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + { + rd.GetValues(values); + var path = values[1] as string; + if (path != null && FilterExpression.Matches(filter, path.ToString()!)) + { + await cmd2 + .SetParameterValue("@ID", values[0]) + .SetParameterValue("@Path", values[1]) + .SetParameterValue("@BlocksetID", values[2]) + .SetParameterValue("@MetadataID", values[3]) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } + } + } + + //creating indexes after insertion is much faster + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{m_tempfiletable}_ID"" + ON ""{m_tempfiletable}"" (""ID"") + ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{m_tempfiletable}_TargetPath"" + ON ""{m_tempfiletable}"" (""TargetPath"") + ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{m_tempfiletable}_Path"" + ON ""{m_tempfiletable}"" (""Path"") + ", token) + .ConfigureAwait(false); + + cmd.SetCommandAndParameters($@" + SELECT + COUNT(DISTINCT ""{m_tempfiletable}"".""Path""), + SUM(""Blockset"".""Length"") + FROM + ""{m_tempfiletable}"", + ""Blockset"" + WHERE ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" + "); + await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + { + var filecount = 0L; + var filesize = 0L; + + if (await rd.ReadAsync(token).ConfigureAwait(false)) + { + filecount = rd.ConvertValueToInt64(0, 0); + filesize = rd.ConvertValueToInt64(1, 0); + } + + if (filecount > 0) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreTargetFileCount", "Needs to restore {0} files ({1})", filecount, Library.Utility.Utility.FormatSizeString(filesize)); + return new Tuple(filecount, filesize); + } + } + } + + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); + } + + return new Tuple(0, 0); + } + + /// + /// Retrieves the first path from the temporary file table. + /// + /// A cancellation token to monitor for cancellation requests. + /// A task that, when awaited, returns the first path found in the temporary file table, or null if no paths are found. + public async Task GetFirstPath(CancellationToken token) + { + await using var cmd = m_connection.CreateCommand($@" + SELECT ""Path"" + FROM ""{m_tempfiletable}"" + ORDER BY LENGTH(""Path"") DESC + LIMIT 1 + ") + .SetTransaction(m_rtr); + + var v0 = await cmd.ExecuteScalarAsync(token).ConfigureAwait(false); + if (v0 == null || v0 == DBNull.Value) + return null; + + return v0.ToString(); + } + + /// + /// Retrieves the largest prefix path from the temporary file table. + /// + /// A cancellation token to monitor for cancellation requests. + /// A task that, when awaited, returns the largest prefix path found in the temporary file table. + public async Task GetLargestPrefix(CancellationToken token) + { + await using var cmd = m_connection.CreateCommand($@" + SELECT ""Path"" + FROM ""{m_tempfiletable}"" + ORDER BY LENGTH(""Path"") DESC + LIMIT 1 + ") + .SetTransaction(m_rtr); + + var v0 = await cmd.ExecuteScalarAsync(token).ConfigureAwait(false); + var maxpath = ""; + if (v0 != null && v0 != DBNull.Value) + maxpath = v0.ToString()!; + + var dirsep = Util.GuessDirSeparator(maxpath); + + var filecount = await cmd.ExecuteScalarInt64Async($@" + SELECT COUNT(*) + FROM ""{m_tempfiletable}"" + ", -1, token) + .ConfigureAwait(false); + + var foundfiles = -1L; + + //TODO: Handle FS case-sensitive? + cmd.SetCommandAndParameters($@" + SELECT COUNT(*) + FROM ""{m_tempfiletable}"" + WHERE SUBSTR(""Path"", 1, @PrefixLength) = @Prefix + "); + + while (filecount != foundfiles && maxpath.Length > 0) + { + var mp = Util.AppendDirSeparator(maxpath, dirsep); + foundfiles = await cmd + .SetParameterValue("@PrefixLength", mp.Length) + .SetParameterValue("@Prefix", mp) + .ExecuteScalarInt64Async(-1, token) + .ConfigureAwait(false); + + if (filecount != foundfiles) + { + var oldlen = maxpath.Length; + + var lix = maxpath.LastIndexOf(dirsep, maxpath.Length - 2, StringComparison.Ordinal); + maxpath = maxpath.Substring(0, lix + 1); + if (string.IsNullOrWhiteSpace(maxpath) || maxpath.Length == oldlen) + maxpath = ""; + } + } + + return maxpath == "" ? "" : Util.AppendDirSeparator(maxpath, dirsep); + } + + /// + /// Sets the target paths for files in the temporary file table. + /// This method adjusts the target paths based on the largest prefix and destination provided. + /// + /// The largest prefix path to use for adjusting target paths. + /// The destination path to prepend to the target paths. + /// A cancellation token to monitor for cancellation requests. + /// A task that completes when the target paths have been set. + public async Task SetTargetPaths(string largest_prefix, string destination, CancellationToken token) + { + var dirsep = Util.GuessDirSeparator(string.IsNullOrWhiteSpace(largest_prefix) ? + await GetFirstPath(token).ConfigureAwait(false) + : + largest_prefix); + + await using var cmd = m_connection.CreateCommand() + .SetTransaction(m_rtr); + if (string.IsNullOrEmpty(destination)) + { + //The string fixing here is meant to provide some non-random + // defaults when restoring cross OS, e.g. backup on Linux, restore on Windows + //This is mostly meaningless, and the user really should use --restore-path + + if ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) && dirsep == "\\") + { + // For Win -> Linux, we remove the colon from the drive letter, and use the drive letter as root folder + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET ""Targetpath"" = + CASE + WHEN SUBSTR(""Path"", 2, 1) == ':' + THEN '\\' || SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) + ELSE ""Path"" + END + ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET ""Targetpath"" = + CASE + WHEN SUBSTR(""Path"", 1, 2) == '\\' + THEN '\\' || SUBSTR(""Path"", 2) + ELSE ""Path"" + END + ", token) + .ConfigureAwait(false); + + } + else if (OperatingSystem.IsWindows() && dirsep == "/") + { + // For Linux -> Win, we use the temporary folder's drive as the root path + await cmd.SetCommandAndParameters($@" + UPDATE ""{m_tempfiletable}"" + SET ""Targetpath"" = + CASE + WHEN SUBSTR(""Path"", 1, 1) == '/' + THEN @Path || SUBSTR(""Path"", 2) + ELSE ""Path"" + END + ") + .SetParameterValue("@Path", Util.AppendDirSeparator(System.IO.Path.GetPathRoot(Library.Utility.TempFolder.SystemTempPath)).Replace("\\", "/")) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } + else + { + // Same OS, just use the path directly + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET ""Targetpath"" = ""Path"" + ", token) + .ConfigureAwait(false); + } + } + else + { + if (string.IsNullOrEmpty(largest_prefix)) + { + //Special case, restoring to new folder, but files are from different drives (no shared root on Windows) + + // We use the format / / + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET ""TargetPath"" = + CASE + WHEN SUBSTR(""Path"", 2, 1) == ':' + THEN SUBSTR(""Path"", 1, 1) || SUBSTR(""Path"", 3) + ELSE ""Path"" + END + ", token) + .ConfigureAwait(false); + + // For UNC paths, we use \\server\folder -> / / + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET ""TargetPath"" = + CASE + WHEN SUBSTR(""Path"", 1, 2) == '\\' + THEN SUBSTR(""Path"", 2) + ELSE ""TargetPath"" + END + ", token) + .ConfigureAwait(false); + } + else + { + largest_prefix = Util.AppendDirSeparator(largest_prefix, dirsep); + await cmd.SetCommandAndParameters($@" + UPDATE ""{m_tempfiletable}"" + SET ""TargetPath"" = SUBSTR(""Path"", @PrefixLength) + ") + .SetParameterValue("@PrefixLength", largest_prefix.Length + 1) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } + } + + // Cross-os path remapping support + if ((OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) && dirsep == "\\") + // For Win paths on Linux + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET ""TargetPath"" = REPLACE(""TargetPath"", '\', '/') + ", token) + .ConfigureAwait(false); + else if (OperatingSystem.IsWindows() && dirsep == "/") + // For Linux paths on Windows + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET ""TargetPath"" = REPLACE(REPLACE(""TargetPath"", '\', '_'), '/', '\') + ", token) + .ConfigureAwait(false); + + if (!string.IsNullOrEmpty(destination)) + { + // Paths are now relative with target-os naming system + // so we prefix them with the target path + await cmd.SetCommandAndParameters($@" + UPDATE ""{m_tempfiletable}"" + SET ""TargetPath"" = @Destination || ""TargetPath"" + ") + .SetParameterValue("@Destination", Util.AppendDirSeparator(destination)) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } + + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); + } + + /// + /// Finds missing blocks in the temporary block table. + /// + /// If true, skips metadata blocks when searching for missing blocks. + /// A cancellation token to monitor for cancellation requests. + /// A task that completes when the missing blocks have been found and inserted into the temporary block table. + public async Task FindMissingBlocks(bool skipMetadata, CancellationToken token) + { + await using var cmd = m_connection.CreateCommand() + .SetTransaction(m_rtr); + var p1 = await cmd.ExecuteNonQueryAsync($@" + INSERT INTO ""{m_tempblocktable}"" ( + ""FileID"", + ""Index"", + ""Hash"", + ""Size"", + ""Restored"", + ""Metadata"", + ""VolumeId"", + ""BlockId"" + ) + SELECT DISTINCT + ""{m_tempfiletable}"".""ID"", + ""BlocksetEntry"".""Index"", + ""Block"".""Hash"", + ""Block"".""Size"", + 0, + 0, + ""Block"".""VolumeID"", + ""Block"".""ID"" + FROM + ""{m_tempfiletable}"", + ""BlocksetEntry"", + ""Block"" + WHERE + ""{m_tempfiletable}"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + ", token) + .ConfigureAwait(false); + + var p2 = 0; + if (!skipMetadata) + p2 = await cmd.ExecuteNonQueryAsync($@" + INSERT INTO ""{m_tempblocktable}"" ( + ""FileID"", + ""Index"", + ""Hash"", + ""Size"", + ""Restored"", + ""Metadata"", + ""VolumeId"", + ""BlockId"" + ) + SELECT DISTINCT + ""{m_tempfiletable}"".""ID"", + ""BlocksetEntry"".""Index"", + ""Block"".""Hash"", + ""Block"".""Size"", + 0, + 1, + ""Block"".""VolumeID"", + ""Block"".""ID"" + FROM + ""{m_tempfiletable}"", + ""BlocksetEntry"", + ""Block"", + ""Metadataset"" + WHERE + ""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID"" + AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + ", token) + .ConfigureAwait(false); + + // Creating indexes after insertion is much faster + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{m_tempblocktable}_HashSizeIndex"" + ON ""{m_tempblocktable}"" (""Hash"", ""Size"") + ", token) + .ConfigureAwait(false); + + // Better suited to speed up commit on UpdateBlocks + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{m_tempblocktable}_FileIdIndexIndex"" + ON ""{m_tempblocktable}"" (""FileId"", ""Index"") + ", token) + .ConfigureAwait(false); + + var size = await cmd.ExecuteScalarInt64Async($@" + SELECT SUM(""Size"") + FROM ""{m_tempblocktable}"" + ", 0, token) + .ConfigureAwait(false); + + Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreSourceSize", "Restore list contains {0} blocks with a total size of {1}", p1 + p2, Library.Utility.Utility.FormatSizeString(size)); + + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); + } + + /// + /// Updates the target path of a file in the temporary file table. + /// + /// The ID of the file in the temporary file table. + /// The new target path to set for the file. + /// A cancellation token to monitor for cancellation requests. + /// A task that completes when the target path has been updated. + public async Task UpdateTargetPath(long ID, string newname, CancellationToken token) + { + await using var cmd = m_connection.CreateCommand($@" + UPDATE ""{m_tempfiletable}"" + SET ""TargetPath"" = @TargetPath + WHERE ""ID"" = @ID + "); + await cmd + .SetTransaction(m_rtr) + .SetParameterValue("@TargetPath", newname) + .SetParameterValue("@ID", ID) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); + } + + /// + /// Interface for an object describing a block in an existing file. + /// + public interface IExistingFileBlock + { + /// + /// Gets the hash of the block. + /// + string Hash { get; } + /// + /// Gets the index of the block within the file. + /// + long Index { get; } + /// + /// Gets the size of the block in bytes. + /// + long Size { get; } + } + + /// + /// Interface for an existing file with its blocks. + /// + public interface IExistingFile + { + /// + /// Gets the target path of the file. + /// + string TargetPath { get; } + /// + /// Gets the hash of the file. + /// + string TargetHash { get; } + /// + /// Gets the ID of the file in the database. + /// + long TargetFileID { get; } + /// + /// Gets the length of the file in bytes. + /// + long Length { get; } + /// + /// An asynchronous enumerable of the blocks in the file. + /// + /// A cancellation token to monitor for cancellation requests. + IAsyncEnumerable Blocks(CancellationToken token); + } + + /// + /// Interface for a block source, which can be a file or a metadata source. + /// + public interface IBlockSource + { + /// + /// Gets the path of the block source. + /// + string Path { get; } + /// + /// Gets the offset of the block within the source. + /// + long Offset { get; } + /// + /// Indicates whether the block source is metadata. + /// + bool IsMetadata { get; } + } + + /// + /// Interface for a block descriptor, which describes a block in a file. + /// + public interface IBlockDescriptor + { + /// + /// Gets the hash of the block. + /// + string Hash { get; } + /// + /// Gets the size of the block in bytes. + /// + long Size { get; } + /// + /// Gets the offset of the block within the file. + /// + long Offset { get; } + /// + /// Gets the index of the block within the file. + /// + long Index { get; } + /// + /// Indicates whether the block is metadata. + /// + bool IsMetadata { get; } + /// + /// An asynchronous enumerable of block sources for this block descriptor. + /// + /// A cancellation token to monitor for cancellation requests. + IAsyncEnumerable BlockSources(CancellationToken token); + } + + /// + /// Interface for a local block source, which provides information about a file and its blocks. + /// + public interface ILocalBlockSource + { + /// + /// Gets the target path of the file. + /// + string TargetPath { get; } + /// + /// Gets the ID of the target file in the database. + /// + long TargetFileID { get; } + /// + /// An asynchronous enumerable of block descriptors for the file. + /// + /// A cancellation token to monitor for cancellation requests. + IAsyncEnumerable Blocks(CancellationToken token); + } + + /// + /// Interface for an object describing a file to restore. + /// + public interface IFileToRestore + { + /// + /// Gets the target path of the file. + /// + string Path { get; } + /// + /// Gets the hash of the file. + /// + string Hash { get; } + /// + /// Gets the ID of the file in the database. + /// + long Length { get; } + } + + /// + /// Interface for a remote volume, which contains blocks to be restored. + /// + public interface IPatchBlock + { + /// + /// Gets the hash of the block. + /// + long Offset { get; } + /// + /// Gets the size of the block in bytes. + /// + long Size { get; } + /// + /// Gets the path of the block source. + /// + string Key { get; } + } + + /// + /// Interface for a remote volume that contains blocks to be restored. + /// + public interface IVolumePatch + { + /// + /// Gets the name of the remote volume. + /// + string Path { get; } + /// + /// Gets the hash of the remote volume. + /// + long FileID { get; } + /// + /// Gets the size of the remote volume in bytes. + /// + /// A cancellation token to monitor for cancellation requests. + IAsyncEnumerable Blocks(CancellationToken token); + } + + /// + /// Implementation of the IExistingFile interface that retrieves existing files and their blocks from the database. + /// + private class ExistingFile : IExistingFile + { + /// + /// The data reader used to read existing files from the database. + /// + private readonly SqliteDataReader m_reader; + + /// + /// Initializes a new instance of the ExistingFile class. + /// + /// The SqliteDataReader used to read existing files. + public ExistingFile(SqliteDataReader rd) + { + m_reader = rd; + HasMore = true; + } + + public string TargetPath { get { return m_reader.ConvertValueToString(0) ?? ""; } } + public string TargetHash { get { return m_reader.ConvertValueToString(1) ?? ""; } } + public long TargetFileID { get { return m_reader.ConvertValueToInt64(2); } } + public long Length { get { return m_reader.ConvertValueToInt64(3); } } + + /// + /// Indicates whether there are more blocks to read for the current file. + /// + public bool HasMore { get; private set; } + + /// + /// Represents a block in an existing file. + /// + private class ExistingFileBlock : IExistingFileBlock + { + /// + /// The data reader used to read block information for existing files. + /// + private readonly SqliteDataReader m_reader; + + /// + /// Initializes a new instance of the ExistingFileBlock class. + /// + public ExistingFileBlock(SqliteDataReader rd) + { + m_reader = rd; + } + + public string Hash { get { return m_reader.ConvertValueToString(4) ?? ""; } } + public long Index { get { return m_reader.ConvertValueToInt64(5); } } + public long Size { get { return m_reader.ConvertValueToInt64(6); } } + } + + /// + /// Gets the blocks in the existing file. + /// + /// A cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of IExistingFileBlock objects representing the blocks in the file. + public async IAsyncEnumerable Blocks([EnumeratorCancellation] CancellationToken token) + { + string p = TargetPath; + while (HasMore && p == TargetPath) + { + yield return new ExistingFileBlock(m_reader); + HasMore = await m_reader.ReadAsync(token).ConfigureAwait(false); + } + } + + /// + /// Retrieves existing files with their blocks from the database. + /// + /// The LocalDatabase instance to query. + /// The name of the table containing existing files. + /// A cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of IExistingFile objects representing the existing files with their blocks. + public static async IAsyncEnumerable GetExistingFilesWithBlocks(LocalDatabase db, string tablename, [EnumeratorCancellation] CancellationToken token) + { + await using var cmd = db.Connection.CreateCommand($@" + SELECT + ""{tablename}"".""TargetPath"", + ""Blockset"".""FullHash"", + ""{tablename}"".""ID"", + ""Blockset"".""Length"", + ""Block"".""Hash"", + ""BlocksetEntry"".""Index"", + ""Block"".""Size"" + FROM + ""{tablename}"", + ""Blockset"", + ""BlocksetEntry"", + ""Block"" + WHERE + ""{tablename}"".""BlocksetID"" = ""Blockset"".""ID"" + AND ""BlocksetEntry"".""BlocksetID"" = ""{tablename}"".""BlocksetID"" + AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + ORDER BY + ""{tablename}"".""TargetPath"", + ""BlocksetEntry"".""Index"" + ") + .SetTransaction(db.Transaction); + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + if (await rd.ReadAsync(token).ConfigureAwait(false)) + { + var more = true; + while (more) + { + var f = new ExistingFile(rd); + string current = f.TargetPath; + yield return f; + + more = f.HasMore; + while (more && current == f.TargetPath) + more = await rd.ReadAsync(token).ConfigureAwait(false); + } + } + } + } + + /// + /// Retrieves existing files with their blocks from the temporary file table. + /// + /// A cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of IExistingFile objects representing the existing files with their blocks. + public IAsyncEnumerable GetExistingFilesWithBlocks(CancellationToken token) + { + if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) + throw new InvalidOperationException("No temporary file table set up for this restore."); + return ExistingFile.GetExistingFilesWithBlocks(this, m_tempfiletable, token); + } + + /// + /// Implements the ILocalBlockSource interface to provide access to local block sources. + /// + private class LocalBlockSource : ILocalBlockSource + { + /// + /// Implements the IBlockDescriptor interface to provide access to block descriptors. + /// + private class BlockDescriptor : IBlockDescriptor + { + /// + /// Implements the IBlockSource interface to provide access to block sources. + /// + private class BlockSource : IBlockSource + { + /// + /// The data reader used to read block source information. + /// + private readonly SqliteDataReader m_reader; + + /// + /// Initializes a new instance of the class. + /// + /// The SqliteDataReader used to read block source information. + public BlockSource(SqliteDataReader rd) + { + m_reader = rd; + } + + public string Path { get { return m_reader.ConvertValueToString(6) ?? ""; } } + public long Offset { get { return m_reader.ConvertValueToInt64(7); } } + public bool IsMetadata { get { return false; } } + } + + /// + /// The data reader used to read block descriptor information. + /// + private readonly SqliteDataReader m_reader; + + /// + /// Initializes a new instance of the class. + /// + /// The SqliteDataReader used to read block descriptor information. + public BlockDescriptor(SqliteDataReader rd) + { + m_reader = rd; + HasMore = true; + } + + /// + /// Gets the target path of the block source. + /// + private string TargetPath { get { return m_reader.ConvertValueToString(0) ?? ""; } } + + public string Hash { get { return m_reader.ConvertValueToString(2) ?? ""; } } + public long Offset { get { return m_reader.ConvertValueToInt64(3); } } + public long Index { get { return m_reader.ConvertValueToInt64(4); } } + public long Size { get { return m_reader.ConvertValueToInt64(5); } } + public bool IsMetadata { get { return !(m_reader.ConvertValueToInt64(9) == 0); } } + + /// + /// Indicates whether there are more block sources to read for the current block descriptor. + /// + public bool HasMore { get; private set; } + + /// + public async IAsyncEnumerable BlockSources([EnumeratorCancellation] CancellationToken token) + { + var p = TargetPath; + var h = Hash; + var s = Size; + + while (HasMore && p == TargetPath && h == Hash && s == Size) + { + yield return new BlockSource(m_reader); + HasMore = await m_reader.ReadAsync(token).ConfigureAwait(false); + } + } + + } + + /// + /// The data reader used to read local block source information. + /// + private readonly SqliteDataReader m_reader; + + /// + /// Initializes a new instance of the class. + /// + /// The SqliteDataReader used to read local block source information. + public LocalBlockSource(SqliteDataReader rd) + { + m_reader = rd; + HasMore = true; + } + + public string TargetPath { get { return m_reader.ConvertValueToString(0) ?? ""; } } + public long TargetFileID { get { return m_reader.ConvertValueToInt64(1); } } + + /// + /// Indicates whether there are more blocks to read for the current file. + /// + public bool HasMore { get; private set; } + + /// + public async IAsyncEnumerable Blocks([EnumeratorCancellation] CancellationToken token) + { + var p = TargetPath; + while (HasMore && p == TargetPath) + { + var c = new BlockDescriptor(m_reader); + var h = c.Hash; + var s = c.Size; + + yield return c; + + HasMore = c.HasMore; + while (HasMore && c.Hash == h && c.Size == s && TargetPath == p) + HasMore = await m_reader.ReadAsync(token).ConfigureAwait(false); + } + } + + /// + /// Retrieves files and their source blocks from the database. + /// + /// The LocalDatabase instance to query. + /// The name of the file table to query. + /// The name of the block table to query. + /// The size of each block in bytes. + /// If true, skips metadata blocks when retrieving files and source blocks. + /// A cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of ILocalBlockSource objects representing the files and their source blocks. + public static async IAsyncEnumerable GetFilesAndSourceBlocks(LocalDatabase db, string filetablename, string blocktablename, long blocksize, bool skipMetadata, [EnumeratorCancellation] CancellationToken token) + { + // TODO: Skip metadata as required + // Have to order by target path and hash, to ensure BlockDescriptor and BlockSource match adjacent rows + await using var cmd = db.Connection.CreateCommand($@" + SELECT DISTINCT + ""A"".""TargetPath"", + ""A"".""ID"", + ""B"".""Hash"", + (""B"".""Index"" * {blocksize}), + ""B"".""Index"", + ""B"".""Size"", + ""C"".""Path"", + (""D"".""Index"" * {blocksize}), + ""E"".""Size"", + ""B"".""Metadata"" + FROM + ""{filetablename}"" ""A"", + ""{blocktablename}"" ""B"", + ""File"" ""C"", + ""BlocksetEntry"" ""D"", + ""Block"" ""E"" + WHERE + ""A"".""ID"" = ""B"".""FileID"" + AND ""C"".""BlocksetID"" = ""D"".""BlocksetID"" + AND ""D"".""BlockID"" = ""E"".""ID"" + AND ""B"".""Hash"" = ""E"".""Hash"" + AND ""B"".""Size"" = ""E"".""Size"" + AND ""B"".""Restored"" = 0 + ORDER BY + ""A"".""TargetPath"", + ""B"".""Index"" + ") + .SetTransaction(db.Transaction); + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + var more = await rd.ReadAsync(token).ConfigureAwait(false); + + while (more) + { + var f = new LocalBlockSource(rd); + string current = f.TargetPath; + yield return f; + + more = f.HasMore; + while (more && current == f.TargetPath) + more = await rd.ReadAsync(token).ConfigureAwait(false); + } + } + } + + /// + /// Retrieves files and their source blocks from the temporary file and block tables. + /// + /// If true, skips metadata blocks when retrieving files and source blocks. + /// The size of each block in bytes. + /// A cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of ILocalBlockSource objects representing the files and their source blocks. + public IAsyncEnumerable GetFilesAndSourceBlocks(bool skipMetadata, long blocksize, CancellationToken token) + { + if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) + throw new InvalidOperationException("No temporary file table set up for this restore."); + return LocalBlockSource.GetFilesAndSourceBlocks(this, m_tempfiletable, m_tempblocktable, blocksize, skipMetadata, token); + } + + /// + /// Retrieves the missing remote volumes that need to be restored. + /// + /// A cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of IRemoteVolume objects representing the missing remote volumes. + public async IAsyncEnumerable GetMissingVolumes([EnumeratorCancellation] CancellationToken token) + { + await using var cmd = m_connection.CreateCommand($@" + SELECT + ""RV"".""Name"", + ""RV"".""Hash"", + ""RV"".""Size"", + ""BB"".""MaxIndex"" + FROM ""RemoteVolume"" ""RV"" + INNER JOIN ( + SELECT + ""TB"".""VolumeID"", + MAX(""TB"".""Index"") as ""MaxIndex"" + FROM ""{m_tempblocktable}"" ""TB"" + WHERE ""TB"".""Restored"" = 0 + GROUP BY ""TB"".""VolumeID"" + ) as ""BB"" + ON ""RV"".""ID"" = ""BB"".""VolumeID"" + ORDER BY ""BB"".""MaxIndex"" + ") + .SetTransaction(m_rtr); + + // Return order from SQLite-DISTINCT is likely to be sorted by Name, which is bad for restore. + // If the end of very large files (e.g. iso's) is restored before the beginning, most OS write out zeros to fill the file. + // If we manage to get the volumes in an order restoring front blocks first, this can save time. + // An optimal algorithm would build a dependency net with cycle resolution to find the best near topological + // order of volumes, but this is a bit too fancy here. + // We will just put a very simple heuristic to work, that will try to prefer volumes containing lower block indexes: + // We just order all volumes by the maximum block index they contain. This query is slow, but should be worth the effort. + // Now it is likely to restore all files from front to back. Large files will always be done last. + // One could also use like the average block number in a volume, that needs to be measured. + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + { + yield return new RemoteVolume( + rd.ConvertValueToString(0), + rd.ConvertValueToString(1), + rd.ConvertValueToInt64(2, -1) + ); + } + } + + /// + /// Interface for retrieving files and metadata with missing blocks. + /// + public interface IFilesAndMetadata : IDisposable, IAsyncDisposable + { + /// + /// Retrieves files with missing blocks. + /// + /// A cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of IVolumePatch objects representing the files with missing blocks. + IAsyncEnumerable FilesWithMissingBlocks(CancellationToken token); + /// + /// Retrieves metadata with missing blocks. + /// + /// A cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of IVolumePatch objects representing the metadata with missing blocks. + IAsyncEnumerable MetadataWithMissingBlocks(CancellationToken token); + } + + private class FilesAndMetadata : IFilesAndMetadata + { + private string m_tmptable = null!; + private string m_filetablename = null!; + private string m_blocktablename = null!; + private long m_blocksize; + + private LocalDatabase m_db = null!; + + [Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")] + public FilesAndMetadata(SqliteConnection connection, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume) + { + throw new NotImplementedException("Use CreateAsync instead of the constructor"); + } + + private FilesAndMetadata() { } + + /// + /// Creates a new instance of FilesAndMetadata, initializing the temporary table and inserting blocks from the current volume. + /// + /// The LocalDatabase instance to use for database operations. + /// The name of the file table in the database. + /// The name of the block table in the database. + /// The size of each block in bytes. + /// The BlockVolumeReader containing the current volume's blocks. + /// A cancellation token to monitor for cancellation requests. + /// A task that when awaited returns an instance of . + public static async Task CreateAsync(LocalDatabase db, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume, CancellationToken token) + { + var fam = new FilesAndMetadata() + { + m_db = db, + m_filetablename = filetablename, + m_blocktablename = blocktablename, + m_blocksize = blocksize, + }; + + await using var c = db.Connection.CreateCommand() + .SetTransaction(db.Transaction); + fam.m_tmptable = "VolumeFiles-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + await c.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{fam.m_tmptable}"" ( + ""Hash"" TEXT NOT NULL, + ""Size"" INTEGER NOT NULL + ) + ", token) + .ConfigureAwait(false); + + c.SetCommandAndParameters($@" + INSERT INTO ""{fam.m_tmptable}"" ( + ""Hash"", + ""Size"" + ) + VALUES ( + @Hash, + @Size + ) + "); + foreach (var s in curvolume.Blocks) + { + await c.SetParameterValue("@Hash", s.Key) + .SetParameterValue("@Size", s.Value) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } + + // The index _HashSizeIndex is not needed anymore. Index on "Blocks-..." is used on Join in GetMissingBlocks + + await db.Transaction.CommitAsync(token: token).ConfigureAwait(false); + + return fam; + } + + public void Dispose() + { + DisposeAsync().AsTask().Await(); + } + + public async ValueTask DisposeAsync() + { + if (m_tmptable != null) + { + await using var c = m_db.Connection.CreateCommand(@$"DROP TABLE IF EXISTS ""{m_tmptable}""") + .SetTransaction(m_db.Transaction); + await c.ExecuteNonQueryAsync().ConfigureAwait(false); + await m_db.Transaction.CommitAsync().ConfigureAwait(false); + } + } + + /// + /// Implements the IVolumePatch interface to provide access to files with missing blocks. + /// + private class VolumePatch : IVolumePatch + { + /// + /// Implements the IPatchBlock interface to provide access to blocks in a volume. + /// + private class PatchBlock : IPatchBlock + { + /// + /// The data reader used to read patch block information. + /// + private readonly SqliteDataReader m_reader; + /// + /// Initializes a new instance of the class. + /// + public PatchBlock(SqliteDataReader rd) + { + m_reader = rd; + } + + public long Offset { get { return m_reader.ConvertValueToInt64(2); } } + public long Size { get { return m_reader.ConvertValueToInt64(3); } } + public string Key { get { return m_reader.ConvertValueToString(4) ?? ""; } } + } + + /// + /// The data reader used to read volume patch information. + /// + private readonly SqliteDataReader m_reader; + /// + /// Initializes a new instance of the class. + /// + public VolumePatch(SqliteDataReader rd) + { + m_reader = rd; + HasMore = true; + } + + public string Path { get { return m_reader.ConvertValueToString(0) ?? ""; } } + public long FileID { get { return m_reader.ConvertValueToInt64(1); } } + public bool HasMore { get; private set; } + + /// + public async IAsyncEnumerable Blocks([EnumeratorCancellation] CancellationToken token) + { + string p = Path; + while (HasMore && p == Path) + { + yield return new PatchBlock(m_reader); + HasMore = await m_reader.ReadAsync(token).ConfigureAwait(false); + } + } + } + + /// + /// Retrieves the files with missing blocks. + /// + /// A cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of IVolumePatch objects representing the files with missing blocks. + public async IAsyncEnumerable FilesWithMissingBlocks([EnumeratorCancellation] CancellationToken token) + { + // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! + await using var cmd = m_db.Connection.CreateCommand($@" + SELECT DISTINCT + ""A"".""TargetPath"", + ""BB"".""FileID"", + (""BB"".""Index"" * {m_blocksize}), + ""BB"".""Size"", + ""BB"".""Hash"" + FROM + ""{m_filetablename}"" ""A"", + ""{m_blocktablename}"" ""BB"" + WHERE + ""A"".""ID"" = ""BB"".""FileID"" + AND ""BB"".""Restored"" = 0 + AND ""BB"".""Metadata"" = {"0"} + AND ""BB"".""ID"" IN ( + SELECT ""B"".""ID"" + FROM + ""{m_blocktablename}"" ""B"", + ""{m_tmptable}"" ""C"" + WHERE + ""B"".""Hash"" = ""C"".""Hash"" + AND ""B"".""Size"" = ""C"".""Size"" + ) + ORDER BY + ""A"".""TargetPath"", + ""BB"".""Index"" + ") + .SetTransaction(m_db.Transaction); + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + if (await rd.ReadAsync(token).ConfigureAwait(false)) + { + var more = true; + while (more) + { + var f = new VolumePatch(rd); + var current = f.Path; + yield return f; + + more = f.HasMore; + while (more && current == f.Path) + more = await rd.ReadAsync(token).ConfigureAwait(false); + } + } + } + + /// + /// Retrieves the metadata with missing blocks. + /// + /// A cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of IVolumePatch objects representing the metadata with missing blocks. + public async IAsyncEnumerable MetadataWithMissingBlocks([EnumeratorCancellation] CancellationToken token) + { + // The IN-clause with subquery enables SQLite to use indexes better. Three way join (A,B,C) is slow here! + await using var cmd = m_db.Connection.CreateCommand($@" + SELECT DISTINCT + ""A"".""TargetPath"", + ""BB"".""FileID"", + (""BB"".""Index"" * {m_blocksize}), + ""BB"".""Size"", + ""BB"".""Hash"" + FROM + ""{m_filetablename}"" ""A"", + ""{m_blocktablename}"" ""BB"" + WHERE + ""A"".""ID"" = ""BB"".""FileID"" + AND ""BB"".""Restored"" = 0 + AND ""BB"".""Metadata"" = {"1"} + AND ""BB"".""ID"" IN ( + SELECT ""B"".""ID"" + FROM + ""{m_blocktablename}"" ""B"", + ""{m_tmptable}"" ""C"" + WHERE + ""B"".""Hash"" = ""C"".""Hash"" + AND ""B"".""Size"" = ""C"".""Size"" + ) + ORDER BY + ""A"".""TargetPath"", + ""BB"".""Index"" + ") + .SetTransaction(m_db.Transaction); + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + if (await rd.ReadAsync(token).ConfigureAwait(false)) + { + var more = true; + while (more) + { + var f = new VolumePatch(rd); + string current = f.Path; + yield return f; + + more = f.HasMore; + while (more && current == f.Path) + more = await rd.ReadAsync(token).ConfigureAwait(false); + } + } + } + } + + /// + /// Retrieves files and metadata with missing blocks for the given volume. + /// + /// The size of each block in bytes. + /// The current volume reader containing the blocks to restore. + /// A cancellation token to monitor for cancellation requests. + /// A task that when awaited returns an IFilesAndMetadata instance containing the files and metadata with missing blocks. + public async Task GetMissingBlockData(BlockVolumeReader curvolume, long blocksize, CancellationToken token) + { + if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) + throw new InvalidOperationException("No temporary file table set up for this restore."); + return await FilesAndMetadata.CreateAsync(this, m_tempfiletable, m_tempblocktable, blocksize, curvolume, token) + .ConfigureAwait(false); + } + + /// + /// Returns a connection and transaction from the connection pool. + /// + /// A cancellation token to monitor for cancellation requests. + /// A task that when awaited returns a tuple containing the SqliteConnection and ReusableTransaction. + public async Task<(SqliteConnection, ReusableTransaction)> GetConnectionFromPool(CancellationToken token) + { + if (!m_connection_pool.TryTake(out var entry)) + { + var connection = await SQLiteLoader.LoadConnectionAsync() + .ConfigureAwait(false); + + connection.ConnectionString = m_connection.ConnectionString + ";Cache=Shared"; + await connection.OpenAsync(token).ConfigureAwait(false); + await SQLiteLoader.ApplyCustomPragmasAsync(connection) + .ConfigureAwait(false); + + var transaction = new ReusableTransaction(connection); + + return (connection, transaction); + } + + return entry; + } + + /// + /// Represents a file to restore, containing its path, hash, and length. + /// + private class FileToRestore : IFileToRestore + { + public string Path { get; private set; } + public string Hash { get; private set; } + public long Length { get; private set; } + + /// + /// Initializes a new instance of the class. + /// + /// The ID of the file to restore. + /// The path of the file to restore. + /// The hash of the file to restore. + /// The length of the file to restore in bytes. + public FileToRestore(long id, string path, string hash, long length) + { + Path = path; + Hash = hash; + Length = length; + } + } + + /// + /// Returns a list of files to restore, including their target paths, hashes, and lengths. + /// + /// If true, only returns files that have not been verified. + /// A cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of IFileToRestore objects representing the files to restore. + public async IAsyncEnumerable GetFilesToRestore(bool onlyNonVerified, [EnumeratorCancellation] CancellationToken token) + { + await using var cmd = m_connection.CreateCommand($@" + SELECT + ""{m_tempfiletable}"".""ID"", + ""{m_tempfiletable}"".""TargetPath"", + ""Blockset"".""FullHash"", + ""Blockset"".""Length"" + FROM + ""{m_tempfiletable}"", + ""Blockset"" + WHERE + ""{m_tempfiletable}"".""BlocksetID"" = ""Blockset"".""ID"" + AND ""{m_tempfiletable}"".""DataVerified"" <= @Verified + ") + .SetTransaction(m_rtr) + .SetParameterValue("@Verified", !onlyNonVerified); + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + yield return new FileToRestore( + rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? "", rd.ConvertValueToString(2) ?? "", rd.ConvertValueToInt64(3)); + } + + /// + /// Returns a list of files and symlinks to restore. + /// + /// At its current state, this method is designed to be called by Duplicati.Library.Main.Operation.Restore.FileLister. It locks the database to ensure that calls from Duplicati.Library.Main.Operation.Restore.BlockManager do not interfere with each other. + /// A cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of FileRequest objects representing the files and symlinks to restore. + public async IAsyncEnumerable GetFilesAndSymlinksToRestore([EnumeratorCancellation] CancellationToken token) + { + await m_dbLock.WaitAsync(token).ConfigureAwait(false); + try + { + // Order by length descending, so that larger files are restored first. + await using var cmd = m_connection.CreateCommand($@" + SELECT + ""F"".""ID"", + ""F"".""Path"", + ""F"".""TargetPath"", + IFNULL(""B"".""FullHash"", ''), + IFNULL(""B"".""Length"", 0) AS ""Length"", + ""F"".""BlocksetID"" + FROM ""{m_tempfiletable}"" ""F"" + LEFT JOIN ""Blockset"" ""B"" + ON ""F"".""BlocksetID"" = ""B"".""ID"" + WHERE ""F"".""BlocksetID"" != {FOLDER_BLOCKSET_ID} + ORDER BY ""Length"" DESC + ") + .SetTransaction(m_rtr); + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + yield return new FileRequest( + rd.ConvertValueToInt64(0), + rd.ConvertValueToString(1), + rd.ConvertValueToString(2), + rd.ConvertValueToString(3), + rd.ConvertValueToInt64(4), + rd.ConvertValueToInt64(5) + ); + } + finally + { + m_dbLock.Release(); + } + } + + /// + /// Returns a list of folders to restore. Used to restore folder metadata. + /// + /// At its current state, this method is designed to be called by Duplicati.Library.Main.Operation.Restore.FileLister. It locks the database to ensure that calls from Duplicati.Library.Main.Operation.Restore.BlockManager do not interfere with each other. + /// A cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of FileRequest objects representing the folders to restore. + public async IAsyncEnumerable GetFolderMetadataToRestore([EnumeratorCancellation] CancellationToken token) + { + await m_dbLock.WaitAsync(token).ConfigureAwait(false); + + try + { + await using var cmd = m_connection.CreateCommand(); + cmd.SetTransaction(m_rtr); + await using var rd = await cmd.ExecuteReaderAsync($@" + SELECT + ""F"".""ID"", + '', + ""F"".""TargetPath"", + '', + 0, + {FOLDER_BLOCKSET_ID} + FROM ""{m_tempfiletable}"" ""F"" + WHERE + ""F"".""BlocksetID"" = {FOLDER_BLOCKSET_ID} + AND ""F"".""MetadataID"" IS NOT NULL + AND ""F"".""MetadataID"" >= 0 + ", token) + .ConfigureAwait(false); + + while (await rd.ReadAsync(token).ConfigureAwait(false)) + yield return new FileRequest( + rd.ConvertValueToInt64(0), + rd.ConvertValueToString(1), + rd.ConvertValueToString(2), + rd.ConvertValueToString(3), + rd.ConvertValueToInt64(4), + rd.ConvertValueToInt64(5) + ); + } + finally + { + m_dbLock.Release(); + } + } + + /// + /// Returns a list of blocks and their volume IDs. Used by the to keep track of blocks and volumes to automatically evict them from the respective caches. + /// + /// At its current state, this method is designed to be called by Duplicati.Library.Main.Operation.Restore.BlockManager. It locks the database to ensure that calls from Duplicati.Library.Main.Operation.Restore.FileLister do not interfere with each other. + /// Flag indicating whether the returned blocks should exclude the metadata blocks. + /// A cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of tuples containing the block ID and volume ID. + public async IAsyncEnumerable<(long, long)> GetBlocksAndVolumeIDs(bool skipMetadata, [EnumeratorCancellation] CancellationToken token) + { + await m_dbLock.WaitAsync(token).ConfigureAwait(false); + + try + { + var metadata_query = skipMetadata ? "" : $@" + UNION ALL + SELECT + ""Block"".""ID"", + ""Block"".""VolumeID"" + FROM ""{m_tempfiletable}"" + INNER JOIN ""Metadataset"" + ON ""{m_tempfiletable}"".""MetadataID"" = ""Metadataset"".""ID"" + INNER JOIN ""BlocksetEntry"" + ON ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + INNER JOIN ""Block"" + ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + "; + + await using var cmd = m_connection.CreateCommand($@" + SELECT + ""Block"".""ID"", + ""Block"".""VolumeID"" + FROM ""BlocksetEntry"" + INNER JOIN ""{m_tempfiletable}"" + ON ""BlocksetEntry"".""BlocksetID"" = ""{m_tempfiletable}"".""BlocksetID"" + INNER JOIN ""Block"" + ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + {metadata_query} + ") + .SetTransaction(m_rtr); + + await using var reader = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await reader.ReadAsync(token).ConfigureAwait(false)) + yield return ( + reader.ConvertValueToInt64(0), + reader.ConvertValueToInt64(1) + ); + } + finally + { + m_dbLock.Release(); + } + } + + /// + /// Returns a list of for the given blockset ID. It is used by the to restore the blocks of a file. + /// + /// The BlocksetID of the file. + /// A cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of representing the blocks of the file. + public async IAsyncEnumerable GetBlocksFromFile(long blocksetID, [EnumeratorCancellation] CancellationToken token) + { + var (connection, transaction) = await GetConnectionFromPool(token) + .ConfigureAwait(false); + + try + { + await using var cmd = connection.CreateCommand(@$" + SELECT + ""Block"".""ID"", + ""Block"".""Hash"", + ""Block"".""Size"", + ""Block"".""VolumeID"" + FROM ""BlocksetEntry"" + INNER JOIN ""Block"" + ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + WHERE ""BlocksetEntry"".""BlocksetID"" = @BlocksetID + ORDER BY ""BlocksetEntry"".""Index"" + ") + .SetTransaction(transaction) + .SetParameterValue("@BlocksetID", blocksetID); + + await using var reader = await cmd.ExecuteReaderAsync(token) + .ConfigureAwait(false); + for (long i = 0; await reader.ReadAsync(token).ConfigureAwait(false); i++) + yield return new BlockRequest( + reader.ConvertValueToInt64(0), + i, + reader.ConvertValueToString(1), + reader.ConvertValueToInt64(2), + reader.ConvertValueToInt64(3), + false + ); + } + finally + { + // Return the connection to the pool + m_connection_pool.Add((connection, transaction)); + } + } + + /// + /// Returns a list of for the metadata blocks of the given file. It is used by the to restore the metadata of a file. + /// + /// The ID of the file. + /// A cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of representing the metadata blocks of the file. + public async IAsyncEnumerable GetMetadataBlocksFromFile(long fileID, [EnumeratorCancellation] CancellationToken token) + { + var (connection, transaction) = await GetConnectionFromPool(token) + .ConfigureAwait(false); + + try + { + await using var cmd = connection.CreateCommand($@" + SELECT + ""Block"".""ID"", + ""Block"".""Hash"", + ""Block"".""Size"", + ""Block"".""VolumeID"" + FROM ""File"" + INNER JOIN ""Metadataset"" + ON ""File"".""MetadataID"" = ""Metadataset"".""ID"" + INNER JOIN ""BlocksetEntry"" + ON ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" + INNER JOIN ""Block"" + ON ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" + WHERE ""File"".""ID"" = @FileID + ") + .SetTransaction(transaction) + .SetParameterValue("@FileID", fileID); + + await using var reader = await cmd.ExecuteReaderAsync(token) + .ConfigureAwait(false); + for (long i = 0; await reader.ReadAsync(token).ConfigureAwait(false); i++) + { + yield return new BlockRequest( + reader.ConvertValueToInt64(0), + i, + reader.ConvertValueToString(1), + reader.ConvertValueToInt64(2), + reader.ConvertValueToInt64(3), + false + ); + } + } + finally + { + // Return the connection to the pool + m_connection_pool.Add((connection, transaction)); + } + } + + /// + /// Returns the volume information for the given volume ID. It is used by the to get the volume information for the given volume ID. + /// + /// The ID of the volume. + /// A cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of tuples containing the volume name, size, and hash. + public async IAsyncEnumerable<(string, long, string)> GetVolumeInfo(long VolumeID, [EnumeratorCancellation] CancellationToken token) + { + var (connection, transaction) = await GetConnectionFromPool(token) + .ConfigureAwait(false); + + try + { + await using var cmd = connection.CreateCommand(@" + SELECT + ""Name"", + ""Size"", + ""Hash"" + FROM ""RemoteVolume"" + WHERE ""ID"" = @VolumeID + ") + .SetTransaction(transaction) + .SetParameterValue("@VolumeID", VolumeID); + + await using var reader = await cmd.ExecuteReaderAsync(token) + .ConfigureAwait(false); + while (await reader.ReadAsync(token).ConfigureAwait(false)) + yield return ( + reader.ConvertValueToString(0) ?? "", + reader.ConvertValueToInt64(1), + reader.ConvertValueToString(2) ?? "" + ); + } + finally + { + m_connection_pool.Add((connection, transaction)); + } + } + + /// + /// Drops all temporary tables used for the restore process and commits the transaction. + /// + /// A cancellation token to monitor for cancellation requests. + /// A task that completes when the temporary tables are dropped and the transaction is committed. + public async Task DropRestoreTable(CancellationToken token) + { + await using var cmd = m_connection.CreateCommand() + .SetTransaction(m_rtr); + + if (m_tempfiletable != null) + try + { + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempfiletable}""", token) + .ConfigureAwait(false); + } + catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } + finally { m_tempfiletable = null; } + + if (m_tempblocktable != null) + try + { + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tempblocktable}""", token) + .ConfigureAwait(false); + } + catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } + finally { m_tempblocktable = null; } + + if (m_latestblocktable != null) + try + { + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_latestblocktable}""", token) + .ConfigureAwait(false); + } + catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } + finally { m_latestblocktable = null; } + + if (m_fileprogtable != null) + try + { + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_fileprogtable}""", token) + .ConfigureAwait(false); + } + catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } + finally { m_fileprogtable = null; } + + if (m_totalprogtable != null) + try + { + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_totalprogtable}""", token) + .ConfigureAwait(false); + } + catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } + finally { m_totalprogtable = null; } + + if (m_filesnewlydonetable != null) + try + { + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_filesnewlydonetable}""", token) + .ConfigureAwait(false); + } + catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CleanupError", ex, "Cleanup error: {0}", ex.Message); } + finally { m_filesnewlydonetable = null; } + + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); + } + + /// + /// Interface for marking blocks as restored, missing, or verified. + /// + public interface IBlockMarker : IDisposable, IAsyncDisposable + { + /// + /// Marks a specific block as restored. + /// + /// The ID of the target file. + /// The index of the block. + /// The hash of the block. + /// The size of the block. + /// Indicates whether the block is metadata. + /// A cancellation token to monitor for cancellation requests. + /// A task that completes when the block is marked as restored. + Task SetBlockRestored(long targetfileid, long index, string hash, long blocksize, bool metadata, CancellationToken token); + + /// + /// Marks all blocks of a specific file as missing. + /// + /// The ID of the target file. + /// A cancellation token to monitor for cancellation requests. + /// A task that completes when all blocks of the file are marked as missing. + Task SetAllBlocksMissing(long targetfileid, CancellationToken token); + + /// + /// Marks all blocks of a specific file as restored. + /// + /// The ID of the target file. + /// Indicates whether to include metadata blocks. + /// A cancellation token to monitor for cancellation requests. + /// A task that completes when all blocks of the file are marked as restored. + Task SetAllBlocksRestored(long targetfileid, bool includeMetadata, CancellationToken token); + + /// + /// Marks the data of a specific file as verified. + /// + /// The ID of the target file. + /// A cancellation token to monitor for cancellation requests. + /// A task that completes when the file data is marked as verified. + Task SetFileDataVerified(long targetfileid, CancellationToken token); + + /// + /// Commits the changes made by the block marker. + /// + /// A cancellation token to monitor for cancellation requests. + /// A task that completes when the changes are committed. + Task CommitAsync(CancellationToken token); + + /// + /// Updates the processed statistics for the blocks and files. + /// + /// An operation progress updater to report the processed statistics. + /// A cancellation token to monitor for cancellation requests. + /// A task that completes when the processed statistics are updated. + Task UpdateProcessed(IOperationProgressUpdater writer, CancellationToken token); + } + + /// + /// A new implementation of IBlockMarker, marking the blocks directly in the blocks table as restored + /// and reading statistics about progress from DB (kept up-to-date by triggers). + /// There is no negative influence on performance, esp. since the block table is temporary anyway. + /// + private class DirectBlockMarker : IBlockMarker + { + /// + /// Updates the "Restored" status to 1 for a specific block in the block table, matching by file ID, index, hash, size, and metadata, but only if it is currently not restored. + /// + private SqliteCommand m_insertblockCommand = null!; + + /// + /// Resets the "Restored" status to 0 for all blocks belonging to a specific file in the block table. + /// + private SqliteCommand m_resetfileCommand = null!; + + /// + /// Updates the "Restored" status to 1 for all blocks of a file in the block table, optionally including metadata blocks depending on the parameter. + /// + private SqliteCommand m_updateAsRestoredCommand = null!; + + /// + /// Marks a file as data verified by setting the "DataVerified" field to 1 in the file table for a specific file ID. + /// + private SqliteCommand m_updateFileAsDataVerifiedCommand = null!; + + /// + /// Retrieves the number of fully restored files and the total restored size from the progress/statistics table, or, if unavailable, counts restored blocks and their size directly from the block table. + /// + private SqliteCommand m_statUpdateCommand = null!; + + /// + /// The database instance used for executing commands. + /// + private LocalDatabase m_db = null!; + + /// + /// Indicates whether there are updates to be processed. + /// + private bool m_hasUpdates = false; + + /// + /// The name of the temporary block table used for marking blocks as restored. + /// + private string m_blocktablename = null!; + /// + /// The name of the temporary file table used for marking files as data verified. + /// + private string m_filetablename = null!; + + /// + /// Calling this constructor will throw an exception. Use CreateAsync instead. + /// + [Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")] + public DirectBlockMarker(SqliteConnection connection, string blocktablename, string filetablename, string statstablename) + { + throw new NotImplementedException("Use CreateAsync instead of the constructor"); + } + + /// + /// Private constructor to prevent direct instantiation. Use CreateAsync to create an instance. + /// + private DirectBlockMarker() { } + + /// + /// Creates a new instance of the class asynchronously. + /// + /// The local database instance to use for executing commands. + /// The name of the temporary block table. + /// The name of the temporary file table. + /// The name of the statistics table, or null if not available. + /// A cancellation token to monitor for cancellation requests. + /// A task that when awaited returns a new instance of the class. + public static async Task CreateAsync(LocalDatabase db, string blocktablename, string filetablename, string statstablename, CancellationToken token) + { + var dbm = new DirectBlockMarker() + { + m_db = db, + m_blocktablename = blocktablename, + m_filetablename = filetablename, + + m_insertblockCommand = await db.Connection.CreateCommandAsync($@" + UPDATE ""{blocktablename}"" SET ""Restored"" = 1 + WHERE + ""FileID"" = @TargetFileId + AND ""Index"" = @Index + AND ""Hash"" = @Hash + AND ""Size"" = @Size + AND ""Metadata"" = @Metadata + AND ""Restored"" = 0 + ", token) + .ConfigureAwait(false), + + m_resetfileCommand = await db.Connection.CreateCommandAsync($@" + UPDATE ""{blocktablename}"" + SET ""Restored"" = 0 + WHERE ""FileID"" = @TargetFileId + ", token) + .ConfigureAwait(false), + + m_updateAsRestoredCommand = await db.Connection.CreateCommandAsync($@" + UPDATE ""{blocktablename}"" + SET ""Restored"" = 1 + WHERE + ""FileID"" = @TargetFileId + AND ""Metadata"" <= @Metadata + ", token) + .ConfigureAwait(false), + + m_updateFileAsDataVerifiedCommand = await db.Connection.CreateCommandAsync($@" + UPDATE ""{filetablename}"" + SET ""DataVerified"" = 1 + WHERE ""ID"" = @TargetFileId + ", token) + .ConfigureAwait(false), + + m_statUpdateCommand = statstablename == null ? + // very slow fallback if stats tables were not created + await db.Connection.CreateCommandAsync($@" + SELECT + COUNT(DISTINCT ""FileID""), + SUM(""Size"") + FROM ""{blocktablename}"" + WHERE ""Restored"" = 1 + ", token) + .ConfigureAwait(false) + : + // Fields in Stats: TotalFiles, TotalBlocks, TotalSize + // FilesFullyRestored, FilesPartiallyRestored, BlocksRestored, SizeRestored + await db.Connection.CreateCommandAsync($@" + SELECT + SUM(""FilesFullyRestored""), + SUM(""SizeRestored"") + FROM ""{statstablename}"" + ", token) + .ConfigureAwait(false) + }; + + return dbm; + } + + /// + public async Task UpdateProcessed(IOperationProgressUpdater updater, CancellationToken token) + { + if (!m_hasUpdates) + return; + + m_hasUpdates = false; + await using var rd = await m_statUpdateCommand + .SetTransaction(m_db.Transaction) + .ExecuteReaderAsync(token) + .ConfigureAwait(false); + + var filesprocessed = 0L; + var processedsize = 0L; + + if (rd.Read()) + { + filesprocessed += rd.ConvertValueToInt64(0, 0); + processedsize += rd.ConvertValueToInt64(1, 0); + } + + updater.UpdatefilesProcessed(filesprocessed, processedsize); + } + + // + public async Task SetAllBlocksMissing(long targetfileid, CancellationToken token) + { + m_hasUpdates = true; + m_resetfileCommand + .SetTransaction(m_db.Transaction) + .SetParameterValue("@TargetFileId", targetfileid); + + var r = await m_resetfileCommand.ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + if (r <= 0) + throw new Exception("Unexpected reset result"); + } + + /// + public async Task SetAllBlocksRestored(long targetfileid, bool includeMetadata, CancellationToken token) + { + m_hasUpdates = true; + m_updateAsRestoredCommand + .SetTransaction(m_db.Transaction) + .SetParameterValue("@TargetFileId", targetfileid) + .SetParameterValue("@Metadata", includeMetadata ? 1 : 0); + + var r = await m_updateAsRestoredCommand + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + if (r <= 0) + throw new Exception("Unexpected reset result"); + } + + /// + public async Task SetFileDataVerified(long targetfileid, CancellationToken token) + { + m_hasUpdates = true; + m_updateFileAsDataVerifiedCommand + .SetTransaction(m_db.Transaction) + .SetParameterValue("@TargetFileId", targetfileid); + + var r = await m_updateFileAsDataVerifiedCommand + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + if (r != 1) + throw new Exception("Unexpected result when marking file as verified."); + } + + /// + public async Task SetBlockRestored(long targetfileid, long index, string hash, long size, bool metadata, CancellationToken token) + { + m_hasUpdates = true; + m_insertblockCommand + .SetTransaction(m_db.Transaction) + .SetParameterValue("@TargetFileId", targetfileid) + .SetParameterValue("@Index", index) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .SetParameterValue("@Metadata", metadata); + + var r = await m_insertblockCommand + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + if (r != 1) + throw new Exception("Unexpected result when marking block."); + } + + /// + public async Task CommitAsync(CancellationToken token) + { + using (new Logging.Timer(LOGTAG, "CommitBlockMarker", "CommitBlockMarker")) + await m_db.Transaction.CommitAsync(token: token).ConfigureAwait(false); + } + + public void Dispose() + { + DisposeAsync().AsTask().Await(); + } + + public async ValueTask DisposeAsync() + { + await m_insertblockCommand.DisposeAsync().ConfigureAwait(false); + await m_resetfileCommand.DisposeAsync().ConfigureAwait(false); + await m_updateAsRestoredCommand.DisposeAsync().ConfigureAwait(false); + await m_updateFileAsDataVerifiedCommand.DisposeAsync().ConfigureAwait(false); + await m_statUpdateCommand.DisposeAsync().ConfigureAwait(false); + } + } + + /// + /// Creates a new block marker for marking blocks as restored, missing, or verified. + /// + /// A cancellation token to monitor for cancellation requests. + /// A task that when awaited returns an instance of . + public async Task CreateBlockMarkerAsync(CancellationToken token) + { + if (string.IsNullOrWhiteSpace(m_tempfiletable) || string.IsNullOrWhiteSpace(m_tempblocktable)) + throw new InvalidOperationException("No temporary file table set up for this restore."); + if (string.IsNullOrWhiteSpace(m_totalprogtable)) + throw new InvalidOperationException("No progress table set up for this restore."); + return await DirectBlockMarker.CreateAsync(this, m_tempblocktable, m_tempfiletable, m_totalprogtable, token) + .ConfigureAwait(false); + } + + public override void Dispose() + { + DisposeAsync().AsTask().Await(); + } + + public override async ValueTask DisposeAsync() + { + await DisposePoolAsync().ConfigureAwait(false); + await DropRestoreTable(default).ConfigureAwait(false); + await base.DisposeAsync().ConfigureAwait(false); + } + + /// + /// Disposes all connections and their transactions in the connection pool. + /// + /// A task that completes when all connections and transactions are disposed. + public async Task DisposePoolAsync() + { + foreach (var (connection, transaction) in m_connection_pool) + { + await transaction.DisposeAsync().ConfigureAwait(false); + await connection.CloseAsync().ConfigureAwait(false); + await connection.DisposeAsync().ConfigureAwait(false); + } + m_connection_pool.Clear(); + } + + /// + /// Returns a list of target folders for the folder blockset. + /// + /// A cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of strings representing the target folders. + public async IAsyncEnumerable GetTargetFolders([EnumeratorCancellation] CancellationToken token) + { + await using var cmd = m_connection.CreateCommand($@" + SELECT ""TargetPath"" + FROM ""{m_tempfiletable}"" + WHERE ""BlocksetID"" == @BlocksetID + ") + .SetTransaction(m_rtr) + .SetParameterValue("@BlocksetID", FOLDER_BLOCKSET_ID); + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + yield return rd.ConvertValueToString(0) ?? ""; + } + + /// + /// Returns a list of files and their source blocks for fast restoration. + /// + public interface IFastSource + { + /// + /// The target path of the file to restore. + /// + string TargetPath { get; } + /// + /// The ID of the target file. + /// + long TargetFileID { get; } + /// + /// The source path of the file to restore. + /// + string SourcePath { get; } + /// + /// Gets an asynchronous enumerable of block entries for the file. + /// + /// A cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of representing the blocks of the file. + IAsyncEnumerable Blocks(CancellationToken token); + } + + /// + /// Interface representing a block entry in the fast source. + /// + public interface IBlockEntry + { + /// + /// The offset of the block in the source file. + /// + long Offset { get; } + /// + /// The size of the block. + /// + long Size { get; } + /// + /// The index of the block in the source file. + /// + long Index { get; } + /// + /// The hash of the block. + /// + string Hash { get; } + } + + /// + /// Implementation of that provides access to file and block information for fast restoration. + /// + private class FastSource : IFastSource + { + /// + /// Implementation of that provides access to block information. + /// + private class BlockEntry : IBlockEntry + { + /// + /// The SqliteDataReader used to read block data. + /// + private readonly SqliteDataReader m_rd; + /// + /// The block size used to calculate the offset. + /// + private readonly long m_blocksize; + + /// + /// Initializes a new instance of the class with the specified SqliteDataReader and block size. + /// + /// The SqliteDataReader used to read block data. + /// The block size used to calculate the offset. + public BlockEntry(SqliteDataReader rd, long blocksize) + { + m_rd = rd; + m_blocksize = blocksize; + } + + public long Offset { get { return m_rd.ConvertValueToInt64(3) * m_blocksize; } } + public long Index { get { return m_rd.ConvertValueToInt64(3); } } + public long Size { get { return m_rd.ConvertValueToInt64(5); } } + public string Hash { get { return m_rd.ConvertValueToString(4) ?? ""; } } + } + + /// + /// The SqliteDataReader used to read file and block data. + /// + private readonly SqliteDataReader m_rd; + /// + /// The block size used to calculate the offset of blocks. + /// + private readonly long m_blocksize; + + /// + /// Initializes a new instance of the class with the specified SqliteDataReader and block size. + /// + /// The SqliteDataReader used to read file and block data. + /// The block size used to calculate the offset of blocks. + public FastSource(SqliteDataReader rd, long blocksize) + { + m_rd = rd; + m_blocksize = blocksize; + MoreData = true; + } + + /// + /// Indicates whether there is more data to read from the SqliteDataReader. + /// + public bool MoreData { get; private set; } + + public string TargetPath { get { return m_rd.ConvertValueToString(0) ?? ""; } } + public long TargetFileID { get { return m_rd.ConvertValueToInt64(2); } } + public string SourcePath { get { return m_rd.ConvertValueToString(1) ?? ""; } } + + /// + public async IAsyncEnumerable Blocks([EnumeratorCancellation] CancellationToken token) + { + var tid = TargetFileID; + + do + { + yield return new BlockEntry(m_rd, m_blocksize); + } while ((MoreData = await m_rd.ReadAsync(token).ConfigureAwait(false)) && tid == TargetFileID); + } + } + + /// + /// Returns a list of files and their source blocks for fast restoration. + /// + /// The size of the blocks to be used for restoration. + /// A cancellation token to monitor for cancellation requests. + /// An asynchronous enumerable of representing the files and their source blocks. + public async IAsyncEnumerable GetFilesAndSourceBlocksFast(long blocksize, [EnumeratorCancellation] CancellationToken token) + { + await using (var cmdReader = m_connection.CreateCommand()) + await using (var cmd = m_connection.CreateCommand()) + { + cmdReader.SetTransaction(m_rtr); + cmd.SetTransaction(m_rtr); + cmd.SetCommandAndParameters($@" + UPDATE ""{m_tempfiletable}"" + SET ""LocalSourceExists"" = 1 + WHERE ""Path"" = @Path + "); + cmdReader.SetCommandAndParameters($@" + SELECT DISTINCT ""{m_tempfiletable}"".""Path"" + FROM ""{m_tempfiletable}"" + "); + await using (var rd = await cmdReader.ExecuteReaderAsync(token).ConfigureAwait(false)) + { + while (await rd.ReadAsync(token).ConfigureAwait(false)) + { + var sourcepath = rd.ConvertValueToString(0); + if (SystemIO.IO_OS.FileExists(sourcepath)) + { + await cmd + .SetParameterValue("@Path", sourcepath) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } + else + { + Logging.Log.WriteVerboseMessage(LOGTAG, "LocalSourceMissing", "Local source file not found: {0}", sourcepath); + } + } + } + + //This localSourceExists index will make the query engine to start by searching FileSet table. As the result is ordered by FileSet.ID, we will get the cursor "instantly" + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{m_tempfiletable}_LocalSourceExists"" + ON ""{m_tempfiletable}"" (""LocalSourceExists"") + ", token) + .ConfigureAwait(false); + + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); + } + + m_latestblocktable = "LatestBlocksetIds-" + m_temptabsetguid; + + var whereclause = $@" + ""{m_tempfiletable}"".""LocalSourceExists"" = 1 + AND ""{m_tempblocktable}"".""Restored"" = 0 + AND ""{m_tempblocktable}"".""Metadata"" = 0 + AND ""{m_tempfiletable}"".""TargetPath"" != ""{m_tempfiletable}"".""Path"" + "; + + var latestBlocksetIds = $@" + SELECT + ""File"".""Path"" AS ""PATH"", + ""File"".""BlocksetID"" AS ""BlocksetID"", + MAX(""Fileset"".""Timestamp"") AS ""Timestamp"" + FROM + ""File"", + ""FilesetEntry"", + ""Fileset"" + WHERE + ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""FilesetEntry"".""FilesetID"" = ""Fileset"".""ID"" + AND ""File"".""Path"" IN ( + SELECT DISTINCT ""{m_tempfiletable}"".""Path"" + FROM + ""{m_tempfiletable}"", + ""{m_tempblocktable}"" + WHERE + ""{m_tempfiletable}"".""ID"" = ""{m_tempblocktable}"".""FileID"" + AND {whereclause} + ) + GROUP BY ""File"".""Path"" + "; + + await using (var cmd = m_connection.CreateCommand()) + { + cmd.SetTransaction(m_rtr); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_latestblocktable}"" ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@"CREATE TEMPORARY TABLE ""{m_latestblocktable}"" AS {latestBlocksetIds}", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@" + CREATE INDEX ""{m_latestblocktable}_path"" + ON ""{m_latestblocktable}"" (""Path"") + ", token) + .ConfigureAwait(false); + + await cmd.ExecuteNonQueryAsync($@" + UPDATE ""{m_tempfiletable}"" + SET ""LatestBlocksetId"" = ( + SELECT ""BlocksetId"" + FROM ""{m_latestblocktable}"" + WHERE ""Path"" = ""{m_tempfiletable}"".""Path"" + ) + ", token) + .ConfigureAwait(false); + } + + var sources = $@" + SELECT DISTINCT + ""{m_tempfiletable}"".""TargetPath"", + ""{m_tempfiletable}"".""Path"", + ""{m_tempfiletable}"".""ID"", + ""{m_tempblocktable}"".""Index"", + ""{m_tempblocktable}"".""Hash"", + ""{m_tempblocktable}"".""Size"" + FROM + ""{m_tempfiletable}"", + ""{m_tempblocktable}"", + ""BlocksetEntry"" + WHERE + ""{m_tempfiletable}"".""ID"" = ""{m_tempblocktable}"".""FileID"" + AND ""BlocksetEntry"".""BlocksetID"" = ""{m_tempfiletable}"".""LatestBlocksetID"" + AND ""BlocksetEntry"".""BlockID"" = ""{m_tempblocktable}"".""BlockID"" + AND ""BlocksetEntry"".""Index"" = ""{m_tempblocktable}"".""Index"" + AND {whereclause} + ORDER BY + ""{m_tempfiletable}"".""ID"", + ""{m_tempblocktable}"".""Index"" + "; + + await using (var cmd = m_connection.CreateCommand()) + { + cmd.SetTransaction(m_rtr); + await using var rd = await cmd.ExecuteReaderAsync(sources, token) + .ConfigureAwait(false); + if (await rd.ReadAsync(token).ConfigureAwait(false)) + { + bool more; + do + { + var n = new FastSource(rd, blocksize); + var tid = n.TargetFileID; + yield return n; + + more = n.MoreData; + while (more && n.TargetFileID == tid) + more = await rd.ReadAsync(token).ConfigureAwait(false); + + } while (more); + } + } + + await m_rtr.CommitAsync(token: token).ConfigureAwait(false); + } + + } +} diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index 9ebb3e75b..89d891ad9 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -1,62 +1,131 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable using System; -using System.Data; using System.Linq; +using System.Threading.Tasks; using System.Collections.Generic; +using Microsoft.Data.Sqlite; +using Duplicati.Library.Utility; +using System.Threading; +using System.Runtime.CompilerServices; namespace Duplicati.Library.Main.Database { + /// + /// Represents a local database used for testing and verification. + /// Provides methods for creating test databases, updating verification counts, and comparing file, index, and block lists + /// to support integrity checks and remote volume verification during backup testing. + /// internal class LocalTestDatabase : LocalDatabase { - public LocalTestDatabase(string path, long pagecachesize) - : base(path, "Test", true, pagecachesize) + /// + /// Initializes a new instance of the class. + /// + /// The path to the database file. + /// An optional existing instance to use. Used to mimic constructor chaining. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that represents the asynchronous operation. The task result contains the created instance. + public static async Task CreateAsync(string path, LocalTestDatabase? dbnew, CancellationToken token) { + dbnew ??= new LocalTestDatabase(); + + dbnew = (LocalTestDatabase) + await CreateLocalDatabaseAsync(path, "Test", true, dbnew, token) + .ConfigureAwait(false); + + dbnew.ShouldCloseConnection = true; + + return dbnew; } - public LocalTestDatabase(LocalDatabase parent) - : base(parent) + /// + /// Creates a new instance of the class using an existing parent database. + /// + /// The parent database to use for creating the new test database. + /// An optional existing instance to use. Used to mimic constructor chaining. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that represents the asynchronous operation. The task result contains the created instance. + public static async Task CreateAsync(LocalDatabase dbparent, LocalTestDatabase? dbnew, CancellationToken token) { + dbnew ??= new LocalTestDatabase(); + + return (LocalTestDatabase) + await CreateLocalDatabaseAsync(dbparent, dbnew, token) + .ConfigureAwait(false); } - public void UpdateVerificationCount(string name, IDbTransaction? tr) + /// + /// Updates the verification count for a remote volume with the specified name. + /// Increments the count by 1, or sets it to the maximum of 1 if it was previously 0 or negative. + /// + /// The name of the remote volume to update. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that represents the asynchronous operation. + public async Task UpdateVerificationCount(string name, CancellationToken token) { - using (var cmd = m_connection.CreateCommand(tr)) - cmd.SetCommandAndParameters(@"UPDATE ""RemoteVolume"" SET ""VerificationCount"" = MAX(1, CASE WHEN ""VerificationCount"" <= 0 THEN (SELECT MAX(""VerificationCount"") FROM ""RemoteVolume"") ELSE ""VerificationCount"" + 1 END) WHERE ""Name"" = @Name") - .SetParameterValue("@Name", name) - .ExecuteNonQuery(); + await using var cmd = m_connection.CreateCommand(m_rtr); + await cmd.SetCommandAndParameters(@" + UPDATE ""RemoteVolume"" + SET ""VerificationCount"" = MAX(1, + CASE + WHEN ""VerificationCount"" <= 0 + THEN ( + SELECT MAX(""VerificationCount"") + FROM ""RemoteVolume"" + ) + ELSE ""VerificationCount"" + 1 + END + ) + WHERE ""Name"" = @Name + ") + .SetParameterValue("@Name", name) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); } + /// + /// A record representing a remote volume, which implements the interface. + /// private record RemoteVolume : IRemoteVolume { + /// + /// Gets the ID of the remote volume. + /// public long ID { get; init; } public string Name { get; init; } public long Size { get; init; } public string Hash { get; init; } + /// + /// Gets the verification count of the remote volume, indicating how many times it has been verified. + /// public long VerificationCount { get; init; } - public RemoteVolume(IDataReader rd) + /// + /// Initializes a new instance of the record from a SqliteDataReader. + /// + /// The SqliteDataReader containing the data for the remote volume. + public RemoteVolume(SqliteDataReader rd) { ID = rd.ConvertValueToInt64(0); Name = rd.ConvertValueToString(1) ?? ""; @@ -66,11 +135,19 @@ namespace Duplicati.Library.Main.Database } } - private IEnumerable FilterByVerificationCount(IEnumerable volumes, long samples, long maxverification) + /// + /// Filters a list of remote volumes based on their verification count. + /// The method selects volumes that have not been verified, those with a low verification count, and finally those with a high verification count, ensuring a balanced selection. + /// + /// The collection of remote volumes to filter. + /// The number of samples to select. + /// The maximum verification count to consider for filtering. + /// A list of remote volumes filtered by verification count. + private static List FilterByVerificationCount(IEnumerable volumes, long samples, long maxverification) { var rnd = new Random(); - // First round is the new items + // First round is the new items var res = (from n in volumes where n.VerificationCount == 0 select n).ToList(); while (res.Count > samples) res.RemoveAt(rnd.Next(0, res.Count)); @@ -118,448 +195,1185 @@ namespace Duplicati.Library.Main.Database return res; } - public IEnumerable SelectTestTargets(long samples, Options options, IDbTransaction? tr) + /// + /// Asynchronously selects a set of remote volumes to be tested for integrity, based on the specified sample count and selection options. + /// This method retrieves candidate remote volumes from the database, prioritizes them according to verification count and state, and yields the selected targets for verification. + /// + /// The number of remote volumes to select for testing. + /// The options that define selection criteria, such as time, version, and verification strategy. + /// A cancellation token to observe while waiting for the task to complete. + /// An asynchronous enumerable of representing the selected test targets. + public async IAsyncEnumerable SelectTestTargets(long samples, Options options, [EnumeratorCancellation] CancellationToken token) { - var tp = GetFilelistWhereClause(options.Time, options.Version); + var tp = await GetFilelistWhereClause(options.Time, options.Version, null, false, token) + .ConfigureAwait(false); samples = Math.Max(1, samples); - using (var cmd = m_connection.CreateCommand(tr)) + await using var cmd = m_connection.CreateCommand(m_rtr); + var files = new List(); + var max = cmd.ExecuteScalarInt64(@" + SELECT MAX(""VerificationCount"") + FROM ""RemoteVolume"" + ", 0); + + if (options.FullRemoteVerification != Options.RemoteTestStrategy.IndexesOnly) { - var files = new List(); - var max = cmd.ExecuteScalarInt64(@"SELECT MAX(""VerificationCount"") FROM ""RemoteVolume""", 0); + // Select any broken items + cmd.SetCommandAndParameters(@" + SELECT + ""ID"", + ""Name"", + ""Size"", + ""Hash"", + ""VerificationCount"" + FROM + ""Remotevolume"" + WHERE + (""State"" IN (@States)) + AND ( + ""Hash"" = '' + OR ""Hash"" IS NULL + OR ""Size"" <= 0 + ) + AND (""ArchiveTime"" = 0) + ") + .ExpandInClauseParameterMssqlite("@States", [ + RemoteVolumeState.Verified.ToString(), + RemoteVolumeState.Uploaded.ToString() + ]); - if (options.FullRemoteVerification != Options.RemoteTestStrategy.IndexesOnly) - { - // Select any broken items - cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"" WHERE (""State"" IN (@States)) AND (""Hash"" = '' OR ""Hash"" IS NULL OR ""Size"" <= 0) AND (""ArchiveTime"" = 0)") - .ExpandInClauseParameter("@States", [RemoteVolumeState.Verified.ToString(), RemoteVolumeState.Uploaded.ToString()]); + await using (var rd = await cmd.ExecuteReaderAsync(token)) + while (await rd.ReadAsync(token)) + yield return new RemoteVolume(rd); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - yield return new RemoteVolume(rd); - - //First we select some filesets - var whereClause = string.IsNullOrEmpty(tp.Item1) ? " WHERE " : (" " + tp.Item1 + " AND "); - using (var rd = cmd.SetCommandAndParameters(@"SELECT ""A"".""VolumeID"", ""A"".""Name"", ""A"".""Size"", ""A"".""Hash"", ""A"".""VerificationCount"" FROM (SELECT ""ID"" AS ""VolumeID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"" WHERE ""ArchiveTime"" = 0 AND ""State"" IN (@State1, @State2)) A, ""Fileset"" " + whereClause + @" ""A"".""VolumeID"" = ""Fileset"".""VolumeID"" ORDER BY ""Fileset"".""Timestamp"" ") - .SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()) - .SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()) - .SetParameterValues(tp.Item2) - .ExecuteReader()) - while (rd.Read()) - files.Add(new RemoteVolume(rd)); - - if (files.Count == 0) - yield break; - - if (string.IsNullOrEmpty(tp.Item1)) - files = FilterByVerificationCount(files, samples, max).ToList(); - - foreach (var f in files) - yield return f; - - //Then we select some index files - files.Clear(); - } - - cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"" WHERE ""Type"" = @Type AND ""State"" IN (@States) AND ""ArchiveTime"" = 0") - .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()) - .ExpandInClauseParameter("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); - - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) + //First we select some filesets + var whereClause = string.IsNullOrEmpty(tp.Item1) ? " WHERE " : (" " + tp.Item1 + " AND "); + await using (var rd = await cmd.SetCommandAndParameters(@$" + SELECT + ""A"".""VolumeID"", + ""A"".""Name"", + ""A"".""Size"", + ""A"".""Hash"", + ""A"".""VerificationCount"" + FROM + ( + SELECT + ""ID"" AS ""VolumeID"", + ""Name"", + ""Size"", + ""Hash"", + ""VerificationCount"" + FROM ""Remotevolume"" + WHERE + ""ArchiveTime"" = 0 + AND ""State"" IN ( + @State1, + @State2 + ) + ) ""A"", + ""Fileset"" + {whereClause} + ""A"".""VolumeID"" = ""Fileset"".""VolumeID"" + ORDER BY ""Fileset"".""Timestamp"" + ") + .SetParameterValue("@State1", RemoteVolumeState.Uploaded.ToString()) + .SetParameterValue("@State2", RemoteVolumeState.Verified.ToString()) + .SetParameterValues(tp.Item2) + .ExecuteReaderAsync(token)) + while (await rd.ReadAsync(token)) files.Add(new RemoteVolume(rd)); - foreach (var f in FilterByVerificationCount(files, samples, max)) - yield return f; - - if (options.FullRemoteVerification == Options.RemoteTestStrategy.ListAndIndexes || options.FullRemoteVerification == Options.RemoteTestStrategy.IndexesOnly) + if (files.Count == 0) yield break; - //And finally some block files - files.Clear(); + if (string.IsNullOrEmpty(tp.Item1)) + files = FilterByVerificationCount(files, samples, max).ToList(); - cmd.SetCommandAndParameters(@"SELECT ""ID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"" WHERE ""Type"" = @Type AND ""State"" IN (@States) AND ""ArchiveTime"" = 0") - .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) - .ExpandInClauseParameter("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); - - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - files.Add(new RemoteVolume(rd)); - - foreach (var f in FilterByVerificationCount(files, samples, max)) + foreach (var f in files) yield return f; + + //Then we select some index files + files.Clear(); } + + cmd.SetCommandAndParameters(@" + SELECT + ""ID"", + ""Name"", + ""Size"", + ""Hash"", + ""VerificationCount"" + FROM ""Remotevolume"" + WHERE + ""Type"" = @Type + AND ""State"" IN (@States) + AND ""ArchiveTime"" = 0 + ") + .SetParameterValue("@Type", RemoteVolumeType.Index.ToString()) + .ExpandInClauseParameterMssqlite("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); + + await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) + files.Add(new RemoteVolume(rd)); + + foreach (var f in FilterByVerificationCount(files, samples, max)) + yield return f; + + if (options.FullRemoteVerification == Options.RemoteTestStrategy.ListAndIndexes || options.FullRemoteVerification == Options.RemoteTestStrategy.IndexesOnly) + yield break; + + //And finally some block files + files.Clear(); + + cmd.SetCommandAndParameters(@" + SELECT + ""ID"", + ""Name"", + ""Size"", + ""Hash"", + ""VerificationCount"" + FROM ""Remotevolume"" + WHERE + ""Type"" = @Type + AND ""State"" IN (@States) + AND ""ArchiveTime"" = 0 + ") + .SetParameterValue("@Type", RemoteVolumeType.Blocks.ToString()) + .ExpandInClauseParameterMssqlite("@States", [RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()]); + + await using (var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false)) + while (await rd.ReadAsync(token).ConfigureAwait(false)) + files.Add(new RemoteVolume(rd)); + + foreach (var f in FilterByVerificationCount(files, samples, max)) + yield return f; } - private abstract class Basiclist : IDisposable + /// + /// Base class for basic lists used in the local test database. + /// Provides methods for creating temporary tables, inserting data, and disposing of resources. + /// + private abstract class Basiclist : IDisposable, IAsyncDisposable { - protected readonly IDbConnection m_connection; - protected readonly string m_volumename; - protected string m_tablename; - protected ReusableTransaction m_rtr; - protected IDbCommand m_insertCommand; + /// + /// The database connection used for executing commands. + /// + protected LocalDatabase m_db = null!; + /// + /// The name of the volume associated with this list. + /// + protected string m_volumename = null!; + /// + /// The name of the temporary table used for this list. + /// + protected string m_tablename = null!; + /// + /// Command used for inserting data into the temporary table. + /// + protected SqliteCommand m_insertCommand = null!; - protected Basiclist(IDbConnection connection, ReusableTransaction rtr, string volumename, string tablePrefix, string tableFormat, string insertCommand) + /// + /// Calling this constructor will throw an exception. Use the CreateAsync method instead. + /// + [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] + protected Basiclist(SqliteConnection connection, ReusableTransaction rtr, string volumename, string tablePrefix, string tableFormat, string insertCommand) { - m_connection = connection; - m_volumename = volumename; - m_rtr = rtr; - var tablename = tablePrefix + "-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - - using (var cmd = m_connection.CreateCommand(m_rtr.Transaction)) - { - cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" {tableFormat}")); - m_tablename = tablename; - } - - m_insertCommand = m_connection.CreateCommand(m_rtr.Transaction, FormatInvariant($@"INSERT INTO ""{m_tablename}"" {insertCommand}")); + throw new NotSupportedException("Use CreateAsync method instead."); } - public virtual void Dispose() + /// + /// Protected constructor to allow derived classes to initialize without parameters. + /// + protected Basiclist() { } + + /// + /// Creates a new instance of the class asynchronously. + /// + /// The instance of the to initialize. + /// The local database to use for the list. + /// The name of the volume associated with this list. + /// The prefix for the temporary table name. + /// The SQL format for creating the temporary table. + /// The SQL command for inserting data into the temporary table. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that represents the asynchronous operation. The task result contains the initialized instance. + protected static async Task CreateAsync(Basiclist bl, LocalDatabase db, string volumename, string tablePrefix, string tableFormat, string insertCommand, CancellationToken token) + { + bl.m_db = db; + bl.m_volumename = volumename; + var tablename = tablePrefix + "-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + + await using (var cmd = bl.m_db.Connection.CreateCommand(bl.m_db.Transaction)) + { + await cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{tablename}"" + {tableFormat} + ", token) + .ConfigureAwait(false); + + bl.m_tablename = tablename; + } + + bl.m_insertCommand = await bl.m_db.Connection.CreateCommandAsync($@" + INSERT INTO ""{bl.m_tablename}"" + {insertCommand} + ", token) + .ConfigureAwait(false); + + return bl; + } + + public void Dispose() + { + DisposeAsync().AsTask().Await(); + } + + public virtual async ValueTask DisposeAsync() { if (m_tablename != null) try { - using (var cmd = m_connection.CreateCommand(m_rtr.Transaction)) - cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tablename}""")); + await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction.Transaction); + await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tablename}""", default) + .ConfigureAwait(false); } catch { } finally { m_tablename = null!; } - m_insertCommand?.Dispose(); + await m_insertCommand.DisposeAsync().ConfigureAwait(false); } } - public interface IFilelist : IDisposable + /// + /// Interface for a file list used in the local test database. + /// Provides methods for adding entries, comparing the list with remote volumes, and disposing of resources. + /// + public interface IFilelist : IDisposable, IAsyncDisposable { - void Add(string path, long size, string hash, long metasize, string metahash, IEnumerable blocklistHashes, FilelistEntryType type, DateTime time); - IEnumerable> Compare(); + /// + /// Asynchronously adds a file entry to the file list. + /// + /// The path of the file. + /// The size of the file in bytes. + /// The hash of the file, or null if not applicable. + /// The size of the metadata associated with the file. + /// The hash of the metadata associated with the file. + /// A collection of blocklist hashes associated with the file. + /// The type of the file entry. + /// The timestamp of the file entry. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that completes when the file entry has been added. + Task Add(string path, long size, string hash, long metasize, string metahash, IEnumerable blocklistHashes, FilelistEntryType type, DateTime time, CancellationToken token); + + /// + /// Asynchronously compares the file list with remote volumes and yields differences. + /// + /// A cancellation token to observe while waiting for the task to complete. + /// An asynchronous enumerable of key-value pairs representing the comparison results, where the key is the test entry status and the value is the file path. + IAsyncEnumerable> Compare(CancellationToken token); } + /// + /// Implementation of the interface that manages a list of files in a local test database. + /// private class Filelist : Basiclist, IFilelist { + /// + /// The prefix for the temporary table name used for the file list. + /// private const string TABLE_PREFIX = "Filelist"; - private const string TABLE_FORMAT = @"(""Path"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Hash"" TEXT NULL, ""Metasize"" INTEGER NOT NULL, ""Metahash"" TEXT NOT NULL)"; - private const string INSERT_COMMAND = @"(""Path"", ""Size"", ""Hash"", ""Metasize"", ""Metahash"") VALUES (@Path,@Size,@Hash,@Metasize,@Metahash)"; - public Filelist(IDbConnection connection, string volumename, ReusableTransaction rtr) + + /// + /// The SQL format for creating the temporary table used for the file list. + /// + private const string TABLE_FORMAT = @" + ( + ""Path"" TEXT NOT NULL, + ""Size"" INTEGER NOT NULL, + ""Hash"" TEXT NULL, + ""Metasize"" INTEGER NOT NULL, + ""Metahash"" TEXT NOT NULL + ) + "; + + /// + /// The SQL command for inserting data into the temporary table used for the file list. + /// + private const string INSERT_COMMAND = @" + ( + ""Path"", + ""Size"", + ""Hash"", + ""Metasize"", + ""Metahash"" + ) + VALUES ( + @Path, + @Size, + @Hash, + @Metasize, + @Metahash + ) + "; + + /// + /// Calling this constructor will throw an exception. Use the CreateAsync method instead. + /// + [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] + public Filelist(SqliteConnection connection, string volumename, ReusableTransaction rtr) : base(connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) { + throw new NotSupportedException("Use CreateAsync method instead."); } - public void Add(string path, long size, string hash, long metasize, string metahash, IEnumerable blocklistHashes, FilelistEntryType type, DateTime time) + /// + /// Private constructor to allow derived classes to initialize without parameters and to prevent instantiation from outside, which should only be done through the CreateAsync method. + /// + private Filelist() { } + + /// + /// Asynchronously creates a new instance of the class. + /// + /// The local database to use for the file list. + /// The name of the volume associated with this file list. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that when awaited returns a new instance of the class. + public static async Task CreateAsync(LocalDatabase db, string volumename, CancellationToken token) { - m_insertCommand.SetTransaction(m_rtr.Transaction) + var bl = new Filelist(); + return (Filelist) + await CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND, token) + .ConfigureAwait(false); + } + + /// + public async Task Add(string path, long size, string hash, long metasize, string metahash, IEnumerable blocklistHashes, FilelistEntryType type, DateTime time, CancellationToken token) + { + await m_insertCommand + .SetTransaction(m_db.Transaction) .SetParameterValue("@Path", path) .SetParameterValue("@Size", hash == null ? -1 : size) .SetParameterValue("@Hash", hash) .SetParameterValue("@Metasize", metasize) .SetParameterValue("@Metahash", metahash) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); } - public IEnumerable> Compare() + /// + public async IAsyncEnumerable> Compare([EnumeratorCancellation] CancellationToken token) { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var create = FormatInvariant($@"CREATE TEMPORARY TABLE ""{cmpName}"" AS SELECT ""A"".""Path"" AS ""Path"", CASE WHEN ""B"".""Fullhash"" IS NULL THEN -1 ELSE ""B"".""Length"" END AS ""Size"", ""B"".""Fullhash"" AS ""Hash"", ""C"".""Length"" AS ""Metasize"", ""C"".""Fullhash"" AS ""Metahash"" FROM (SELECT ""File"".""Path"", ""File"".""BlocksetID"" AS ""FileBlocksetID"", ""Metadataset"".""BlocksetID"" AS ""MetadataBlocksetID"" from ""Remotevolume"", ""Fileset"", ""FilesetEntry"", ""File"", ""Metadataset"" WHERE ""Remotevolume"".""Name"" = @Name AND ""Fileset"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""MetadataID"" = ""Metadataset"".""ID"") A LEFT OUTER JOIN ""Blockset"" B ON ""B"".""ID"" = ""A"".""FileBlocksetID"" LEFT OUTER JOIN ""Blockset"" C ON ""C"".""ID""=""A"".""MetadataBlocksetID"" "); - var extra = FormatInvariant($@"SELECT @TypeExtra AS ""Type"", ""{m_tablename}"".""Path"" AS ""Path"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Path"" NOT IN ( SELECT ""Path"" FROM ""{cmpName}"" )"); - var missing = FormatInvariant($@"SELECT @TypeMissing AS ""Type"", ""Path"" AS ""Path"" FROM ""{cmpName}"" WHERE ""Path"" NOT IN (SELECT ""Path"" FROM ""{m_tablename}"")"); - var modified = FormatInvariant($@"SELECT @TypeModified AS ""Type"", ""E"".""Path"" AS ""Path"" FROM ""{m_tablename}"" E, ""{cmpName}"" D WHERE ""D"".""Path"" = ""E"".""Path"" AND (""D"".""Size"" != ""E"".""Size"" OR ""D"".""Hash"" != ""E"".""Hash"" OR ""D"".""Metasize"" != ""E"".""Metasize"" OR ""D"".""Metahash"" != ""E"".""Metahash"") "); - var drop = FormatInvariant($@"DROP TABLE IF EXISTS ""{cmpName}"" "); + var create = $@" + CREATE TEMPORARY TABLE ""{cmpName}"" AS + SELECT + ""A"".""Path"" AS ""Path"", + CASE + WHEN ""B"".""Fullhash"" IS NULL + THEN -1 + ELSE ""B"".""Length"" END AS ""Size"", + ""B"".""Fullhash"" AS ""Hash"", + ""C"".""Length"" AS ""Metasize"", + ""C"".""Fullhash"" AS ""Metahash"" + FROM ( + SELECT + ""File"".""Path"", + ""File"".""BlocksetID"" AS ""FileBlocksetID"", + ""Metadataset"".""BlocksetID"" AS ""MetadataBlocksetID"" + FROM + ""Remotevolume"", + ""Fileset"", + ""FilesetEntry"", + ""File"", + ""Metadataset"" + WHERE + ""Remotevolume"".""Name"" = @Name + AND ""Fileset"".""VolumeID"" = ""Remotevolume"".""ID"" + AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" + AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" + AND ""File"".""MetadataID"" = ""Metadataset"".""ID"" + ) ""A"" + LEFT OUTER JOIN ""Blockset"" ""B"" + ON ""B"".""ID"" = ""A"".""FileBlocksetID"" + LEFT OUTER JOIN ""Blockset"" ""C"" + ON ""C"".""ID""=""A"".""MetadataBlocksetID"" + "; - using (var cmd = m_connection.CreateCommand(m_rtr.Transaction)) + var extra = $@" + SELECT + @TypeExtra AS ""Type"", + ""{m_tablename}"".""Path"" AS ""Path"" + FROM ""{m_tablename}"" + WHERE ""{m_tablename}"".""Path"" NOT IN ( + SELECT ""Path"" + FROM ""{cmpName}"" + )"; + + var missing = $@" + SELECT + @TypeMissing AS ""Type"", + ""Path"" AS ""Path"" + FROM ""{cmpName}"" + WHERE ""Path"" NOT IN ( + SELECT ""Path"" + FROM ""{m_tablename}"" + ) + "; + + var modified = $@" + SELECT + @TypeModified AS ""Type"", + ""E"".""Path"" AS ""Path"" + FROM + ""{m_tablename}"" ""E"", + ""{cmpName}"" ""D"" + WHERE + ""D"".""Path"" = ""E"".""Path"" + AND ( + ""D"".""Size"" != ""E"".""Size"" + OR ""D"".""Hash"" != ""E"".""Hash"" + OR ""D"".""Metasize"" != ""E"".""Metasize"" + OR ""D"".""Metahash"" != ""E"".""Metahash"" + ) + "; + + var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"" "; + + await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + try + { + await cmd + .SetCommandAndParameters(create) + .SetParameterValue("@Name", m_volumename) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + cmd + .SetCommandAndParameters($"{extra} UNION {missing} UNION {modified}") + .SetParameterValue("@TypeExtra", (int)Interface.TestEntryStatus.Extra) + .SetParameterValue("@TypeMissing", (int)Interface.TestEntryStatus.Missing) + .SetParameterValue("@TypeModified", (int)Interface.TestEntryStatus.Modified); + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + yield return new KeyValuePair( + (Interface.TestEntryStatus)rd.ConvertValueToInt64(0), + rd.ConvertValueToString(1) ?? "" + ); + + } + finally { try { - cmd.SetCommandAndParameters(create) - .SetParameterValue("@Name", m_volumename) - .ExecuteNonQuery(); - - cmd.SetCommandAndParameters(FormatInvariant($"{extra} UNION {missing} UNION {modified}")) - .SetParameterValue("@TypeExtra", (int)Interface.TestEntryStatus.Extra) - .SetParameterValue("@TypeMissing", (int)Interface.TestEntryStatus.Missing) - .SetParameterValue("@TypeModified", (int)Interface.TestEntryStatus.Modified); - - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - yield return new KeyValuePair((Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); - - } - finally - { - try { cmd.ExecuteNonQuery(drop); } - catch { } + await cmd + .ExecuteNonQueryAsync(drop, token) + .ConfigureAwait(false); } + catch { } } } } - public interface IIndexlist : IDisposable + /// + /// Interface for an index list used in the local test database. + /// Provides methods for adding block links, comparing the index list with remote volumes, and disposing of resources. + /// + public interface IIndexlist : IDisposable, IAsyncDisposable { - void AddBlockLink(string filename, string hash, long length); - IEnumerable> Compare(); + /// + /// Asynchronously adds a block link to the index list. + /// + /// The name of the file associated with the block link. + /// The hash of the block link. + /// The length of the block link in bytes. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that completes when the block link has been added. + Task AddBlockLink(string filename, string hash, long length, CancellationToken token); + + /// + /// Asynchronously compares the index list with remote volumes and yields differences. + /// + /// A cancellation token to observe while waiting for the task to complete. + /// An asynchronous enumerable of key-value pairs representing the comparison results, where the key is the test entry status and the value is the file path. + IAsyncEnumerable> Compare(CancellationToken token); } + /// + /// Implementation of the interface that manages a list of index entries in a local test database. + /// private class Indexlist : Basiclist, IIndexlist { + /// + /// The prefix for the temporary table name used for the index list. + /// private const string TABLE_PREFIX = "Indexlist"; - private const string TABLE_FORMAT = @"(""Name"" TEXT NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)"; - private const string INSERT_COMMAND = @"(""Name"", ""Hash"", ""Size"") VALUES (@Name,@Hash,@Size)"; - public Indexlist(IDbConnection connection, string volumename, ReusableTransaction rtr) + /// + /// The SQL format for creating the temporary table used for the index list. + /// + private const string TABLE_FORMAT = @" + ( + ""Name"" TEXT NOT NULL, + ""Hash"" TEXT NOT NULL, + ""Size"" INTEGER NOT NULL + ) + "; + + /// + /// The SQL command for inserting data into the temporary table used for the index list. + /// + private const string INSERT_COMMAND = @" + ( + ""Name"", + ""Hash"", + ""Size"" + ) + VALUES ( + @Name, + @Hash, + @Size + ) + "; + + /// + /// Calling this constructor will throw an exception. Use the CreateAsync method instead. + /// + [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] + public Indexlist(SqliteConnection connection, string volumename, ReusableTransaction rtr) : base(connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) { + throw new NotSupportedException("Use CreateAsync method instead."); } - public void AddBlockLink(string filename, string hash, long length) + /// + /// Private constructor to allow derived classes to initialize without parameters and to prevent instantiation from outside, which should only be done through the CreateAsync method. + /// + private Indexlist() { } + + /// + /// Asynchronously creates a new instance of the class. + /// + /// The local database to use for the index list. + /// The name of the volume associated with this index list. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that when awaited returns a new instance of the class. + public static async Task CreateAsync(LocalDatabase db, string volumename, CancellationToken token) { - m_insertCommand.SetTransaction(m_rtr.Transaction) + var bl = new Indexlist(); + return (Indexlist) + await CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND, token) + .ConfigureAwait(false); + } + + /// + public async Task AddBlockLink(string filename, string hash, long length, CancellationToken token) + { + await m_insertCommand + .SetTransaction(m_db.Transaction) .SetParameterValue("@Name", filename) .SetParameterValue("@Hash", hash) .SetParameterValue("@Size", length) - .ExecuteNonQuery(); + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); } - public IEnumerable> Compare() + /// + public async IAsyncEnumerable> Compare([EnumeratorCancellation] CancellationToken token) { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var create = FormatInvariant($@"CREATE TEMPORARY TABLE ""{cmpName}"" AS SELECT ""A"".""Name"", ""A"".""Hash"", ""A"".""Size"" FROM ""Remotevolume"" A, ""Remotevolume"" B, ""IndexBlockLink"" WHERE ""B"".""Name"" = @Name AND ""A"".""ID"" = ""IndexBlockLink"".""BlockVolumeID"" AND ""B"".""ID"" = ""IndexBlockLink"".""IndexVolumeID"" "); - var extra = FormatInvariant($@"SELECT @TypeExtra AS ""Type"", ""{m_tablename}"".""Name"" AS ""Name"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Name"" NOT IN ( SELECT ""Name"" FROM ""{cmpName}"" )"); - var missing = FormatInvariant($@"SELECT @TypeMissing AS ""Type"", ""Name"" AS ""Name"" FROM ""{cmpName}"" WHERE ""Name"" NOT IN (SELECT ""Name"" FROM ""{m_tablename}"")"); - var modified = FormatInvariant($@"SELECT @TypeModified AS ""Type"", ""E"".""Name"" AS ""Name"" FROM ""{m_tablename}"" E, ""{cmpName}"" D WHERE ""D"".""Name"" = ""E"".""Name"" AND (""D"".""Hash"" != ""E"".""Hash"" OR ""D"".""Size"" != ""E"".""Size"") "); - var drop = FormatInvariant($@"DROP TABLE IF EXISTS ""{cmpName}"" "); + var create = $@" + CREATE TEMPORARY TABLE ""{cmpName}"" AS + SELECT + ""A"".""Name"", + ""A"".""Hash"", + ""A"".""Size"" + FROM + ""Remotevolume"" ""A"", + ""Remotevolume"" ""B"", + ""IndexBlockLink"" + WHERE + ""B"".""Name"" = @Name + AND ""A"".""ID"" = ""IndexBlockLink"".""BlockVolumeID"" + AND ""B"".""ID"" = ""IndexBlockLink"".""IndexVolumeID"" + "; - using (var cmd = m_connection.CreateCommand(m_rtr.Transaction)) + var extra = $@" + SELECT + @TypeExtra AS ""Type"", + ""{m_tablename}"".""Name"" AS ""Name"" + FROM ""{m_tablename}"" + WHERE ""{m_tablename}"".""Name"" NOT IN ( + SELECT ""Name"" + FROM ""{cmpName}"" + ) + "; + + var missing = $@" + SELECT + @TypeMissing AS ""Type"", + ""Name"" AS ""Name"" + FROM ""{cmpName}"" + WHERE ""Name"" NOT IN ( + SELECT ""Name"" + FROM ""{m_tablename}"" + ) + "; + + var modified = $@" + SELECT + @TypeModified AS ""Type"", + ""E"".""Name"" AS ""Name"" + FROM + ""{m_tablename}"" ""E"", + ""{cmpName}"" ""D"" + WHERE + ""D"".""Name"" = ""E"".""Name"" + AND ( + ""D"".""Hash"" != ""E"".""Hash"" + OR ""D"".""Size"" != ""E"".""Size"" + ) + "; + + var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"" "; + + await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + try + { + await cmd + .SetCommandAndParameters(create) + .SetParameterValue("@Name", m_volumename) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + cmd + .SetCommandAndParameters($"{extra} UNION {missing} UNION {modified}") + .SetParameterValue("@TypeExtra", (int)Interface.TestEntryStatus.Extra) + .SetParameterValue("@TypeMissing", (int)Interface.TestEntryStatus.Missing) + .SetParameterValue("@TypeModified", (int)Interface.TestEntryStatus.Modified); + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + yield return new KeyValuePair((Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); + + } + finally { try { - cmd.SetCommandAndParameters(create) - .SetParameterValue("@Name", m_volumename) - .ExecuteNonQuery(); - - cmd.SetCommandAndParameters(FormatInvariant($"{extra} UNION {missing} UNION {modified}")) - .SetParameterValue("@TypeExtra", (int)Interface.TestEntryStatus.Extra) - .SetParameterValue("@TypeMissing", (int)Interface.TestEntryStatus.Missing) - .SetParameterValue("@TypeModified", (int)Interface.TestEntryStatus.Modified); - - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - yield return new KeyValuePair((Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); - - } - finally - { - try { cmd.ExecuteNonQuery(drop); } - catch { } + await cmd + .ExecuteNonQueryAsync(drop, token) + .ConfigureAwait(false); } + catch { } } } } - public interface IBlocklist : IDisposable + /// + /// Interface for a blocklist used in the local test database. + /// Provides methods for adding blocks, comparing the blocklist with remote volumes, and disposing of resources. + /// + public interface IBlocklist : IDisposable, IAsyncDisposable { - void AddBlock(string key, long value); - IEnumerable> Compare(); + /// + /// Asynchronously adds a block to the blocklist. + /// + /// The key (hash) of the block. + /// The size of the block in bytes. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that completes when the block has been added. + Task AddBlock(string key, long value, CancellationToken token); + + /// + /// Asynchronously compares the blocklist with remote volumes and yields differences. + /// + /// A cancellation token to observe while waiting for the task to complete. + /// An asynchronous enumerable of key-value pairs representing the comparison results, where the key is the test entry status and the value is the block hash. + IAsyncEnumerable> Compare(CancellationToken token); } - public interface IBlocklistHashList : IDisposable + public interface IBlocklistHashList : IDisposable, IAsyncDisposable { - void AddBlockHash(string hash, long size); - IEnumerable> Compare(int hashesPerBlock, int hashSize, int blockSize); + /// + /// Asynchronously adds a block hash to the blocklist hash list. + /// + /// The hash of the block. + /// The size of the block in bytes. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that completes when the block hash has been added. + Task AddBlockHash(string hash, long size, CancellationToken token); + + /// + /// Asynchronously compares the blocklist hash list with remote volumes and yields differences. + /// + /// The number of hashes per block. + /// The size of each hash in bytes. + /// The size of each block in bytes. + /// A cancellation token to observe while waiting for the task to complete. + /// An asynchronous enumerable of key-value pairs representing the comparison results, where the key is the test entry status and the value is the block hash. + IAsyncEnumerable> Compare(int hashesPerBlock, int hashSize, int blockSize, CancellationToken token); } + /// + /// Implementation of the interface that manages a list of blocks in a local test database. + /// Provides methods for adding blocks, comparing the blocklist with remote volumes, and disposing of resources. + /// private class Blocklist : Basiclist, IBlocklist { + /// + /// The prefix for the temporary table name used for the blocklist. + /// private const string TABLE_PREFIX = "Blocklist"; - private const string TABLE_FORMAT = @"(""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)"; - private const string INSERT_COMMAND = @"(""Hash"", ""Size"") VALUES (@Hash,@Size)"; + /// + /// The SQL format for creating the temporary table used for the blocklist. + /// + private const string TABLE_FORMAT = @"( + ""Hash"" TEXT NOT NULL, + ""Size"" INTEGER NOT NULL + )"; + /// + /// The SQL command for inserting data into the temporary table used for the blocklist. + /// + private const string INSERT_COMMAND = @"( + ""Hash"", + ""Size"" + ) + VALUES ( + @Hash, + @Size + )"; - public Blocklist(IDbConnection connection, string volumename, ReusableTransaction rtr) - : base(connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) - { } - - public void AddBlock(string hash, long size) + /// + /// Calling this constructor will throw an exception. Use the CreateAsync method instead. + /// + [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] + public Blocklist(SqliteConnection connection, string volumename, ReusableTransaction rtr) { - m_insertCommand.SetTransaction(m_rtr.Transaction) - .SetParameterValue("@Hash", hash) - .SetParameterValue("@Size", size) - .ExecuteNonQuery(); + throw new NotSupportedException("Use CreateAsync method instead."); } - public IEnumerable> Compare() + /// + /// Private constructor to allow derived classes to initialize without parameters and to prevent instantiation from outside, which should only be done through the CreateAsync method. + /// + private Blocklist() { } + + /// + /// Asynchronously creates a new instance of the class. + /// + /// The local database to use for the blocklist. + /// A cancellation token to observe while waiting for the task to complete. + /// The name of the volume associated with this blocklist. + public static async Task CreateAsync(LocalDatabase db, string volumename, CancellationToken token) + { + var bl = new Blocklist(); + return (Blocklist) + await Basiclist.CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND, token) + .ConfigureAwait(false); + } + + /// + public async Task AddBlock(string hash, long size, CancellationToken token) + { + await m_insertCommand + .SetTransaction(m_db.Transaction) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } + + /// + public async IAsyncEnumerable> Compare([EnumeratorCancellation] CancellationToken token) { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var curBlocks = @"SELECT ""Block"".""Hash"" AS ""Hash"", ""Block"".""Size"" AS ""Size"" FROM ""Remotevolume"", ""Block"" WHERE ""Remotevolume"".""Name"" = @Name AND ""Remotevolume"".""ID"" = ""Block"".""VolumeID"""; - var duplBlocks = @"SELECT ""Block"".""Hash"" AS ""Hash"", ""Block"".""Size"" AS ""Size"" FROM ""DuplicateBlock"", ""Block"" WHERE ""DuplicateBlock"".""VolumeID"" = (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = @Name) AND ""Block"".""ID"" = ""DuplicateBlock"".""BlockID"""; - var delBlocks = @"SELECT ""DeletedBlock"".""Hash"" AS ""Hash"", ""DeletedBlock"".""Size"" AS ""Size"" FROM ""DeletedBlock"", ""RemoteVolume"" WHERE ""RemoteVolume"".""Name"" = @Name AND ""RemoteVolume"".""ID"" = ""DeletedBlock"".""VolumeID"""; - var create = FormatInvariant($@"CREATE TEMPORARY TABLE ""{cmpName}"" AS SELECT DISTINCT ""Hash"" AS ""Hash"", ""Size"" AS ""Size"" FROM ({curBlocks} UNION {delBlocks} UNION {duplBlocks})"); - var extra = FormatInvariant($@"SELECT @TypeExtra AS ""Type"", ""{m_tablename}"".""Hash"" AS ""Hash"" FROM ""{m_tablename}"" WHERE ""{m_tablename}"".""Hash"" NOT IN ( SELECT ""Hash"" FROM ""{cmpName}"" )"); - var missing = FormatInvariant($@"SELECT @TypeMissing AS ""Type"", ""Hash"" AS ""Hash"" FROM ""{cmpName}"" WHERE ""Hash"" NOT IN (SELECT ""Hash"" FROM ""{m_tablename}"")"); - var modified = FormatInvariant($@"SELECT @TypeModified AS ""Type"", ""E"".""Hash"" AS ""Hash"" FROM ""{m_tablename}"" E, ""{cmpName}"" D WHERE ""D"".""Hash"" = ""E"".""Hash"" AND ""D"".""Size"" != ""E"".""Size"" "); - var drop = FormatInvariant($@"DROP TABLE IF EXISTS ""{cmpName}"" "); - using (var cmd = m_connection.CreateCommand(m_rtr.Transaction)) + var curBlocks = @" + SELECT + ""Block"".""Hash"" AS ""Hash"", + ""Block"".""Size"" AS ""Size"" + FROM + ""Remotevolume"", + ""Block"" + WHERE + ""Remotevolume"".""Name"" = @Name + AND ""Remotevolume"".""ID"" = ""Block"".""VolumeID"" + "; + + var duplBlocks = @" + SELECT + ""Block"".""Hash"" AS ""Hash"", + ""Block"".""Size"" AS ""Size"" + FROM + ""DuplicateBlock"", + ""Block"" + WHERE + ""DuplicateBlock"".""VolumeID"" = ( + SELECT ""ID"" + FROM ""RemoteVolume"" + WHERE ""Name"" = @Name + ) + AND ""Block"".""ID"" = ""DuplicateBlock"".""BlockID"" + "; + + var delBlocks = @" + SELECT + ""DeletedBlock"".""Hash"" AS ""Hash"", + ""DeletedBlock"".""Size"" AS ""Size"" + FROM + ""DeletedBlock"", + ""RemoteVolume"" + WHERE + ""RemoteVolume"".""Name"" = @Name + AND ""RemoteVolume"".""ID"" = ""DeletedBlock"".""VolumeID"" + "; + + var create = $@" + CREATE TEMPORARY TABLE ""{cmpName}"" AS + SELECT DISTINCT + ""Hash"" AS ""Hash"", + ""Size"" AS ""Size"" + FROM ( + {curBlocks} + UNION {delBlocks} + UNION {duplBlocks} + ) + "; + + var extra = $@" + SELECT + @TypeExtra AS ""Type"", + ""{m_tablename}"".""Hash"" AS ""Hash"" + FROM ""{m_tablename}"" + WHERE ""{m_tablename}"".""Hash"" NOT IN ( + SELECT ""Hash"" + FROM ""{cmpName}"" + ) + "; + + var missing = $@" + SELECT + @TypeMissing AS ""Type"", + ""Hash"" AS ""Hash"" + FROM ""{cmpName}"" + WHERE ""Hash"" NOT IN ( + SELECT ""Hash"" + FROM ""{m_tablename}"" + ) + "; + + var modified = $@" + SELECT + @TypeModified AS ""Type"", + ""E"".""Hash"" AS ""Hash"" + FROM + ""{m_tablename}"" E, + ""{cmpName}"" D + WHERE + ""D"".""Hash"" = ""E"".""Hash"" + AND ""D"".""Size"" != ""E"".""Size"" + "; + + var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"" "; + + await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + try + { + await cmd + .SetCommandAndParameters(create) + .SetParameterValue("@Name", m_volumename) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + cmd + .SetCommandAndParameters($@" + {extra} + UNION {missing} + UNION {modified} + ") + .SetParameterValue("@TypeExtra", (int)Library.Interface.TestEntryStatus.Extra) + .SetParameterValue("@TypeMissing", (int)Library.Interface.TestEntryStatus.Missing) + .SetParameterValue("@TypeModified", (int)Library.Interface.TestEntryStatus.Modified); + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + yield return new KeyValuePair((Duplicati.Library.Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); + + } + finally { try { - cmd.SetCommandAndParameters(create) - .SetParameterValue("@Name", m_volumename) - .ExecuteNonQuery(); - - cmd.SetCommandAndParameters(FormatInvariant($"{extra} UNION {missing} UNION {modified}")) - .SetParameterValue("@TypeExtra", (int)Library.Interface.TestEntryStatus.Extra) - .SetParameterValue("@TypeMissing", (int)Library.Interface.TestEntryStatus.Missing) - .SetParameterValue("@TypeModified", (int)Library.Interface.TestEntryStatus.Modified); - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - yield return new KeyValuePair((Duplicati.Library.Interface.TestEntryStatus)rd.ConvertValueToInt64(0), rd.ConvertValueToString(1) ?? ""); - - } - finally - { - try { cmd.ExecuteNonQuery(drop); } - catch { } + await cmd + .ExecuteNonQueryAsync(drop, token) + .ConfigureAwait(false); } + catch { } } } } + /// + /// Implementation of the interface that manages a list of block hashes in a local test database. + /// Provides methods for adding block hashes, comparing the blocklist hash list with remote volumes, and disposing of resources. + /// private class BlocklistHashList : Basiclist, IBlocklistHashList { + /// + /// The prefix for the temporary table name used for the blocklist hash list. + /// private const string TABLE_PREFIX = "BlocklistHashList"; - private const string TABLE_FORMAT = @"(""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)"; - private const string INSERT_COMMAND = @"(""Hash"", ""Size"") VALUES (@Hash,@Size)"; - public BlocklistHashList(IDbConnection connection, string volumename, ReusableTransaction rtr) - : base(connection, rtr, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND) - { } + /// + /// The SQL format for creating the temporary table used for the blocklist hash list. + /// + private const string TABLE_FORMAT = @"( + ""Hash"" TEXT NOT NULL, + ""Size"" INTEGER NOT NULL + )"; - public void AddBlockHash(string hash, long size) + /// + /// The SQL command for inserting data into the temporary table used for the blocklist hash list. + /// + private const string INSERT_COMMAND = @"( + ""Hash"", + ""Size"" + ) + VALUES ( + @Hash, + @Size + )"; + + /// + /// Calling this constructor will throw an exception. Use the CreateAsync method instead. + /// + [Obsolete("Calling this constructor will throw an exception. Use the CreateAsync method instead.")] + public BlocklistHashList(SqliteConnection connection, string volumename, ReusableTransaction rtr) { - m_insertCommand.SetTransaction(m_rtr.Transaction) - .SetParameterValue("@Hash", hash) - .SetParameterValue("@Size", size) - .ExecuteNonQuery(); + throw new NotSupportedException("Use CreateAsync method instead."); } - public IEnumerable> Compare(int hashesPerBlock, int hashSize, int blockSize) + /// + /// Private constructor to allow derived classes to initialize without parameters and to prevent instantiation from outside, which should only be done through the CreateAsync method. + /// + private BlocklistHashList() { } + + /// + /// Asynchronously creates a new instance of the class. + /// + /// The local database to use for the blocklist hash list. + /// The name of the volume associated with this blocklist hash list. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that when awaited returns a new instance of the class. + public static async Task CreateAsync(LocalDatabase db, string volumename, CancellationToken token) + { + var bl = new BlocklistHashList(); + return (BlocklistHashList) + await Basiclist.CreateAsync(bl, db, volumename, TABLE_PREFIX, TABLE_FORMAT, INSERT_COMMAND, token) + .ConfigureAwait(false); + } + + /// + public async Task AddBlockHash(string hash, long size, CancellationToken token) + { + await m_insertCommand + .SetTransaction(m_db.Transaction) + .SetParameterValue("@Hash", hash) + .SetParameterValue("@Size", size) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + } + + /// + public async IAsyncEnumerable> Compare(int hashesPerBlock, int hashSize, int blockSize, [EnumeratorCancellation] CancellationToken token) { var cmpName = "CmpTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - var create = FormatInvariant($@" + var create = $@" CREATE TEMPORARY TABLE ""{cmpName}"" ( ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL ); - - INSERT INTO ""{cmpName}"" (""Hash"", ""Size"") - SELECT b.""Hash"", b.""Size"" - FROM Block b + + INSERT INTO ""{cmpName}"" ( + ""Hash"", + ""Size"" + ) + SELECT + ""b"".""Hash"", + ""b"".""Size"" + FROM ""Block"" ""b"" JOIN ( SELECT - blh.""Hash"", + ""blh"".""Hash"", CASE - WHEN blh.""Index"" = (((bs.""Length"" + {blockSize} - 1) / {blockSize} - 1) / {hashesPerBlock}) - AND ((bs.""Length"" + {blockSize} - 1) / {blockSize}) % {hashesPerBlock} != 0 - THEN {hashSize} * ((bs.""Length"" + {blockSize} - 1) / {blockSize} % {hashesPerBlock}) + WHEN ""blh"".""Index"" = (((""bs"".""Length"" + {blockSize} - 1) / {blockSize} - 1) / {hashesPerBlock}) + AND ((""bs"".""Length"" + {blockSize} - 1) / {blockSize}) % {hashesPerBlock} != 0 + THEN {hashSize} * ((""bs"".""Length"" + {blockSize} - 1) / {blockSize} % {hashesPerBlock}) ELSE {hashSize} * {hashesPerBlock} END AS ""Size"" - FROM BlocklistHash blh - JOIN Blockset bs ON bs.""ID"" = blh.""BlocksetID"" - ) expected ON b.""Hash"" = expected.""Hash"" AND b.""Size"" = expected.""Size"" - WHERE b.""VolumeID"" IN ( - SELECT ibl.""BlockVolumeID"" - FROM Remotevolume idx - JOIN IndexBlockLink ibl ON ibl.""IndexVolumeID"" = idx.""ID"" - WHERE idx.""Name"" = @Name + FROM ""BlocklistHash"" ""blh"" + JOIN ""Blockset"" ""bs"" + ON ""bs"".""ID"" = ""blh"".""BlocksetID"" + ) ""Expected"" + ON + ""b"".""Hash"" = ""Expected"".""Hash"" + AND ""b"".""Size"" = ""Expected"".""Size"" + WHERE ""b"".""VolumeID"" IN ( + SELECT ""ibl"".""BlockVolumeID"" + FROM ""Remotevolume"" ""idx"" + JOIN IndexBlockLink ""ibl"" + ON ""ibl"".""IndexVolumeID"" = ""idx"".""ID"" + WHERE ""idx"".""Name"" = @Name ); - "); + "; - var compare = FormatInvariant($@" + var compare = $@" WITH - Expected AS ( - SELECT ""Hash"", ""Size"" FROM ""{cmpName}"" + ""Expected"" AS ( + SELECT + ""Hash"", + ""Size"" + FROM ""{cmpName}"" ), - Actual AS ( - SELECT ""Hash"", ""Size"" FROM ""{m_tablename}"" + ""Actual"" AS ( + SELECT + ""Hash"", + ""Size"" + FROM ""{m_tablename}"" ), - Extra AS ( - SELECT @TypeExtra AS Type, a.""Hash"" - FROM Actual a - LEFT JOIN Expected e ON a.""Hash"" = e.""Hash"" AND a.""Size"" = e.""Size"" - WHERE e.""Hash"" IS NULL + ""Extra"" AS ( + SELECT + @TypeExtra AS ""Type"", + ""a"".""Hash"" + FROM ""Actual"" ""a"" + LEFT JOIN ""Expected"" ""e"" + ON + ""a"".""Hash"" = ""e"".""Hash"" + AND ""a"".""Size"" = ""e"".""Size"" + WHERE ""e"".""Hash"" IS NULL ), - Missing AS ( - SELECT @TypeMissing AS Type, e.""Hash"" - FROM Expected e - LEFT JOIN Actual a ON a.""Hash"" = e.""Hash"" AND a.""Size"" = e.""Size"" - WHERE a.""Hash"" IS NULL + ""Missing"" AS ( + SELECT + @TypeMissing AS ""Type"", + ""e"".""Hash"" + FROM ""Expected"" ""e"" + LEFT JOIN ""Actual"" ""a"" + ON + ""a"".""Hash"" = ""e"".""Hash"" + AND ""a"".""Size"" = ""e"".""Size"" + WHERE ""a"".""Hash"" IS NULL ), - Modified AS ( - SELECT @TypeModified AS Type, a.""Hash"" - FROM Actual a - JOIN Expected e ON a.""Hash"" = e.""Hash"" - WHERE a.""Size"" != e.""Size"" - AND NOT EXISTS ( - SELECT 1 FROM Extra x WHERE x.""Hash"" = a.""Hash"" - ) + ""Modified"" AS ( + SELECT + @TypeModified AS ""Type"", + ""a"".""Hash"" + FROM ""Actual"" ""a"" + JOIN ""Expected"" ""e"" + ON ""a"".""Hash"" = ""e"".""Hash"" + WHERE + ""a"".""Size"" != ""e"".""Size"" + AND NOT EXISTS ( + SELECT 1 + FROM ""Extra"" ""x"" + WHERE ""x"".""Hash"" = ""a"".""Hash"" + ) ) - SELECT * FROM Extra + SELECT * + FROM ""Extra"" UNION - SELECT * FROM Missing + SELECT * + FROM ""Missing"" UNION - SELECT * FROM Modified; - "); + SELECT * + FROM ""Modified""; + "; - var drop = FormatInvariant($@"DROP TABLE IF EXISTS ""{cmpName}"""); + var drop = $@"DROP TABLE IF EXISTS ""{cmpName}"""; - using (var cmd = m_connection.CreateCommand(m_rtr.Transaction)) + await using var cmd = m_db.Connection.CreateCommand(m_db.Transaction); + try + { + // Create expected hash+size table filtered by volume + await cmd + .SetCommandAndParameters(create) + .SetParameterValue("@Name", m_volumename) + .ExecuteNonQueryAsync(token) + .ConfigureAwait(false); + + // Compare against actual values inserted into temp table + cmd + .SetCommandAndParameters(compare) + .SetParameterValue("@TypeExtra", (int)Library.Interface.TestEntryStatus.Extra) + .SetParameterValue("@TypeMissing", (int)Library.Interface.TestEntryStatus.Missing) + .SetParameterValue("@TypeModified", (int)Library.Interface.TestEntryStatus.Modified); + + await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false); + while (await rd.ReadAsync(token).ConfigureAwait(false)) + yield return new KeyValuePair( + (Library.Interface.TestEntryStatus)rd.ConvertValueToInt64(0), + rd.ConvertValueToString(1) ?? ""); + } + finally { try { - // Create expected hash+size table filtered by volume - cmd.SetCommandAndParameters(create) - .SetParameterValue("@Name", m_volumename) - .ExecuteNonQuery(); - - // Compare against actual values inserted into temp table - cmd.SetCommandAndParameters(compare) - .SetParameterValue("@TypeExtra", (int)Library.Interface.TestEntryStatus.Extra) - .SetParameterValue("@TypeMissing", (int)Library.Interface.TestEntryStatus.Missing) - .SetParameterValue("@TypeModified", (int)Library.Interface.TestEntryStatus.Modified); - - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - yield return new KeyValuePair( - (Library.Interface.TestEntryStatus)rd.ConvertValueToInt64(0), - rd.ConvertValueToString(1) ?? ""); - } - finally - { - try { cmd.ExecuteNonQuery(drop); } catch { } + await cmd + .ExecuteNonQueryAsync(drop, token) + .ConfigureAwait(false); } + catch { } } } } - public IFilelist CreateFilelist(string name, ReusableTransaction rtr) + /// + /// Creates a new filelist in the local test database. + /// + /// The name of the filelist to create. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that when awaited returns a new instance of the interface. + public async Task CreateFilelist(string name, CancellationToken token) { - return new Filelist(m_connection, name, rtr); + return await Filelist.CreateAsync(this, name, token) + .ConfigureAwait(false); } - public IIndexlist CreateIndexlist(string name, ReusableTransaction rtr) + /// + /// Creates a new indexlist in the local test database. + /// + /// The name of the indexlist to create. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that when awaited returns a new instance of the interface. + public async Task CreateIndexlist(string name, CancellationToken token) { - return new Indexlist(m_connection, name, rtr); + return await Indexlist.CreateAsync(this, name, token) + .ConfigureAwait(false); } - public IBlocklist CreateBlocklist(string name, ReusableTransaction rtr) + /// + /// Creates a new blocklist in the local test database. + /// + /// The name of the blocklist to create. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that when awaited returns a new instance of the interface. + public async Task CreateBlocklist(string name, CancellationToken token) { - return new Blocklist(m_connection, name, rtr); + return await Blocklist.CreateAsync(this, name, token) + .ConfigureAwait(false); } - public IBlocklistHashList CreateBlocklistHashList(string name, ReusableTransaction rtr) + /// + /// Creates a new blocklist hash list in the local test database. + /// + /// The name of the blocklist hash list to create. + /// A cancellation token to observe while waiting for the task to complete. + /// A task that when awaited returns a new instance of the interface. + public async Task CreateBlocklistHashList(string name, CancellationToken token) { - return new BlocklistHashList(m_connection, name, rtr); + return await BlocklistHashList.CreateAsync(this, name, token) + .ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Database/RemoteVolume.cs b/Duplicati/Library/Main/Database/RemoteVolume.cs index 92f3bc77b..4d5f5940c 100644 --- a/Duplicati/Library/Main/Database/RemoteVolume.cs +++ b/Duplicati/Library/Main/Database/RemoteVolume.cs @@ -1,28 +1,38 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. namespace Duplicati.Library.Main.Database { + /// + /// Represents a remote volume with its name, hash, size, and an optional file entry. + /// Implements the IRemoteVolume interface. + /// public class RemoteVolume : IRemoteVolume { + /// + /// Initializes a new instance of the class with a file + /// entry and an optional hash. + /// + /// The file entry associated with the remote volume. + /// The hash of the remote volume, can be null. public RemoteVolume(Interface.IFileEntry file, string hash = null) { Name = file.Name; @@ -31,6 +41,13 @@ namespace Duplicati.Library.Main.Database File = file; } + /// + /// Initializes a new instance of the class with a name, + /// hash, and size. + /// + /// The name of the remote volume. + /// The hash of the remote volume. + /// The size of the remote volume in bytes. public RemoteVolume(string name, string hash, long size) { Name = name; diff --git a/Duplicati/Library/Main/Database/RemoteVolumeEntry.cs b/Duplicati/Library/Main/Database/RemoteVolumeEntry.cs index e496f0698..3c264087d 100644 --- a/Duplicati/Library/Main/Database/RemoteVolumeEntry.cs +++ b/Duplicati/Library/Main/Database/RemoteVolumeEntry.cs @@ -1,40 +1,75 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; namespace Duplicati.Library.Main.Database { + /// + /// Represents a remote volume entry with its ID, name, hash, size, type, state, + /// delete grace period, and archive time. + /// Implements the IRemoteVolume interface. + /// public struct RemoteVolumeEntry : IRemoteVolume { + /// + /// The ID of the remote volume entry. + /// public long ID { get; private set; } public string Name { get; private set; } public string Hash { get; private set; } public long Size { get; private set; } + /// + /// The type of the remote volume, indicating whether it is a file or a directory. + /// public RemoteVolumeType Type { get; private set; } + /// + /// The state of the remote volume, indicating its current status. + /// public RemoteVolumeState State { get; private set; } + /// + /// The time period during which the remote volume can be deleted without permanent loss. + /// public DateTime DeleteGracePeriod { get; private set; } + /// + /// The time when the remote volume was archived, indicating when it was moved to a less accessible state. + /// public DateTime ArchiveTime { get; private set; } + /// + /// Represents an empty remote volume entry with default values. + /// This is useful for initializing or resetting remote volume entries. + /// public static readonly RemoteVolumeEntry Empty = new RemoteVolumeEntry(-1, null, null, -1, (RemoteVolumeType)(-1), (RemoteVolumeState)(-1), default(DateTime), default(DateTime)); + /// + /// Initializes a new instance of the RemoteVolumeEntry struct with specified values. + /// + /// The ID of the remote volume entry. + /// The name of the remote volume entry. + /// The hash of the remote volume entry. + /// The size of the remote volume entry. + /// The type of the remote volume entry (file or directory). + /// The state of the remote volume entry. + /// The delete grace period for the remote volume entry. + /// The archive time for the remote volume entry. public RemoteVolumeEntry(long id, string name, string hash, long size, RemoteVolumeType type, RemoteVolumeState state, DateTime deleteGracePeriod, DateTime archiveTime) { ID = id; diff --git a/Duplicati/Library/Main/Database/ReusableTransaction.cs b/Duplicati/Library/Main/Database/ReusableTransaction.cs index 4f681deec..7034b49da 100644 --- a/Duplicati/Library/Main/Database/ReusableTransaction.cs +++ b/Duplicati/Library/Main/Database/ReusableTransaction.cs @@ -1,90 +1,170 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. + using System; -using System.Data; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.Utility; +using Microsoft.Data.Sqlite; #nullable enable namespace Duplicati.Library.Main.Database; /// -/// Wraps a transaction so it can be comitted and restarted +/// Wraps a transaction so it can be comitted and restarted. /// -internal class ReusableTransaction : IDisposable +/// +/// Creates a new reusable transaction. +/// +/// The database to use. +/// The transaction to use. If null, a new transaction is created. +internal class ReusableTransaction(SqliteConnection con, SqliteTransaction? transaction = null) : IDisposable, IAsyncDisposable { /// - /// The tag used for logging + /// The tag used for logging. /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(ReusableTransaction)); /// - /// The database to use + /// The database to use. /// - private readonly LocalDatabase m_db; + private readonly SqliteConnection m_con = con; /// - /// The current transaction + /// The current transaction. /// - private IDbTransaction? m_transaction; + private SqliteTransaction m_transaction = transaction ?? con.BeginTransaction(deferred: true); + /// + /// True if the transaction is disposed. + /// + private bool m_disposed = false; /// - /// Creates a new reusable transaction + /// Creates a new reusable transaction. /// - /// The database to use - public ReusableTransaction(LocalDatabase db, IDbTransaction? transaction = null) + /// The database this transaction relates to. + /// An optional existing transaction to use. If null, a new transaction is created. + public ReusableTransaction(LocalDatabase db, SqliteTransaction? transaction = null) : this(db.Connection, transaction) { } + + /// + /// The current transaction. + /// + public SqliteTransaction Transaction => m_disposed ? throw new InvalidOperationException("Transaction is disposed") : m_transaction; + + /// + /// Commits the transaction and restarts it. + /// + /// A cancellation token to cancel the operation. + /// A task that completes when the commit is done and a new transaction has been started. + public async Task CommitAsync(CancellationToken token) { - m_db = db; - m_transaction = transaction ?? db.BeginTransaction(); + await CommitAsync(null, true, token).ConfigureAwait(false); } /// - /// The current transaction + /// Async version of Commit: /// - public IDbTransaction Transaction => m_transaction ?? throw new InvalidOperationException("Transaction is disposed"); - - /// - /// Commits the current transaction and optionally restarts it - /// - /// The log message to use - /// True if the transaction should be restarted - public void Commit(string? message, bool restart = true) + /// The log message to use. + /// True if the transaction should be restarted. + /// A task that completes when the commit is done and (potentially) a new transaction has been started. + /// If the transaction is already Disposed. + public async Task CommitAsync(string? message = null, bool restart = true, CancellationToken token = default) { - if (m_transaction == null) + if (m_disposed) throw new InvalidOperationException("Transaction is already disposed"); - if (m_transaction != null) - { - using (var timer = string.IsNullOrWhiteSpace(message) ? null : new Logging.Timer(LOGTAG, message, $"CommitTransaction: {message}")) - m_transaction.Commit(); - m_transaction.Dispose(); - m_transaction = null; - } + message ??= "Unnamed commit"; + using (var timer = new Logging.Timer(LOGTAG, message, $"CommitTransaction: {message}")) + await m_transaction.CommitAsync().ConfigureAwait(false); + await m_transaction.DisposeAsync().ConfigureAwait(false); if (restart) - m_transaction = m_db.BeginTransaction(); + m_transaction = m_con.BeginTransaction(deferred: true); + else + m_disposed = true; } /// public void Dispose() { - m_transaction?.Dispose(); - m_transaction = null; + DisposeAsync().AsTask().Await(); + } + + /// + public async ValueTask DisposeAsync() + { + if (!m_disposed) + { + try + { + using (var timer = new Logging.Timer(LOGTAG, "Dispose", "Rollback during transaction dispose")) + await m_transaction.RollbackAsync().ConfigureAwait(false); + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "ReusableTransaction dispose", ex, "Transaction disposed with error: {0}", ex.Message); + throw; + } + finally + { + m_disposed = true; + await m_transaction.DisposeAsync().ConfigureAwait(false); + } + } + } + + /// + /// Rolls back the transaction and restarts it. + /// + /// A cancellation token to cancel the operation. + /// A task that completes when the rollback is done and a new transaction has been started. + public async Task RollBackAsync(CancellationToken token) + { + await RollBackAsync(null, true, token).ConfigureAwait(false); + } + + /// + /// Rolls back the transaction and optionally restarts it. + /// + /// Message to log. + /// Whether to restart the transaction after rolling back. + /// A cancellation token to cancel the operation. + /// A task that completes when the rollback is done and (potentially) a new transaction has been started. + /// If the transaction has already been disposed. + public async Task RollBackAsync(string? message = null, bool restart = true, CancellationToken token = default) + { + if (m_disposed) + throw new InvalidOperationException("Transaction is already disposed"); + + using (var timer = new Logging.Timer(LOGTAG, message, $"RollbackTransaction: {message}")) + await m_transaction.RollbackAsync().ConfigureAwait(false); + await m_transaction.DisposeAsync().ConfigureAwait(false); + + if (restart) + { + m_transaction = m_con.BeginTransaction(); + } + else + { + m_disposed = true; + } } } diff --git a/Duplicati/Library/Main/Database/TemporaryDbValueList.cs b/Duplicati/Library/Main/Database/TemporaryDbValueList.cs index 91f1aa834..d1acad234 100644 --- a/Duplicati/Library/Main/Database/TemporaryDbValueList.cs +++ b/Duplicati/Library/Main/Database/TemporaryDbValueList.cs @@ -1,140 +1,200 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.Data; using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.Utility; +using Microsoft.Data.Sqlite; #nullable enable namespace Duplicati.Library.Main.Database; /// -/// A list of values that can be used in a query +/// A list of values that can be used in a query. /// -public class TemporaryDbValueList : IDisposable +internal class TemporaryDbValueList : IDisposable, IAsyncDisposable { /// - /// The tag used for logging + /// The tag used for logging. /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(TemporaryDbValueList)); /// - /// The command to use + /// The command to use. /// - private IDbCommand? _cmd; + private SqliteCommand? _cmd; /// - /// The connection to use + /// The connection to use. /// - private readonly IDbConnection _con; + private SqliteConnection _con; /// - /// The transaction to use, if any + /// The reusable transaction to use. /// - private readonly IDbTransaction? _transaction; + private ReusableTransaction _rtr; /// - /// The table to use + /// The table to use. /// private readonly string _tableName = $"TemporaryList-{Guid.NewGuid():N}"; /// - /// Flag indicating if the object has been disposed + /// Flag indicating if the object has been disposed. /// private bool _disposed = false; /// - /// Flag indicating if the values are in a table + /// Flag indicating if the values are in a table. /// private bool _isTable = false; /// - /// The values to use + /// The values to use. /// - private IEnumerable _values; + private readonly IEnumerable _values; /// - /// The type of the values + /// The type of the values. /// private readonly string _valuesType; /// - /// Creates a new reusable transaction + /// Creates a new TemporaryDbValueList with the given values. /// - /// The connection to use - /// The transaction to use - /// The values to use - public TemporaryDbValueList(IDbConnection con, IDbTransaction? transaction, IEnumerable values) + /// The connection to use. + /// The reusable transaction to use. + /// The values to use. + /// The type of the values (e.g., "INTEGER", "TEXT"). + private TemporaryDbValueList(SqliteConnection con, ReusableTransaction rtr, IEnumerable values, string valuesType) { _con = con; - _transaction = transaction; - _valuesType = "INTEGER"; - if (values == null) - throw new ArgumentNullException(nameof(values)); - _values = values.Cast(); - if (values.Count() > LocalDatabase.CHUNK_SIZE) - ForceToTable(); + _rtr = rtr; + _valuesType = valuesType; + _values = values; + + ArgumentNullException.ThrowIfNull(values); } /// - /// Creates a new reusable transaction + /// Asynchronously creates a new TemporaryDbValueList with the given values. /// - /// The command to use - /// The connection to use - /// The values to use - public TemporaryDbValueList(IDbConnection con, IDbTransaction? transaction, IEnumerable values) + /// The database to use. + /// The values to use. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains the created TemporaryDbValueList. + internal static async Task CreateAsync(LocalDatabase db, IEnumerable values, CancellationToken token) { - _con = con; - _transaction = transaction; - _valuesType = "TEXT"; - if (values == null) - throw new ArgumentNullException(nameof(values)); - _values = values.Cast(); - if (values.Count() > LocalDatabase.CHUNK_SIZE) - ForceToTable(); + return await DoCreateAsync(new TemporaryDbValueList(db.Connection, db.Transaction, values.Cast(), "INTEGER"), token) + .ConfigureAwait(false); } /// - /// The values in the list + /// Asynchronously creates a new TemporaryDbValueList with the given values. + /// + /// The database to use. + /// The values to use. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains the created TemporaryDbValueList. + internal static async Task CreateAsync(LocalDatabase db, IEnumerable values, CancellationToken token) + { + return await DoCreateAsync(new TemporaryDbValueList(db.Connection, db.Transaction, values.Cast(), "TEXT"), token) + .ConfigureAwait(false); + } + + /// + /// Asynchronously creates a new TemporaryDbValueList with the given values. + /// + /// The connection to use. + /// The reusable transaction to use. + /// The values to use. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains the created TemporaryDbValueList. + internal static async Task CreateAsync(SqliteConnection con, ReusableTransaction rtr, IEnumerable values, CancellationToken token) + { + return await DoCreateAsync(new TemporaryDbValueList(con, rtr, values.Cast(), "INTEGER"), token) + .ConfigureAwait(false); + } + + /// + /// Asynchronously creates a new TemporaryDbValueList with the given values. + /// + /// The connection to use. + /// The reusable transaction to use. + /// The values to use. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains the created TemporaryDbValueList. + internal static async Task CreateAsync(SqliteConnection con, ReusableTransaction rtr, IEnumerable values, CancellationToken token) + { + return await DoCreateAsync(new TemporaryDbValueList(con, rtr, values.Cast(), "TEXT"), token) + .ConfigureAwait(false); + } + + /// + /// Performs the actual creation of the TemporaryDbValueList, ensuring that if the number of values exceeds the chunk size, + /// the values are forced into a table. + /// + /// The TemporaryDbValueList to create. + /// A cancellation token to cancel the operation. + /// A task that when awaited contains the created TemporaryDbValueList. + private static async Task DoCreateAsync(TemporaryDbValueList valueList, CancellationToken token) + { + if (valueList._values.Count() > LocalDatabase.CHUNK_SIZE) + await valueList.ForceToTable(token).ConfigureAwait(false); + + return valueList; + } + + /// + /// The values in the list. /// public IEnumerable Values => _values; /// - /// Flag indicating if the table has been created + /// Flag indicating if the table has been created. /// public bool IsTableCreated => _isTable; /// - /// Get the in clause for the values, creating the table if needed + /// Get the in clause for the values, creating the table if needed. /// - /// The in clause for the values - public string GetInClause() + /// A cancellation token to cancel the operation. + /// A task that when awaited contains the SQL in clause string. + public async Task GetInClause(CancellationToken token) { if (_disposed) throw new ObjectDisposedException(nameof(TemporaryDbValueList)); - ForceToTable(); + await ForceToTable(token).ConfigureAwait(false); - return $@"SELECT ""Value"" FROM ""{_tableName}"""; + return $@" + SELECT ""Value"" + FROM ""{_tableName}"" + "; } /// - /// Force the values to be written to a table, if not already done + /// Force the values to be written to a table, if not already done. /// - public void ForceToTable() + /// A cancellation token to cancel the operation. + /// A task that completes when the table has been created and the values inserted. + public async Task ForceToTable(CancellationToken token) { if (_disposed) throw new ObjectDisposedException(nameof(TemporaryDbValueList)); @@ -142,29 +202,42 @@ public class TemporaryDbValueList : IDisposable if (_isTable) return; - _cmd = _con.CreateCommand(_transaction); - _cmd.ExecuteNonQuery($@"CREATE TEMPORARY TABLE ""{_tableName}"" (""Value"" {_valuesType})"); + _cmd = _con.CreateCommand(_rtr); + await _cmd.ExecuteNonQueryAsync($@" + CREATE TEMPORARY TABLE ""{_tableName}"" (""Value"" {_valuesType}) + ", token) + .ConfigureAwait(false); + _isTable = true; foreach (var slice in _values.Chunk(LocalDatabase.CHUNK_SIZE)) { var parameterNames = slice.Select((_, i) => $"@p{i}").ToArray(); - var sql = $@"INSERT INTO ""{_tableName}"" (""Value"") VALUES {string.Join(", ", parameterNames.Select(p => $"({p})"))}"; + var sql = $@" + INSERT INTO ""{_tableName}"" (""Value"") + VALUES {string.Join(", ", parameterNames.Select(p => $"({p})"))} + "; - _cmd.CommandText = sql; + _cmd.SetCommandAndParameters(sql); for (int i = 0; i < slice.Length; i++) - _cmd.AddNamedParameter(parameterNames[i], slice[i]); + _cmd.SetParameterValue(parameterNames[i], slice[i]); - _cmd.ExecuteNonQuery(); + await _cmd.ExecuteNonQueryAsync().ConfigureAwait(false); } } /// - /// The table name to use + /// The table name to use. /// public string TableName => _tableName; /// public void Dispose() + { + DisposeAsync().AsTask().Await(); + } + + /// + public async ValueTask DisposeAsync() { if (_disposed) return; @@ -175,8 +248,11 @@ public class TemporaryDbValueList : IDisposable try { if (_cmd != null) - _cmd.ExecuteNonQuery($@"DROP TABLE ""{_tableName}"""); - _cmd?.Dispose(); + { + await _cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{_tableName}""", default) + .ConfigureAwait(false); + await _cmd.DisposeAsync().ConfigureAwait(false); + } _cmd = null; } catch (Exception ex) diff --git a/Duplicati/Library/Main/IBackendManager.cs b/Duplicati/Library/Main/IBackendManager.cs index ada2c758e..18df634b5 100644 --- a/Duplicati/Library/Main/IBackendManager.cs +++ b/Duplicati/Library/Main/IBackendManager.cs @@ -28,7 +28,7 @@ internal interface IBackendManager : IDisposable /// The action to call when the database should be updated /// The cancellation token /// An awaitable task - Task PutAsync(VolumeWriterBase blockVolume, IndexVolumeWriter? indexVolume, Action? indexVolumeFinished, bool waitForComplete, Func? onDbUpdate, CancellationToken cancelToken); + Task PutAsync(VolumeWriterBase blockVolume, IndexVolumeWriter? indexVolume, Func? indexVolumeFinished, bool waitForComplete, Func? onDbUpdate, CancellationToken cancelToken); /// /// Uploads a file to the backend without encryption @@ -50,10 +50,9 @@ internal interface IBackendManager : IDisposable /// Waits for the backend queue to be empty and flushes any pending messages to the database /// /// The database to write pending messages to - /// The transaction to use, if any /// The cancellation token /// An awaitable task - Task WaitForEmptyAsync(LocalDatabase database, IDbTransaction? transaction, CancellationToken cancellationToken); + Task WaitForEmptyAsync(LocalDatabase database, CancellationToken cancellationToken); /// /// Lists the files on the backend @@ -130,10 +129,9 @@ internal interface IBackendManager : IDisposable /// Flushes the database messages to the database /// /// The database to write to - /// The transaction to use /// The cancellation token /// - Task FlushPendingMessagesAsync(LocalDatabase database, IDbTransaction? transaction, CancellationToken cancellationToken); + Task FlushPendingMessagesAsync(LocalDatabase database, CancellationToken cancellationToken); /// /// Updates the throttle values for upload and download diff --git a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs index ccaa96d5e..e4f5635f4 100644 --- a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs +++ b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -28,7 +28,7 @@ using System.Collections.Generic; using Duplicati.Library.Main.Volumes; using System.Linq; using Duplicati.Library.Interface; - +using System.Threading; namespace Duplicati.Library.Main.Operation.Backup { @@ -55,255 +55,400 @@ namespace Duplicati.Library.Main.Operation.Backup m_database = database; } - public Task FindBlockIDAsync(string hash, long size) + public Task FindBlockIDAsync(string hash, long size, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.FindBlockID(hash, size, GetTransaction())); + return RunOnMain(async () => + await m_database + .FindBlockID(hash, size, cancellationToken) + .ConfigureAwait(false) + ); } - public Task AddBlockAsync(string hash, long size, long volumeid) + public Task AddBlockAsync(string hash, long size, long volumeid, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.AddBlock(hash, size, volumeid, GetTransaction())); + return RunOnMain(async () => + await m_database + .AddBlock(hash, size, volumeid, cancellationToken) + .ConfigureAwait(false) + ); } - public Task GetFileHashAsync(long fileid) + public Task GetFileHashAsync(long fileid, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.GetFileHash(fileid, GetTransaction())); + return RunOnMain(async () => + await m_database + .GetFileHash(fileid, cancellationToken) + .ConfigureAwait(false) + ); } - public Task> AddMetadatasetAsync(string hash, long size, long blocksetid) + public Task<(bool, long)> AddMetadatasetAsync(string hash, long size, long blocksetid, CancellationToken cancellationToken) { - return RunOnMain(() => + return RunOnMain(async () => + await m_database + .AddMetadataset(hash, size, blocksetid, cancellationToken) + .ConfigureAwait(false) + ); + } + + public Task<(bool, long)> GetMetadataIDAsync(string hash, long size, CancellationToken cancellationToken) + { + return RunOnMain(async () => + await m_database + .GetMetadatasetID(hash, size, cancellationToken) + .ConfigureAwait(false) + ); + } + + public Task AddDirectoryEntryAsync(string filename, long metadataid, DateTime lastModified, CancellationToken cancellationToken) + { + return RunOnMain(async () => + await m_database + .AddDirectoryEntry(filename, metadataid, lastModified, cancellationToken) + .ConfigureAwait(false) + ); + } + + public Task AddSymlinkEntryAsync(string filename, long metadataid, DateTime lastModified, CancellationToken cancellationToken) + { + return RunOnMain(async () => + await m_database + .AddSymlinkEntry(filename, metadataid, lastModified, cancellationToken) + .ConfigureAwait(false) + ); + } + + public Task<(string MetadataHash, long Size)?> GetMetadataHashAndSizeForFileAsync(long fileid, CancellationToken cancellationToken) + { + return RunOnMain(async () => + await m_database + .GetMetadataHashAndSizeForFile(fileid, cancellationToken) + .ConfigureAwait(false) + ); + } + + public Task<(long, DateTime, long)> GetFileLastModifiedAsync(long prefixid, string path, long lastfilesetid, bool includeLength, CancellationToken cancellationToken) + { + return RunOnMain(async () => + await m_database + .GetFileLastModified(prefixid, path, lastfilesetid, includeLength, cancellationToken) + .ConfigureAwait(false) + ); + } + + public Task GetFileEntryAsync(long prefixid, string path, long lastfilesetid, CancellationToken cancellationToken) + { + return RunOnMain(async () => { - long metadataid; - var n = m_database.AddMetadataset(hash, size, blocksetid, out metadataid, GetTransaction()); - return new Tuple(n, metadataid); - }); - } + var (id, oldModified, lastFileSize, oldMetahash, oldMetasize) = + await m_database.GetFileEntry(prefixid, path, lastfilesetid, cancellationToken) + .ConfigureAwait(false); - public Task> GetMetadataIDAsync(string hash, long size) - { - return RunOnMain(() => - { - long metadataid; - var r = m_database.GetMetadatasetID(hash, size, out metadataid, GetTransaction()); - return new Tuple(r, metadataid); - }); - } - - public Task AddDirectoryEntryAsync(string filename, long metadataid, DateTime lastModified) - { - return RunOnMain(() => m_database.AddDirectoryEntry(filename, metadataid, lastModified, GetTransaction())); - } - - public Task AddSymlinkEntryAsync(string filename, long metadataid, DateTime lastModified) - { - return RunOnMain(() => m_database.AddSymlinkEntry(filename, metadataid, lastModified, GetTransaction())); - } - - public Task<(long Size, string MetadataHash)?> GetMetadataHashAndSizeForFileAsync(long fileid) - { - return RunOnMain(() => m_database.GetMetadataHashAndSizeForFile(fileid, GetTransaction())); - } - - public Task> GetFileLastModifiedAsync(long prefixid, string path, long lastfilesetid, bool includeLength) - { - return RunOnMain(() => - { - var id = m_database.GetFileLastModified(prefixid, path, lastfilesetid, includeLength, out var lastModified, out var length, GetTransaction()); - return new Tuple(id, lastModified, length); - }); - } - - public Task GetFileEntryAsync(long prefixid, string path, long lastfilesetid) - { - return RunOnMain(() => - { - DateTime oldModified; - long lastFileSize; - string oldMetahash; - long oldMetasize; - - var id = m_database.GetFileEntry(prefixid, path, lastfilesetid, out oldModified, out lastFileSize, out oldMetahash, out oldMetasize, GetTransaction()); return id < 0 ? - null : - new FileEntryData() { id = id, modified = oldModified, filesize = lastFileSize, metahash = oldMetahash, metasize = oldMetasize }; + null + : + new FileEntryData() + { + id = id, + modified = oldModified, + filesize = lastFileSize, + metahash = oldMetahash, + metasize = oldMetasize + }; }); } - public Task AddBlocksetAsync(string filehash, long size, int blocksize, IEnumerable hashlist, IEnumerable blocklisthashes) + public Task AddBlocksetAsync(string filehash, long size, int blocksize, IEnumerable hashlist, IEnumerable blocklisthashes, CancellationToken cancellationToken) { - return RunOnMain(() => + return RunOnMain(async () => { - long blocksetid; + var (_, blocksetid) = await m_database + .AddBlockset(filehash, size, blocksize, hashlist, blocklisthashes, cancellationToken) + .ConfigureAwait(false); - m_database.AddBlockset(filehash, size, blocksize, hashlist, blocklisthashes, out blocksetid, GetTransaction()); return blocksetid; }); } - public Task GetOrCreatePathPrefix(string prefix) + public Task GetOrCreatePathPrefix(string prefix, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.GetOrCreatePathPrefix(prefix, GetTransaction())); + return RunOnMain(async () => + await m_database + .GetOrCreatePathPrefix(prefix, cancellationToken) + .ConfigureAwait(false) + ); } - public Task AddFileAsync(long prefixid, string filename, DateTime lastmodified, long blocksetid, long metadataid) + public Task AddFileAsync(long prefixid, string filename, DateTime lastmodified, long blocksetid, long metadataid, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.AddFile(prefixid, filename, lastmodified, blocksetid, metadataid, GetTransaction())); + return RunOnMain(async () => + await m_database + .AddFile(prefixid, filename, lastmodified, blocksetid, metadataid, cancellationToken) + .ConfigureAwait(false) + ); } - public Task AddUnmodifiedAsync(long fileid, DateTime lastModified) + public Task AddUnmodifiedAsync(long fileid, DateTime lastModified, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.AddKnownFile(fileid, lastModified, GetTransaction())); + return RunOnMain(async () => + await m_database + .AddKnownFile(fileid, lastModified, cancellationToken) + .ConfigureAwait(false) + ); } - public Task MoveBlockToVolumeAsync(string blockkey, long size, long sourcevolumeid, long targetvolumeid) + public Task MoveBlockToVolumeAsync(string blockkey, long size, long sourcevolumeid, long targetvolumeid, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.MoveBlockToVolume(blockkey, size, sourcevolumeid, targetvolumeid, GetTransaction())); + return RunOnMain(async () => + await m_database + .MoveBlockToVolume(blockkey, size, sourcevolumeid, targetvolumeid, cancellationToken) + .ConfigureAwait(false) + ); } - public Task SafeDeleteRemoteVolumeAsync(string remotename) + public Task SafeDeleteRemoteVolumeAsync(string remotename, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.SafeDeleteRemoteVolume(remotename, GetTransaction())); + return RunOnMain(async () => + await m_database + .SafeDeleteRemoteVolume(remotename, cancellationToken) + .ConfigureAwait(false) + ); } - public Task GetBlocklistHashesAsync(string remotename) + public Task GetBlocklistHashesAsync(string remotename, CancellationToken cancellationToken) { - return RunOnMain(() => Task.FromResult(m_database.GetBlocklistHashes(remotename, GetTransaction()))); + return RunOnMain(async () => + await m_database + .GetBlocklistHashes(remotename, cancellationToken) + .ConfigureAwait(false) + ); } - public Task UpdateIndexVolumeAsync(IndexVolumeWriter indexvolume, BlockVolumeWriter blockvolume) + public Task UpdateIndexVolumeAsync(IndexVolumeWriter indexvolume, BlockVolumeWriter blockvolume, CancellationToken cancellationToken) { if (indexvolume != null) return Task.FromResult(true); - return RunOnMain(() => + return RunOnMain(async () => { - m_database.AddIndexBlockLink(indexvolume.VolumeID, blockvolume.VolumeID, GetTransaction()); + await m_database + .AddIndexBlockLink(indexvolume.VolumeID, blockvolume.VolumeID, cancellationToken) + .ConfigureAwait(false); + indexvolume.StartVolume(blockvolume.RemoteFilename); - foreach (var b in m_database.GetBlocks(blockvolume.VolumeID, GetTransaction())) + await foreach (var b in m_database.GetBlocks(blockvolume.VolumeID, cancellationToken).ConfigureAwait(false)) indexvolume.AddBlock(b.Hash, b.Size); - m_database.UpdateRemoteVolume(indexvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, GetTransaction()); + await m_database + .UpdateRemoteVolume(indexvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, cancellationToken) + .ConfigureAwait(false); }); } - public Task AppendFilesFromPreviousSetAsync() + public Task AppendFilesFromPreviousSetAsync(CancellationToken cancellationToken) { - return RunOnMain(() => m_database.AppendFilesFromPreviousSet(GetTransaction())); + return RunOnMain(async () => + await m_database + .AppendFilesFromPreviousSet(null, cancellationToken) + .ConfigureAwait(false) + ); } - public Task AppendFilesFromPreviousSetAsync(string[] deletedFilelist) + public Task AppendFilesFromPreviousSetAsync(string[] deletedFilelist, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.AppendFilesFromPreviousSet(GetTransaction(), deletedFilelist)); + return RunOnMain(async () => + await m_database + .AppendFilesFromPreviousSet(deletedFilelist, cancellationToken) + .ConfigureAwait(false) + ); } - public Task AppendFilesFromPreviousSetAsync(string[] deletedFilelist, long filesetid, long prevId, DateTime timestamp) + public Task AppendFilesFromPreviousSetAsync(string[] deletedFilelist, long filesetid, long prevId, DateTime timestamp, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.AppendFilesFromPreviousSet(GetTransaction(), deletedFilelist, filesetid, prevId, timestamp)); + return RunOnMain(async () => + await m_database + .AppendFilesFromPreviousSet(deletedFilelist, filesetid, prevId, timestamp, cancellationToken) + .ConfigureAwait(false) + ); } /// - /// Populates FilesetEntry table with files from previous fileset, which aren't - /// yet part of the new fileset, and which aren't excluded by the (optional) exclusion + /// Populates FilesetEntry table with files from previous fileset, which aren't + /// yet part of the new fileset, and which aren't excluded by the (optional) exclusion /// predicate. /// /// Optional exclusion predicate (true = exclude file) - public Task AppendFilesFromPreviousSetWithPredicateAsync(Func exclusionPredicate) + public Task AppendFilesFromPreviousSetWithPredicateAsync(Func exclusionPredicate, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.AppendFilesFromPreviousSetWithPredicate(GetTransaction(), exclusionPredicate)); + return RunOnMain(async () => + await m_database + .AppendFilesFromPreviousSetWithPredicate(exclusionPredicate, cancellationToken) + .ConfigureAwait(false) + ); } /// - /// Populates FilesetEntry table with files from previous fileset, which aren't - /// yet part of the new fileset, and which aren't excluded by the (optional) exclusion + /// Populates FilesetEntry table with files from previous fileset, which aren't + /// yet part of the new fileset, and which aren't excluded by the (optional) exclusion /// predicate. /// - /// Optional exclusion predicate (true = exclude file) - /// Current fileset ID - /// Source fileset ID - /// If prevFileSetId == -1, used to locate previous fileset - public Task AppendFilesFromPreviousSetWithPredicateAsync(Func exclusionPredicate, long fileSetId, long prevFileSetId, DateTime timestamp) + /// Optional exclusion predicate (true = exclude file). + /// Current fileset ID. + /// Source fileset ID. + /// If prevFileSetId == -1, used to locate previous fileset. + /// The cancellation token to cancel the operation. + /// Task that completes when the files have been appended. + public Task AppendFilesFromPreviousSetWithPredicateAsync(Func exclusionPredicate, long fileSetId, long prevFileSetId, DateTime timestamp, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.AppendFilesFromPreviousSetWithPredicate(GetTransaction(), exclusionPredicate, fileSetId, prevFileSetId, timestamp)); + return RunOnMain(async () => + await m_database + .AppendFilesFromPreviousSetWithPredicate(exclusionPredicate, fileSetId, prevFileSetId, timestamp, cancellationToken) + .ConfigureAwait(false) + ); } - public Task[]> GetIncompleteFilesetsAsync() + public Task[]> GetIncompleteFilesetsAsync(CancellationToken cancellationToken) { - return RunOnMain(() => m_database.GetIncompleteFilesets(GetTransaction()).OrderBy(x => x.Value).ToArray()); + return RunOnMain(async () => + await m_database + .GetIncompleteFilesets(cancellationToken) + .OrderBy(x => x.Value) + .ToArrayAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false) + ); } - public Task[]> GetFilesetTimesAsync() + public Task[]> GetFilesetTimesAsync(CancellationToken cancellationToken) { - return RunOnMain(() => m_database.FilesetTimes.ToArray()); + return RunOnMain(async () => + await m_database + .FilesetTimes(cancellationToken) + .ToArrayAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false) + ); } - public Task CreateFilesetAsync(long volumeID, DateTime fileTime) + public Task CreateFilesetAsync(long volumeID, DateTime fileTime, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.CreateFileset(volumeID, fileTime, GetTransaction())); + return RunOnMain(async () => + { + var fs = await m_database + .CreateFileset(volumeID, fileTime, cancellationToken) + .ConfigureAwait(false); + + await m_database.Transaction + .CommitAsync(cancellationToken) + .ConfigureAwait(false); + + return fs; + }); } - public Task LinkFilesetToVolumeAsync(long filesetid, long volumeid) + public Task LinkFilesetToVolumeAsync(long filesetid, long volumeid, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.LinkFilesetToVolume(filesetid, volumeid, GetTransaction())); + return RunOnMain(async () => + await m_database + .LinkFilesetToVolume(filesetid, volumeid, cancellationToken) + .ConfigureAwait(false) + ); } - public Task WriteFilesetAsync(FilesetVolumeWriter fsw, long filesetid) + public Task WriteFilesetAsync(FilesetVolumeWriter fsw, long filesetid, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.WriteFileset(fsw, filesetid, GetTransaction())); + return RunOnMain(async () => + await m_database + .WriteFileset(fsw, filesetid, cancellationToken) + .ConfigureAwait(false) + ); } - public Task PushTimestampChangesToPreviousVersionAsync(long filesetid) + public Task PushTimestampChangesToPreviousVersionAsync(long filesetid, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.PushTimestampChangesToPreviousVersion(filesetid, GetTransaction())); + return RunOnMain(async () => + await m_database + .PushTimestampChangesToPreviousVersion(filesetid, cancellationToken) + .ConfigureAwait(false) + ); } - public Task UpdateFilesetAndMarkAsFullBackupAsync(long filesetid) + public Task UpdateFilesetAndMarkAsFullBackupAsync(long filesetid, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.UpdateFullBackupStateInFileset(filesetid, true, GetTransaction())); + return RunOnMain(async () => + await m_database + .UpdateFullBackupStateInFileset(filesetid, true, cancellationToken) + .ConfigureAwait(false) + ); } - // TODO: Make IAsyncEnumerable or get rid of the SingleRunner setup - public Task GetMissingIndexFilesAsync() + public Task> GetMissingIndexFilesAsync(CancellationToken cancellationToken) { - return RunOnMain(() => m_database.GetMissingIndexFiles(GetTransaction()).ToArray()); + return RunOnMain(() => m_database.GetMissingIndexFiles(cancellationToken)); } - public Task UpdateChangeStatisticsAsync(BackupResults result) + public Task UpdateChangeStatisticsAsync(BackupResults result, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.UpdateChangeStatistics(result, GetTransaction())); + return RunOnMain(async () => + await m_database + .UpdateChangeStatistics(result, cancellationToken) + .ConfigureAwait(false) + ); } - public Task VerifyConsistencyAsync(int blocksize, int blockhashSize, bool verifyfilelists) + public Task VerifyConsistencyAsync(int blocksize, int blockhashSize, bool verifyfilelists, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.VerifyConsistency(blocksize, blockhashSize, verifyfilelists, GetTransaction())); + return RunOnMain(async () => + await m_database + .VerifyConsistency(blocksize, blockhashSize, verifyfilelists, cancellationToken) + .ConfigureAwait(false) + ); } - public Task RemoveRemoteVolumeAsync(string remoteFilename) + public Task RemoveRemoteVolumeAsync(string remoteFilename, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.RemoveRemoteVolume(remoteFilename, GetTransaction())); + return RunOnMain(async () => + await m_database + .RemoveRemoteVolume(remoteFilename, cancellationToken) + .ConfigureAwait(false) + ); } - public Task GetRemoteVolumeFromFilesetIDAsync(long filesetID) + public Task GetRemoteVolumeFromFilesetIDAsync(long filesetID, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.GetRemoteVolumeFromFilesetID(filesetID, GetTransaction())); + return RunOnMain(async () => + await m_database + .GetRemoteVolumeFromFilesetID(filesetID, cancellationToken) + .ConfigureAwait(false) + ); } - public Task CreateChangeJournalDataAsync(IEnumerable journalData) + public Task CreateChangeJournalDataAsync(IEnumerable journalData, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.CreateChangeJournalData(journalData, GetTransaction())); + return RunOnMain(async () => + await m_database + .CreateChangeJournalData(journalData, cancellationToken) + .ConfigureAwait(false) + ); } - public Task UpdateChangeJournalDataAsync(IEnumerable journalData, long lastfilesetid) + public Task UpdateChangeJournalDataAsync(IEnumerable journalData, long lastfilesetid, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.UpdateChangeJournalData(journalData, lastfilesetid, GetTransaction())); + return RunOnMain(async () => + await m_database + .UpdateChangeJournalData(journalData, lastfilesetid, cancellationToken) + .ConfigureAwait(false) + ); } - public Task IsBlocklistHashKnownAsync(string hash) + public Task IsBlocklistHashKnownAsync(string hash, CancellationToken cancellationToken) { - return RunOnMain(() => m_database.IsBlocklistHashKnown(hash, GetTransaction())); + return RunOnMain(async () => + await m_database + .IsBlocklistHashKnown(hash, cancellationToken) + .ConfigureAwait(false) + ); } } diff --git a/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs b/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs index 01e4f123a..bcee2e525 100644 --- a/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs +++ b/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using CoCoL; @@ -82,7 +82,7 @@ namespace Duplicati.Library.Main.Operation.Backup // There can be a race, such that two workers determine that // the block is missing, but this will be solved by the AddBlock call // which runs atomically - if (!isMandatoryBlocklistHash && await database.FindBlockIDAsync(b.HashKey, b.Size) >= 0) + if (!isMandatoryBlocklistHash && await database.FindBlockIDAsync(b.HashKey, b.Size, taskreader.ProgressToken) >= 0) { b.TaskCompletion.TrySetResult(false); sw_workload.Stop(); @@ -90,13 +90,13 @@ namespace Duplicati.Library.Main.Operation.Backup } blockvolume = new BlockVolumeWriter(options); - blockvolume.VolumeID = await database.RegisterRemoteVolumeAsync(blockvolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary); + blockvolume.VolumeID = await database.RegisterRemoteVolumeAsync(blockvolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, taskreader.ProgressToken); indexvolume = noIndexFiles ? null : new TemporaryIndexVolume(options); blocklistHashesAdded = 0; } - var newBlock = await database.AddBlockAsync(b.HashKey, b.Size, blockvolume.VolumeID); + var newBlock = await database.AddBlockAsync(b.HashKey, b.Size, blockvolume.VolumeID, taskreader.ProgressToken); b.TaskCompletion.TrySetResult(newBlock); // If we are recording blocklist hashes, add them to the index file, @@ -105,7 +105,7 @@ namespace Duplicati.Library.Main.Operation.Backup { // This can cause a race between workers, // but the side-effect is that the index files are slightly larger - if (newBlock || !await database.IsBlocklistHashKnownAsync(b.HashKey)) + if (newBlock || !await database.IsBlocklistHashKnownAsync(b.HashKey, taskreader.ProgressToken)) { blocklistHashesAdded++; indexvolume.AddBlockListHash(b.HashKey, b.Size, b.Data); @@ -114,7 +114,8 @@ namespace Duplicati.Library.Main.Operation.Backup if (newBlock) { - blockvolume.AddBlock(b.HashKey, b.Data, b.Offset, (int)b.Size, b.Hint); + await blockvolume.AddBlock(b.HashKey, b.Data, b.Offset, (int)b.Size, b.Hint) + .ConfigureAwait(false); if (indexvolume != null) indexvolume.AddBlock(b.HashKey, b.Size); @@ -123,7 +124,7 @@ namespace Duplicati.Library.Main.Operation.Backup { //When uploading a new volume, we register the volumes and then flush the transaction // this ensures that the local database and remote storage are as closely related as possible - await database.UpdateRemoteVolumeAsync(blockvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null); + await database.UpdateRemoteVolumeAsync(blockvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, false, default, null, taskreader.ProgressToken); blockvolume.Close(); @@ -132,17 +133,17 @@ namespace Duplicati.Library.Main.Operation.Backup { // TODO: It is much easier to let the BackendManager deal with index files, // but it adds a bit of strain to the database - indexVolumeCopy = await indexvolume.CreateVolume(blockvolume.RemoteFilename, options, database); + indexVolumeCopy = await indexvolume.CreateVolume(blockvolume.RemoteFilename, options, database, taskreader.ProgressToken).ConfigureAwait(false); // Create link before upload is started, it will be removed later if upload fails - await database.AddIndexBlockLinkAsync(indexVolumeCopy.VolumeID, blockvolume.VolumeID).ConfigureAwait(false); + await database.AddIndexBlockLinkAsync(indexVolumeCopy.VolumeID, blockvolume.VolumeID, taskreader.ProgressToken).ConfigureAwait(false); } var blockVolumeCopy = blockvolume; blockvolume = null; indexvolume = null; - await database.CommitTransactionAsync("CommitAddBlockToOutputFlush"); - await backendManager.PutAsync(blockVolumeCopy, indexVolumeCopy, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager), taskreader.ProgressToken); + await database.CommitTransactionAsync("CommitAddBlockToOutputFlush", true, taskreader.ProgressToken).ConfigureAwait(false); + await backendManager.PutAsync(blockVolumeCopy, indexVolumeCopy, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken).ConfigureAwait(false); } @@ -163,11 +164,15 @@ namespace Duplicati.Library.Main.Operation.Backup { if (blockvolume.SourceSize > 0 || blocklistHashesAdded > 0) { - await self.SpillPickup.WriteAsync(new SpillVolumeRequest(blockvolume, indexvolume)); + await self.SpillPickup + .WriteAsync(new SpillVolumeRequest(blockvolume, indexvolume)) + .ConfigureAwait(false); } else { - await database.RemoveRemoteVolumeAsync(blockvolume.RemoteFilename); + await database + .RemoveRemoteVolumeAsync(blockvolume.RemoteFilename, taskreader.ProgressToken) + .ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs index b404232f0..d1ffd97d3 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -49,7 +49,7 @@ namespace Duplicati.Library.Main.Operation.Backup { while (true) { - var e = await self.Input.ReadAsync(); + var e = await self.Input.ReadAsync().ConfigureAwait(false); // We ignore the stop signal, but not the pause and terminate await taskreader.ProgressRendevouz().ConfigureAwait(false); @@ -57,7 +57,7 @@ namespace Duplicati.Library.Main.Operation.Backup try { var hint = options.GetCompressionHintFromFilename(e.Entry.Path); - var oldHash = e.OldId < 0 ? null : await database.GetFileHashAsync(e.OldId); + var oldHash = e.OldId < 0 ? null : await database.GetFileHashAsync(e.OldId, taskreader.ProgressToken).ConfigureAwait(false); StreamProcessResult filestreamdata; @@ -68,7 +68,7 @@ namespace Duplicati.Library.Main.Operation.Backup // If we have determined that metadata has not changed, just grab the ID if (!e.MetadataChanged) { - var res = await database.GetMetadataIDAsync(e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length); + var res = await database.GetMetadataIDAsync(e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length, taskreader.ProgressToken).ConfigureAwait(false); if (res.Item1) return res.Item2; @@ -76,15 +76,15 @@ namespace Duplicati.Library.Main.Operation.Backup e.MetadataChanged = true; } - return (await MetadataPreProcess.AddMetadataToOutputAsync(e.Entry.Path, e.MetaHashAndSize, database, self.StreamBlockChannel)).Item2; + return (await MetadataPreProcess.AddMetadataToOutputAsync(e.Entry.Path, e.MetaHashAndSize, database, self.StreamBlockChannel, taskreader.ProgressToken).ConfigureAwait(false)).Item2; }); - using (var fs = await e.Entry.OpenRead(taskreader.ProgressToken)) - filestreamdata = await StreamBlock.ProcessStream(self.StreamBlockChannel, e.Entry.Path, fs, false, hint); + using (var fs = await e.Entry.OpenRead(taskreader.ProgressToken).ConfigureAwait(false)) + filestreamdata = await StreamBlock.ProcessStream(self.StreamBlockChannel, e.Entry.Path, fs, false, hint).ConfigureAwait(false); - await stats.AddOpenedFile(filestreamdata.Streamlength); + await stats.AddOpenedFile(filestreamdata.Streamlength).ConfigureAwait(false); - var metadataid = await metatask; + var metadataid = await metatask.ConfigureAwait(false); var filekey = filestreamdata.Streamhash; var filesize = filestreamdata.Streamlength; @@ -110,18 +110,18 @@ namespace Duplicati.Library.Main.Operation.Backup Logging.Log.WriteVerboseMessage(FILELOGTAG, "WouldAddChangedFile", "Would add changed file {0}, size {1}", e.Entry.Path, Library.Utility.Utility.FormatSizeString(filesize)); } - await database.AddFileAsync(e.PathPrefixID, e.Filename, e.LastWrite, filestreamdata.Blocksetid, metadataid); + await database.AddFileAsync(e.PathPrefixID, e.Filename, e.LastWrite, filestreamdata.Blocksetid, metadataid, taskreader.ProgressToken).ConfigureAwait(false); } else if (e.MetadataChanged) { Logging.Log.WriteVerboseMessage(FILELOGTAG, "FileMetadataChanged", "File has only metadata changes {0}", e.Entry.Path); - await database.AddFileAsync(e.PathPrefixID, e.Filename, e.LastWrite, filestreamdata.Blocksetid, metadataid); + await database.AddFileAsync(e.PathPrefixID, e.Filename, e.LastWrite, filestreamdata.Blocksetid, metadataid, taskreader.ProgressToken).ConfigureAwait(false); } else if (e.TimestampChanged) { await stats.AddTimestampChangedFile(); Logging.Log.WriteVerboseMessage(FILELOGTAG, "FileTimestampChanged", "File has only timestamp changes {0}", e.Entry.Path); - await database.AddUnmodifiedAsync(e.OldId, e.LastWrite); + await database.AddUnmodifiedAsync(e.OldId, e.LastWrite, taskreader.ProgressToken).ConfigureAwait(false); } else /*if (e.OldId >= 0)*/ { @@ -130,7 +130,7 @@ namespace Duplicati.Library.Main.Operation.Backup try { - await database.AddUnmodifiedAsync(e.OldId, e.LastWrite); + await database.AddUnmodifiedAsync(e.OldId, e.LastWrite, taskreader.ProgressToken).ConfigureAwait(false); } catch (Exception ex) { diff --git a/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs b/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs index 744606614..7001cae05 100644 --- a/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -40,7 +40,7 @@ namespace Duplicati.Library.Main.Operation.Backup /// private static readonly string FILELOGTAG = Logging.Log.LogTagFromType(typeof(FilePreFilterProcess)) + ".FileEntry"; - public static Task Run(Channels channels, Options options, BackupStatsCollector stats, BackupDatabase database) + public static Task Run(Channels channels, Options options, BackupStatsCollector stats, BackupDatabase database, ITaskReader taskreader) { return AutomationExtensions.RunTask( new @@ -105,7 +105,7 @@ namespace Duplicati.Library.Main.Operation.Backup Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Entry.Path); try { - await database.AddUnmodifiedAsync(e.OldId, e.LastWrite); + await database.AddUnmodifiedAsync(e.OldId, e.LastWrite, taskreader.ProgressToken); } catch (Exception ex) { @@ -121,7 +121,7 @@ namespace Duplicati.Library.Main.Operation.Backup // but we want to know if the metadata is potentially changed if (!isNewFile && DISABLEFILETIMECHECK) { - var tp = await database.GetMetadataHashAndSizeForFileAsync(e.OldId); + var tp = await database.GetMetadataHashAndSizeForFileAsync(e.OldId, taskreader.ProgressToken); if (tp != null) { e.OldMetaSize = tp.Value.Size; @@ -145,7 +145,7 @@ namespace Duplicati.Library.Main.Operation.Backup Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoMetadataChange", "Skipped checking file, because no metadata was updated {0}", e.Entry.Path); try { - await database.AddUnmodifiedAsync(e.OldId, e.LastWrite); + await database.AddUnmodifiedAsync(e.OldId, e.LastWrite, taskreader.ProgressToken); } catch (Exception ex) { diff --git a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs index defdee0bb..465e7c4c5 100644 --- a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -26,6 +26,7 @@ using System.IO; using System.Collections.Generic; using Duplicati.Library.Interface; using Duplicati.Library.Main.Operation.Common; +using System.Threading; namespace Duplicati.Library.Main.Operation.Backup { @@ -63,7 +64,7 @@ namespace Duplicati.Library.Main.Operation.Backup public bool TimestampChanged; } - public static Task Run(Channels channels, Options options, BackupDatabase database, long lastfilesetid, ITaskReader taskReader) + public static Task Run(Channels channels, Options options, BackupDatabase database, long lastfilesetid, ITaskReader taskreader) { return AutomationExtensions.RunTask(new { @@ -85,7 +86,7 @@ namespace Duplicati.Library.Main.Operation.Backup var entry = await self.Input.ReadAsync(); // We ignore the stop signal, but not the pause and terminate - await taskReader.ProgressRendevouz().ConfigureAwait(false); + await taskreader.ProgressRendevouz().ConfigureAwait(false); var lastwrite = new DateTime(0, DateTimeKind.Utc); var attributes = entry.IsFolder @@ -111,7 +112,7 @@ namespace Duplicati.Library.Main.Operation.Backup } // If we only have metadata, stop here - if (await ProcessMetadata(entry, attributes, lastwrite, options, emptymetadata, database, self.StreamBlockChannel).ConfigureAwait(false)) + if (await ProcessMetadata(entry, attributes, lastwrite, options, emptymetadata, database, self.StreamBlockChannel, taskreader.ProgressToken).ConfigureAwait(false)) { try { @@ -122,13 +123,13 @@ namespace Duplicati.Library.Main.Operation.Backup prefixid = prevprefix.Value; else { - prefixid = await database.GetOrCreatePathPrefix(split.Key); + prefixid = await database.GetOrCreatePathPrefix(split.Key, taskreader.ProgressToken); prevprefix = new KeyValuePair(split.Key, prefixid); } if (CHECKFILETIMEONLY || DISABLEFILETIMECHECK) { - var tmp = await database.GetFileLastModifiedAsync(prefixid, split.Value, lastfilesetid, false); + var tmp = await database.GetFileLastModifiedAsync(prefixid, split.Value, lastfilesetid, false, taskreader.ProgressToken); await self.Output.WriteAsync(new FileEntry { OldId = tmp.Item1, @@ -145,7 +146,7 @@ namespace Duplicati.Library.Main.Operation.Backup } else { - var res = await database.GetFileEntryAsync(prefixid, split.Value, lastfilesetid); + var res = await database.GetFileEntryAsync(prefixid, split.Value, lastfilesetid, taskreader.ProgressToken); await self.Output.WriteAsync(new FileEntry { OldId = res == null ? -1 : res.id, @@ -178,8 +179,16 @@ namespace Duplicati.Library.Main.Operation.Backup /// /// Processes the metadata for the given path. /// + /// The source provider entry to process. + /// The file attributes of the entry. + /// The last write timestamp of the entry. + /// The options for the backup operation. + /// An empty metadata object to use if no metadata is generated. + /// The backup database to use for storing metadata. + /// The channel to write stream blocks to. + /// The cancellation token to use for the operation. /// True if the path should be submitted to more analysis, false if there is nothing else to do - private static async Task ProcessMetadata(ISourceProviderEntry entry, FileAttributes attributes, DateTime lastwrite, Options options, IMetahash emptymetadata, BackupDatabase database, IWriteChannel streamblockchannel) + private static async Task ProcessMetadata(ISourceProviderEntry entry, FileAttributes attributes, DateTime lastwrite, Options options, IMetahash emptymetadata, BackupDatabase database, IWriteChannel streamblockchannel, CancellationToken cancellationToken) { if (entry.IsSymlink) { @@ -213,7 +222,7 @@ namespace Duplicati.Library.Main.Operation.Backup metadata["CoreSymlinkTarget"] = symlinkTarget; var metahash = Utility.WrapMetadata(metadata, options); - await AddSymlinkToOutputAsync(entry.Path, DateTime.UtcNow, metahash, database, streamblockchannel).ConfigureAwait(false); + await AddSymlinkToOutputAsync(entry.Path, DateTime.UtcNow, metahash, database, streamblockchannel, cancellationToken).ConfigureAwait(false); Logging.Log.WriteVerboseMessage(FILELOGTAG, "StoreSymlink", "Stored symlink {0}", entry.Path); // Don't process further @@ -241,7 +250,7 @@ namespace Duplicati.Library.Main.Operation.Backup } Logging.Log.WriteVerboseMessage(FILELOGTAG, "AddDirectory", "Adding directory {0}", entry.Path); - await AddFolderToOutputAsync(entry.Path, lastwrite, metahash, database, streamblockchannel).ConfigureAwait(false); + await AddFolderToOutputAsync(entry.Path, lastwrite, metahash, database, streamblockchannel, cancellationToken).ConfigureAwait(false); return false; } @@ -252,29 +261,19 @@ namespace Duplicati.Library.Main.Operation.Backup /// /// Adds metadata to output, and returns the metadataset ID /// - /// The metadataset ID. /// The path for which metadata is processed. /// The metadata entry. /// The database connection. /// The channel to write streams to. - internal static async Task> AddMetadataToOutputAsync(string path, IMetahash meta, BackupDatabase database, IWriteChannel streamblockchannel) + /// The cancellation token to use. + /// A task that when awaited returns a tuple with the first value indicating if the metadata set was added, and the second value being the metadata ID. + internal static async Task<(bool, long)> AddMetadataToOutputAsync(string path, IMetahash meta, BackupDatabase database, IWriteChannel streamblockchannel, CancellationToken cancellationToken) { StreamProcessResult res; using (var ms = new MemoryStream(meta.Blob)) res = await StreamBlock.ProcessStream(streamblockchannel, path, ms, true, CompressionHint.Default); - return await database.AddMetadatasetAsync(res.Streamhash, res.Streamlength, res.Blocksetid); - } - - /// - /// Adds a file to the output, - /// - /// The name of the file to record - /// The value of the lastModified timestamp - private static async Task AddFolderToOutputAsync(string filename, DateTime lastModified, IMetahash meta, BackupDatabase database, IWriteChannel streamblockchannel) - { - var metadataid = await AddMetadataToOutputAsync(filename, meta, database, streamblockchannel).ConfigureAwait(false); - await database.AddDirectoryEntryAsync(filename, metadataid.Item2, lastModified); + return await database.AddMetadatasetAsync(res.Streamhash, res.Streamlength, res.Blocksetid, cancellationToken).ConfigureAwait(false); } /// @@ -282,13 +281,31 @@ namespace Duplicati.Library.Main.Operation.Backup /// /// The name of the file to record /// The value of the lastModified timestamp + /// The metadata to record /// The database to use /// The channel to write blocks to - /// The metadata ti record - private static async Task AddSymlinkToOutputAsync(string filename, DateTime lastModified, IMetahash meta, BackupDatabase database, IWriteChannel streamblockchannel) + /// The cancellation token to use + /// A task that completes when the file has been added to the output. + private static async Task AddFolderToOutputAsync(string filename, DateTime lastModified, IMetahash meta, BackupDatabase database, IWriteChannel streamblockchannel, CancellationToken cancellationToken) { - var metadataid = await AddMetadataToOutputAsync(filename, meta, database, streamblockchannel).ConfigureAwait(false); - await database.AddSymlinkEntryAsync(filename, metadataid.Item2, lastModified); + var metadataid = await AddMetadataToOutputAsync(filename, meta, database, streamblockchannel, cancellationToken).ConfigureAwait(false); + await database.AddDirectoryEntryAsync(filename, metadataid.Item2, lastModified, cancellationToken).ConfigureAwait(false); + } + + /// + /// Adds a file to the output, + /// + /// The name of the file to record + /// The value of the lastModified timestamp + /// The metadata ti record + /// The database to use + /// The channel to write blocks to + /// The cancellation token to use + /// A task that completes when the file has been added to the output. + private static async Task AddSymlinkToOutputAsync(string filename, DateTime lastModified, IMetahash meta, BackupDatabase database, IWriteChannel streamblockchannel, CancellationToken cancellationToken) + { + var metadataid = await AddMetadataToOutputAsync(filename, meta, database, streamblockchannel, cancellationToken).ConfigureAwait(false); + await database.AddSymlinkEntryAsync(filename, metadataid.Item2, lastModified, cancellationToken).ConfigureAwait(false); } } diff --git a/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs b/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs index 608ff7b7e..0b1008d23 100644 --- a/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs +++ b/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.Threading.Tasks; @@ -35,20 +35,20 @@ internal static class RecreateMissingIndexFiles { if (options.IndexfilePolicy != Options.IndexFileStrategy.None) { - foreach (var blockfile in await database.GetMissingIndexFilesAsync().ConfigureAwait(false)) + await foreach (var blockfile in (await database.GetMissingIndexFilesAsync(taskreader.ProgressToken).ConfigureAwait(false)).ConfigureAwait(false)) { if (!await taskreader.ProgressRendevouz().ConfigureAwait(false)) return; Logging.Log.WriteInformationMessage(LOGTAG, "RecreateMissingIndexFile", "Re-creating missing index file for {0}", blockfile); - var w = await Common.IndexVolumeCreator.CreateIndexVolume(blockfile, options, database).ConfigureAwait(false); + var w = await Common.IndexVolumeCreator.CreateIndexVolume(blockfile, options, database, taskreader.ProgressToken).ConfigureAwait(false); if (!await taskreader.ProgressRendevouz().ConfigureAwait(false)) return; - await database.UpdateRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null).ConfigureAwait(false); - await database.CommitTransactionAsync("RecreateMissingIndexFile").ConfigureAwait(false); - await backendManager.PutAsync(w, null, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager), taskreader.ProgressToken).ConfigureAwait(false); + await database.UpdateRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, false, default, null, taskreader.ProgressToken).ConfigureAwait(false); + await database.CommitTransactionAsync("RecreateMissingIndexFile", true, taskreader.ProgressToken).ConfigureAwait(false); + await backendManager.PutAsync(w, null, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken).ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs b/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs index 46d7fb02c..6a906da16 100644 --- a/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using CoCoL; @@ -79,7 +79,7 @@ namespace Duplicati.Library.Main.Operation.Backup { // No more targets, make one target = new SpillVolumeRequest(new BlockVolumeWriter(options), source.IndexVolume == null ? null : new TemporaryIndexVolume(options)); - target.BlockVolume.VolumeID = await database.RegisterRemoteVolumeAsync(target.BlockVolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary).ConfigureAwait(false); + target.BlockVolume.VolumeID = await database.RegisterRemoteVolumeAsync(target.BlockVolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, taskreader.ProgressToken).ConfigureAwait(false); } else { @@ -124,8 +124,8 @@ namespace Duplicati.Library.Main.Operation.Backup target = await GetNextTarget(source).ConfigureAwait(false); var len = rd.ReadBlock(file.Key, buffer); - target.BlockVolume.AddBlock(file.Key, buffer, 0, len, Duplicati.Library.Interface.CompressionHint.Default); - await database.MoveBlockToVolumeAsync(file.Key, len, source.BlockVolume.VolumeID, target.BlockVolume.VolumeID).ConfigureAwait(false); + await target.BlockVolume.AddBlock(file.Key, buffer, 0, len, Duplicati.Library.Interface.CompressionHint.Default).ConfigureAwait(false); + await database.MoveBlockToVolumeAsync(file.Key, len, source.BlockVolume.VolumeID, target.BlockVolume.VolumeID, taskreader.ProgressToken).ConfigureAwait(false); if (target.IndexVolume != null) target.IndexVolume.AddBlock(file.Key, len); @@ -141,7 +141,7 @@ namespace Duplicati.Library.Main.Operation.Backup // Make sure they are out of the database System.IO.File.Delete(source.BlockVolume.LocalFilename); - await database.SafeDeleteRemoteVolumeAsync(source.BlockVolume.RemoteFilename).ConfigureAwait(false); + await database.SafeDeleteRemoteVolumeAsync(source.BlockVolume.RemoteFilename, taskreader.ProgressToken).ConfigureAwait(false); // Re-inject the target if it has content if (target != null) @@ -167,13 +167,13 @@ namespace Duplicati.Library.Main.Operation.Backup { // TODO: It is much easier to let the BackendManager deal with index files, // but it adds a bit of strain to the database - indexVolumeCopy = await target.IndexVolume.CreateVolume(target.BlockVolume.RemoteFilename, options, database).ConfigureAwait(false); + indexVolumeCopy = await target.IndexVolume.CreateVolume(target.BlockVolume.RemoteFilename, options, database, taskreader.ProgressToken).ConfigureAwait(false); // Create link before upload is started, it will be removed later if upload fails - await database.AddIndexBlockLinkAsync(indexVolumeCopy.VolumeID, target.BlockVolume.VolumeID).ConfigureAwait(false); + await database.AddIndexBlockLinkAsync(indexVolumeCopy.VolumeID, target.BlockVolume.VolumeID, taskreader.ProgressToken).ConfigureAwait(false); } - await database.CommitTransactionAsync("UploadSpillVolume").ConfigureAwait(false); - await backendManager.PutAsync(target.BlockVolume, indexVolumeCopy, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager), taskreader.ProgressToken).ConfigureAwait(false); + await database.CommitTransactionAsync("UploadSpillVolume", true, taskreader.ProgressToken).ConfigureAwait(false); + await backendManager.PutAsync(target.BlockVolume, indexVolumeCopy, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken).ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Operation/Backup/StreamBlockSplitter.cs b/Duplicati/Library/Main/Operation/Backup/StreamBlockSplitter.cs index 6d63dfd91..d41d477fa 100644 --- a/Duplicati/Library/Main/Operation/Backup/StreamBlockSplitter.cs +++ b/Duplicati/Library/Main/Operation/Backup/StreamBlockSplitter.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -162,7 +162,7 @@ namespace Duplicati.Library.Main.Operation.Backup filehasher.TransformFinalBlock(new byte[0], 0, 0); var filehash = Convert.ToBase64String(filehasher.Hash); - var blocksetid = await database.AddBlocksetAsync(filehash, filesize, blocksize, hashcollector, blocklisthashes); + var blocksetid = await database.AddBlocksetAsync(filehash, filesize, blocksize, hashcollector, blocklisthashes, taskreader.ProgressToken); cur.SetResult(new StreamProcessResult() { Streamlength = filesize, Streamhash = filehash, Blocksetid = blocksetid }); cur = null; } diff --git a/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs b/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs index aa8a85d9c..f7ba8f087 100644 --- a/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs +++ b/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -39,7 +39,7 @@ internal static class UploadRealFilelist // Update the reported source and backend changes using (new Logging.Timer(LOGTAG, "UpdateChangeStatistics", "UpdateChangeStatistics")) - await db.UpdateChangeStatisticsAsync(result); + await db.UpdateChangeStatisticsAsync(result, taskreader.ProgressToken).ConfigureAwait(false); var changeCount = result.AddedFiles + result.ModifiedFiles + result.DeletedFiles + @@ -58,16 +58,16 @@ internal static class UploadRealFilelist // We ignore the stop signal, but not the pause and terminate await taskreader.ProgressRendevouz().ConfigureAwait(false); - await db.WriteFilesetAsync(filesetvolume, filesetid).ConfigureAwait(false); + await db.WriteFilesetAsync(filesetvolume, filesetid, taskreader.ProgressToken).ConfigureAwait(false); filesetvolume.Close(); // We ignore the stop signal, but not the pause and terminate await taskreader.ProgressRendevouz().ConfigureAwait(false); - await db.UpdateRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null).ConfigureAwait(false); - await db.CommitTransactionAsync("CommitUpdateRemoteVolume").ConfigureAwait(false); + await db.UpdateRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, false, default, null, taskreader.ProgressToken).ConfigureAwait(false); + await db.CommitTransactionAsync("CommitUpdateRemoteVolume", true, taskreader.ProgressToken).ConfigureAwait(false); - await backendManager.PutAsync(filesetvolume, null, null, false, () => db.FlushBackendMessagesAndCommitAsync(backendManager), taskreader.ProgressToken).ConfigureAwait(false); + await backendManager.PutAsync(filesetvolume, null, null, false, () => db.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken).ConfigureAwait(false); } } else @@ -75,14 +75,14 @@ internal static class UploadRealFilelist if (result.TimestampChangedFiles != 0) { Logging.Log.WriteInformationMessage(LOGTAG, "DetectedTimestampChanges", "Detected timestamp changes, but no data needs to be uploaded, pushing timestamp changes to the latest fileset"); - await db.PushTimestampChangesToPreviousVersionAsync(filesetid).ConfigureAwait(false); + await db.PushTimestampChangesToPreviousVersionAsync(filesetid, taskreader.ProgressToken).ConfigureAwait(false); } Logging.Log.WriteVerboseMessage(LOGTAG, "RemovingLeftoverTempFile", "removing temp files, as no data needs to be uploaded"); - await db.RemoveRemoteVolumeAsync(filesetvolume.RemoteFilename); + await db.RemoveRemoteVolumeAsync(filesetvolume.RemoteFilename, taskreader.ProgressToken).ConfigureAwait(false); } - await db.CommitTransactionAsync("CommitUpdateRemoteVolume"); + await db.CommitTransactionAsync("CommitUpdateRemoteVolume", true, taskreader.ProgressToken).ConfigureAwait(false); } } diff --git a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs index 76b80f852..e369779bc 100644 --- a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs +++ b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Main.Database; @@ -62,8 +62,8 @@ namespace Duplicati.Library.Main.Operation.Backup } // Ready to build and upload the synthetic list - await database.CommitTransactionAsync("PreSyntheticFilelist"); - var incompleteFilesets = (await database.GetIncompleteFilesetsAsync()).OrderBy(x => x.Value).ToList(); + await database.CommitTransactionAsync("PreSyntheticFilelist", true, taskreader.ProgressToken).ConfigureAwait(false); + var incompleteFilesets = (await database.GetIncompleteFilesetsAsync(taskreader.ProgressToken)).OrderBy(x => x.Value).ToList(); if (!incompleteFilesets.Any()) return; @@ -77,7 +77,7 @@ namespace Duplicati.Library.Main.Operation.Backup var incompleteSet = incompleteFilesets.Last(); var badIds = incompleteFilesets.Select(n => n.Key); - var prevs = (await database.GetFilesetTimesAsync()) + var prevs = (await database.GetFilesetTimesAsync(taskreader.ProgressToken)) .Where(n => n.Key < incompleteSet.Key && !badIds.Contains(n.Key)) .OrderBy(n => n.Key) .Select(n => n.Key) @@ -88,9 +88,9 @@ namespace Duplicati.Library.Main.Operation.Backup FilesetVolumeWriter fsw = null; try { - var fileTime = await FilesetVolumeWriter.ProbeUnusedFilenameName(database, options, incompleteSet.Value).ConfigureAwait(false); + var fileTime = await FilesetVolumeWriter.ProbeUnusedFilenameName(database, options, incompleteSet.Value, taskreader.ProgressToken).ConfigureAwait(false); fsw = new FilesetVolumeWriter(options, fileTime); - fsw.VolumeID = await database.RegisterRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary); + fsw.VolumeID = await database.RegisterRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, taskreader.ProgressToken).ConfigureAwait(false); if (!string.IsNullOrEmpty(options.ControlFiles)) foreach (var p in options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) @@ -99,27 +99,27 @@ namespace Duplicati.Library.Main.Operation.Backup // We declare this to be a partial backup since the synthetic filelist is only created // when a backup is interrupted. fsw.CreateFilesetFile(false); - var newFilesetID = await database.CreateFilesetAsync(fsw.VolumeID, fileTime); - await database.LinkFilesetToVolumeAsync(newFilesetID, fsw.VolumeID); - await database.AppendFilesFromPreviousSetAsync(null, newFilesetID, prevId, fileTime); + var newFilesetID = await database.CreateFilesetAsync(fsw.VolumeID, fileTime, taskreader.ProgressToken).ConfigureAwait(false); + await database.LinkFilesetToVolumeAsync(newFilesetID, fsw.VolumeID, taskreader.ProgressToken).ConfigureAwait(false); + await database.AppendFilesFromPreviousSetAsync(null, newFilesetID, prevId, fileTime, taskreader.ProgressToken).ConfigureAwait(false); - await database.WriteFilesetAsync(fsw, newFilesetID); + await database.WriteFilesetAsync(fsw, newFilesetID, taskreader.ProgressToken).ConfigureAwait(false); fsw.Close(); if (!await taskreader.ProgressRendevouz().ConfigureAwait(false)) return; - await database.UpdateRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeState.Uploading, -1, null); + await database.UpdateRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeState.Uploading, -1, null, false, default, null, taskreader.ProgressToken).ConfigureAwait(false); // If the previous filelist was not uploaded, we register it for deletion, as we have created a new synthetic one // Because it is registered as "Deleting", it will be removed from remote storage by the cleanup process if it exists if (!string.IsNullOrWhiteSpace(lastTempFilelist.Name) && (lastTempFilelist.State == RemoteVolumeState.Uploading || lastTempFilelist.State == RemoteVolumeState.Temporary)) - await database.UpdateRemoteVolumeAsync(lastTempFilelist.Name, RemoteVolumeState.Deleting, -1, null); - await database.CommitTransactionAsync("CommitUpdateFilelistVolume"); - await backendManager.PutAsync(fsw, null, null, true, () => database.FlushBackendMessagesAndCommitAsync(backendManager), taskreader.ProgressToken); + await database.UpdateRemoteVolumeAsync(lastTempFilelist.Name, RemoteVolumeState.Deleting, -1, null, false, default, null, taskreader.ProgressToken).ConfigureAwait(false); + await database.CommitTransactionAsync("CommitUpdateFilelistVolume", true, taskreader.ProgressToken).ConfigureAwait(false); + await backendManager.PutAsync(fsw, null, null, true, () => database.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken); } catch { - await database.RollbackTransactionAsync(); + await database.RollbackTransactionAsync(taskreader.ProgressToken).ConfigureAwait(false); fsw?.Dispose(); throw; } diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 66596f68a..938a8dc30 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -208,7 +208,13 @@ namespace Duplicati.Library.Main.Operation throw new UserInformationException("Multiple USN journal services are not supported", "MultipleUSNJournalServices"); var fileProvider = providers.First().Provider as LocalFileSource; - var journalData = m_database.GetChangeJournalData(lastfilesetid); + // TODO because of the new SQLite backend, this method returns an + // async enumerable. The proper way to handle this would be to + // change the UsnJournalService to use factory construction (see how + // all of the local database classes handle this), as the constructor + // is not async and shouldn't be executing "heavy" code, such as + // database queries. + var journalData = m_database.GetChangeJournalData(lastfilesetid, m_taskReader.ProgressToken).ToEnumerable(); var service = new UsnJournalService(fileProvider.SnapshotService, filter, m_options.FileAttributeFilter, m_options.SkipFilesLargerThan, journalData, cancellationTokenSource.Token); @@ -294,12 +300,14 @@ namespace Duplicati.Library.Main.Operation { try { - database = new LocalBackupDatabase(options.Dbpath, options); - var lastFilestTime = database.FilesetTimes + database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options, result.TaskControl.ProgressToken) + .ConfigureAwait(false); + var lastFilestTime = await database.FilesetTimes(result.TaskControl.ProgressToken) .Select(x => x.Value) .Append(DateTime.UnixEpoch) .Select(Library.Utility.Utility.NormalizeDateTimeToEpochSeconds) - .Max(); + .MaxAsync() + .ConfigureAwait(false); var currentStamp = Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(database.OperationTimestamp); if (lastFilestTime > currentStamp) @@ -311,43 +319,53 @@ namespace Duplicati.Library.Main.Operation Log.WriteVerboseMessage(LOGTAG, "DatabaseTimestampError", "The database has a timestamp of {0}, but the last fileset has a timestamp of {1}. Sleeping for {2} seconds to allow the clock to catch up.", database.OperationTimestamp.ToLocalTime(), Library.Utility.Utility.EPOCH.AddSeconds(lastFilestTime).ToLocalTime(), sleepSeconds); Thread.Sleep(TimeSpan.FromSeconds(sleepSeconds)); database.Dispose(); - database = new LocalBackupDatabase(options.Dbpath, options); + database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options, result.TaskControl.ProgressToken) + .ConfigureAwait(false); } result.Dryrun = options.Dryrun; // Check the database integrity - Utility.UpdateOptionsFromDb(database, options); - Utility.VerifyOptionsAndUpdateDatabase(database, options); + await Utility.UpdateOptionsFromDb(database, options, result.TaskControl.ProgressToken) + .ConfigureAwait(false); + await Utility.VerifyOptionsAndUpdateDatabase(database, options, result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + var probe_path = await database + .GetFirstPath(result.TaskControl.ProgressToken) + .ConfigureAwait(false); - var probe_path = database.GetFirstPath(); if (probe_path != null && Util.GuessDirSeparator(probe_path) != Util.DirectorySeparatorString) throw new UserInformationException(string.Format("The backup contains files that belong to another operating system. Proceeding with a backup would cause the database to contain paths from two different operation systems, which is not supported. To proceed without losing remote data, delete all filesets and make sure the --{0} option is set, then run the backup again to re-use the existing data on the remote store.", "no-auto-compact"), "CrossOsDatabaseReuseNotSupported"); - if (database.PartiallyRecreated) + if (await database.PartiallyRecreated(result.TaskControl.ProgressToken).ConfigureAwait(false)) throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated"); - if (database.RepairInProgress) + if (await database.RepairInProgress(result.TaskControl.ProgressToken).ConfigureAwait(false)) throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the backup process cannot continue. You may delete the local database and attempt to repair it again.", "DatabaseRepairInProgress"); // Make sure the database is sane - database.VerifyConsistency(options.Blocksize, options.BlockhashSize, !options.DisableFilelistConsistencyChecks, null); + await database + .VerifyConsistency(options.Blocksize, options.BlockhashSize, !options.DisableFilelistConsistencyChecks, result.TaskControl.ProgressToken) + .ConfigureAwait(false); // Guard the last filelist if (!options.DisableSyntheticFilelist) - lastTempFilelist = database.GetLastIncompleteFilesetVolume(null); + lastTempFilelist = await database + .GetLastIncompleteFilesetVolume(result.TaskControl.ProgressToken) + .ConfigureAwait(false); try { if (options.NoBackendverification) { - await FilelistProcessor.VerifyLocalList(backendManager, database, null, result.TaskControl.ProgressToken).ConfigureAwait(false); + await FilelistProcessor.VerifyLocalList(backendManager, database, result.TaskControl.ProgressToken).ConfigureAwait(false); await UpdateStorageStatsFromDatabase(result, database, options, backendManager, result.TaskControl.ProgressToken).ConfigureAwait(false); } else { - await FilelistProcessor.VerifyRemoteList(backendManager, options, database, null, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: false, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean).ConfigureAwait(false); - var updatedLastTemp = string.IsNullOrWhiteSpace(lastTempFilelist.Name) ? default : database.GetRemoteVolume(lastTempFilelist.Name, null); + await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: false, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean, result.TaskControl.ProgressToken).ConfigureAwait(false); + var updatedLastTemp = string.IsNullOrWhiteSpace(lastTempFilelist.Name) ? default : await database.GetRemoteVolume(lastTempFilelist.Name, result.TaskControl.ProgressToken).ConfigureAwait(false); if (!string.IsNullOrWhiteSpace(updatedLastTemp.Name) && updatedLastTemp.State == RemoteVolumeState.Deleting) { // The last temporary filelist was emptied, and scheduled for deletion, so we need to delete it @@ -360,9 +378,11 @@ namespace Duplicati.Library.Main.Operation Log.WriteMessage(LogMessageType.Information, LOGTAG, "BackendDeleteFailed", ex, "Backend delete failed: {0}", ex.Message); } - await backendManager.WaitForEmptyAsync(database, null, result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(database, result.TaskControl.ProgressToken).ConfigureAwait(false); // In case the backend delete failed, manually set the state to deleted - database.UpdateRemoteVolume(updatedLastTemp.Name, RemoteVolumeState.Deleted, updatedLastTemp.Size, updatedLastTemp.Hash, false, TimeSpan.FromHours(2), null, null); + await database + .UpdateRemoteVolume(updatedLastTemp.Name, RemoteVolumeState.Deleted, updatedLastTemp.Size, updatedLastTemp.Hash, false, TimeSpan.FromHours(2), null, result.TaskControl.ProgressToken) + .ConfigureAwait(false); // The last temporary filelist is no more lastTempFilelist = default; } @@ -382,17 +402,18 @@ namespace Duplicati.Library.Main.Operation .ConfigureAwait(false); // Re-open the database and backend manager - database = new LocalBackupDatabase(options.Dbpath, options); + database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options, result.TaskControl.ProgressToken) + .ConfigureAwait(false); Log.WriteInformationMessage(LOGTAG, "BackendCleanupFinished", "Backend cleanup finished, retrying verification"); - await FilelistProcessor.VerifyRemoteList(backendManager, options, database, null, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, result.TaskControl.ProgressToken).ConfigureAwait(false); } else throw; } // Return the updated version of the fileset, in case it was actually complete - return new PreBackupVerifyResult(database, database.GetLastIncompleteFilesetVolume(null)); + return new PreBackupVerifyResult(database, await database.GetLastIncompleteFilesetVolume(result.TaskControl.ProgressToken).ConfigureAwait(false)); } catch { @@ -424,7 +445,7 @@ namespace Duplicati.Library.Main.Operation options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames, GetBlacklistedPaths(options), options.ChangedFilelist, taskreader, () => result.PartialBackup = true, CancellationToken.None), - Backup.FilePreFilterProcess.Run(channels, options, stats, database), + Backup.FilePreFilterProcess.Run(channels, options, stats, database, taskreader), Backup.MetadataPreProcess.Run(channels, options, database, lastfilesetid, taskreader), Backup.SpillCollectorProcess.Run(channels, options, database, backendManager, taskreader), Backup.ProgressHandler.Run(channels, result) @@ -451,7 +472,9 @@ namespace Duplicati.Library.Main.Operation if (options.ChangedFilelist != null && options.ChangedFilelist.Length >= 1) { - await database.AppendFilesFromPreviousSetAsync(options.DeletedFilelist); + await database + .AppendFilesFromPreviousSetAsync(options.DeletedFilelist, taskreader.ProgressToken) + .ConfigureAwait(false); } else if (journalService != null) { @@ -472,7 +495,8 @@ namespace Duplicati.Library.Main.Operation stats.AddExaminedFile(fileSize); return false; - }); + }, taskreader.ProgressToken) + .ConfigureAwait(false); // store journal data in database, unless job is being canceled if (!result.PartialBackup) @@ -481,10 +505,14 @@ namespace Duplicati.Library.Main.Operation if (data.Any()) { // always record change journal data for current fileset (entry may be dropped later if nothing is uploaded) - await database.CreateChangeJournalDataAsync(data); + await database + .CreateChangeJournalDataAsync(data, taskreader.ProgressToken) + .ConfigureAwait(false); // update the previous fileset's change journal entry to resume at this point in case nothing was backed up - await database.UpdateChangeJournalDataAsync(data, lastfilesetid); + await database + .UpdateChangeJournalDataAsync(data, lastfilesetid, taskreader.ProgressToken) + .ConfigureAwait(false); } } } @@ -495,14 +523,16 @@ namespace Duplicati.Library.Main.Operation } else { - await database.UpdateFilesetAndMarkAsFullBackupAsync(filesetid); + await database + .UpdateFilesetAndMarkAsFullBackupAsync(filesetid, taskreader.ProgressToken) + .ConfigureAwait(false); } result.OperationProgressUpdater.UpdatefileCount(result.ExaminedFiles, result.SizeOfExaminedFiles, true); } } - private async Task CompactIfRequired(IBackendManager backendManager, ReusableTransaction transaction, long lastVolumeSize) + private async Task CompactIfRequired(IBackendManager backendManager, long lastVolumeSize) { var currentIsSmall = lastVolumeSize != -1 && lastVolumeSize <= m_options.SmallFileSize; @@ -510,36 +540,41 @@ namespace Duplicati.Library.Main.Operation { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Delete); m_result.DeleteResults = new DeleteResults(m_result); - using (var db = new LocalDeleteDatabase(m_database)) - await new DeleteHandler(m_options, (DeleteResults)m_result.DeleteResults) - .DoRunAsync(db, transaction, true, currentIsSmall, backendManager) - .ConfigureAwait(false); + await using var db = await LocalDeleteDatabase.CreateAsync(m_database, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await new DeleteHandler(m_options, (DeleteResults)m_result.DeleteResults) + .DoRunAsync(db, true, currentIsSmall, backendManager) + .ConfigureAwait(false); } else if (currentIsSmall && !m_options.NoAutoCompact) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Compact); m_result.CompactResults = new CompactResults(m_result); - using (var db = new LocalDeleteDatabase(m_database)) - await new CompactHandler(m_options, (CompactResults)m_result.CompactResults) - .DoCompactAsync(db, true, transaction, backendManager) - .ConfigureAwait(false); + await using var db = await LocalDeleteDatabase.CreateAsync(m_database, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await new CompactHandler(m_options, (CompactResults)m_result.CompactResults) + .DoCompactAsync(db, true, backendManager) + .ConfigureAwait(false); } } - private async Task PostBackupVerification(string currentFilelistVolume, string previousTemporaryFilelist, IBackendManager backendManager, ReusableTransaction rtr) + private async Task PostBackupVerification(string currentFilelistVolume, string previousTemporaryFilelist, IBackendManager backendManager) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupVerify); using (new Logging.Timer(LOGTAG, "AfterBackupVerify", "AfterBackupVerify")) - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, rtr.Transaction, m_result.BackendWriter, [currentFilelistVolume], [previousTemporaryFilelist], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); - await backendManager.WaitForEmptyAsync(m_database, rtr.Transaction, m_taskReader.ProgressToken); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, m_result.BackendWriter, [currentFilelistVolume], [previousTemporaryFilelist], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + + await backendManager + .WaitForEmptyAsync(m_database, m_taskReader.ProgressToken) + .ConfigureAwait(false); // Calculate the number of samples to test, using the largest number of file of a given type - long remoteVolumeCount = m_database.GetRemoteVolumes() + long remoteVolumeCount = await m_database + .GetRemoteVolumes(m_result.TaskControl.ProgressToken) .Where(x => x.State == RemoteVolumeState.Verified) .GroupBy(x => x.Type) - .Select(x => x.LongCount()) - .Max(); + .SelectAwait(async x => await x.LongCountAsync()) + .MaxAsync() + .ConfigureAwait(false); long samplesToTest = Math.Max(m_options.BackupTestSampleCount, (long)Math.Round(remoteVolumeCount * (m_options.BackupTestPercentage / 100m), MidpointRounding.AwayFromZero)); if (samplesToTest > 0 && remoteVolumeCount > 0) @@ -547,10 +582,10 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupTest); m_result.TestResults = new TestResults(m_result); - using (var testdb = new LocalTestDatabase(m_database)) - await new TestHandler(m_options, (TestResults)m_result.TestResults) - .DoRunAsync(samplesToTest, testdb, rtr, backendManager) - .ConfigureAwait(false); + await using var testdb = await LocalTestDatabase.CreateAsync(m_database, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await new TestHandler(m_options, (TestResults)m_result.TestResults) + .DoRunAsync(samplesToTest, testdb, backendManager) + .ConfigureAwait(false); } } @@ -561,15 +596,37 @@ namespace Duplicati.Library.Main.Operation { if (result.BackendWriter != null) { - result.BackendWriter.KnownFileCount = database.GetRemoteVolumes().Count(); - result.BackendWriter.KnownFilesets = database.GetRemoteVolumes().Count(x => x.Type == RemoteVolumeType.Files); - result.BackendWriter.KnownFileSize = database.GetRemoteVolumes().Select(x => Math.Max(0, x.Size)).Sum(); + result.BackendWriter.KnownFileCount = await database + .GetRemoteVolumes(cancelToken) + .CountAsync(cancelToken) + .ConfigureAwait(false); + + result.BackendWriter.KnownFilesets = await database + .GetRemoteVolumes(cancelToken) + .CountAsync(x => x.Type == RemoteVolumeType.Files) + .ConfigureAwait(false); + + result.BackendWriter.KnownFileSize = await database + .GetRemoteVolumes(cancelToken) + .Select(x => Math.Max(0, x.Size)) + .SumAsync(cancelToken) + .ConfigureAwait(false); result.BackendWriter.UnknownFileCount = 0; result.BackendWriter.UnknownFileSize = 0; - result.BackendWriter.BackupListCount = database.FilesetTimes.Count(); - result.BackendWriter.LastBackupDate = database.FilesetTimes.FirstOrDefault().Value.ToLocalTime(); + result.BackendWriter.BackupListCount = await database + .FilesetTimes(cancelToken) + .CountAsync(cancelToken) + .ConfigureAwait(false); + + result.BackendWriter.LastBackupDate = ( + await database + .FilesetTimes(cancelToken) + .FirstOrDefaultAsync(cancelToken) + .ConfigureAwait(false) + ) + .Value.ToLocalTime(); if (!options.QuotaDisable) { @@ -618,9 +675,9 @@ namespace Duplicati.Library.Main.Operation try { - await database.FlushBackendMessagesAndCommitAsync(backendManager).ConfigureAwait(false); + await database.FlushBackendMessagesAndCommitAsync(backendManager, result.TaskControl.ProgressToken).ConfigureAwait(false); await backendManager.WaitForEmptyAsync(result.TaskControl.ProgressToken).ConfigureAwait(false); - await database.FlushBackendMessagesAndCommitAsync(backendManager).ConfigureAwait(false); + await database.FlushBackendMessagesAndCommitAsync(backendManager, result.TaskControl.ProgressToken).ConfigureAwait(false); } catch (RetiredException) { @@ -632,7 +689,9 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Begin); // Do a remote verification, unless disabled - var (database, lastTempFilelist) = await PreBackupVerify(m_options, m_result, backendManager); + var (database, lastTempFilelist) = await PreBackupVerify(m_options, m_result, backendManager) + .ConfigureAwait(false); + var lastTempVolumeIncomplete = false; if (!string.IsNullOrWhiteSpace(lastTempFilelist.Name)) @@ -651,9 +710,11 @@ namespace Duplicati.Library.Main.Operation { // If we crash now, there is a chance we leave behind partial remote files if (!m_options.Dryrun) - database.TerminatedWithActiveUploads = true; + await database + .TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, true) + .ConfigureAwait(false); - using (m_database = database) + await using (m_database = database) using (var db = new Backup.BackupDatabase(m_database, m_options)) // Setup runners and instances here using (var filesetvolume = new FilesetVolumeWriter(m_options, m_database.OperationTimestamp)) @@ -665,37 +726,51 @@ namespace Duplicati.Library.Main.Operation try { // If the previous backup was interrupted, send a synthetic list - await Backup.UploadSyntheticFilelist.Run(db, m_options, m_result, m_result.TaskControl, backendManager, lastTempFilelist); + await Backup.UploadSyntheticFilelist.Run(db, m_options, m_result, m_result.TaskControl, backendManager, lastTempFilelist) + .ConfigureAwait(false); // Grab the previous backup ID, if any - var prevfileset = m_database.FilesetTimes.FirstOrDefault(); + var prevfileset = await m_database + .FilesetTimes(m_result.TaskControl.ProgressToken) + .FirstOrDefaultAsync() + .ConfigureAwait(false); + if (prevfileset.Value.ToUniversalTime() > m_database.OperationTimestamp.ToUniversalTime()) throw new Exception(string.Format("The previous backup has time {0}, but this backup has time {1}. Something is wrong with the clock.", prevfileset.Value.ToLocalTime(), m_database.OperationTimestamp.ToLocalTime())); var lastfilesetid = prevfileset.Value.Ticks == 0 ? -1 : prevfileset.Key; // Rebuild any index files that are missing - await Backup.RecreateMissingIndexFiles.Run(db, backendManager, m_options, m_result.TaskControl); + await Backup.RecreateMissingIndexFiles.Run(db, backendManager, m_options, m_result.TaskControl) + .ConfigureAwait(false); // Prepare the operation by registering the filelist m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_ProcessingFiles); var repcnt = 0; - while (repcnt < 100 && await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename) >= 0) + while (repcnt < 100 && await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename, m_taskReader.ProgressToken).ConfigureAwait(false) >= 0) filesetvolume.ResetRemoteFilename(m_options, m_database.OperationTimestamp.AddSeconds(repcnt++)); - if (await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename) >= 0) + if (await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename, m_taskReader.ProgressToken).ConfigureAwait(false) >= 0) throw new Exception("Unable to generate a unique fileset name"); - var filesetvolumeid = await db.RegisterRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary); - filesetid = await db.CreateFilesetAsync(filesetvolumeid, VolumeBase.ParseFilename(filesetvolume.RemoteFilename).Time); + var filesetvolumeid = await db + .RegisterRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, m_taskReader.ProgressToken) + .ConfigureAwait(false); + + filesetid = await db + .CreateFilesetAsync(filesetvolumeid, VolumeBase.ParseFilename(filesetvolume.RemoteFilename).Time, m_taskReader.ProgressToken) + .ConfigureAwait(false); var journalService = GetJournalService(source, filter, lastfilesetid); // Start parallel scan, or use the database if (m_options.DisableFileScanner) { - var d = m_database.GetLastBackupFileCountAndSize(); + var d = await m_database + .GetLastBackupFileCountAndSize(m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + m_result.OperationProgressUpdater.UpdatefileCount(d.Item1, d.Item2, true); } else @@ -722,49 +797,68 @@ namespace Duplicati.Library.Main.Operation // Ensure the database is in a sane state after adding data using (new Logging.Timer(LOGTAG, "VerifyConsistency", "VerifyConsistency")) - await db.VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, false); + await db + .VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, false, m_taskReader.ProgressToken) + .ConfigureAwait(false); - await FlushBackend(db, m_result, backendManager).ConfigureAwait(false); + await FlushBackend(db, m_result, backendManager) + .ConfigureAwait(false); // Send the actual filelist - await Backup.UploadRealFilelist.Run(m_result, db, backendManager, m_options, filesetvolume, filesetid, m_result.TaskControl, lastTempVolumeIncomplete); + await Backup.UploadRealFilelist.Run(m_result, db, backendManager, m_options, filesetvolume, filesetid, m_result.TaskControl, lastTempVolumeIncomplete) + .ConfigureAwait(false); // Wait for upload completion m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload); - await FlushBackend(db, m_result, backendManager).ConfigureAwait(false); + await FlushBackend(db, m_result, backendManager) + .ConfigureAwait(false); if (!m_options.Dryrun) - database.TerminatedWithActiveUploads = false; - // Make sure we have the database up-to-date - await db.CommitTransactionAsync("CommitAfterUpload", false); + await database + .TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, false) + .ConfigureAwait(false); - using (var rtr = new ReusableTransaction(m_database)) + // Make sure we have the database up-to-date + await db + .CommitTransactionAsync("CommitAfterUpload", true, m_taskReader.ProgressToken) + .ConfigureAwait(false); + + // If this throws, we should roll back the transaction + if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - // If this throws, we should roll back the transaction + var lastVolumeSize = await m_database + .GetLastWrittenDBlockVolumeSize(m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + await CompactIfRequired(backendManager, lastVolumeSize) + .ConfigureAwait(false); + } + + if (m_options.UploadVerificationFile && await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + { + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload); + await FilelistProcessor.UploadVerificationFile(backendManager, m_options, m_database, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + } + + if (!m_options.Dryrun) + { + await database.Transaction + .CommitAsync("CommitFinalizingBackup") + .ConfigureAwait(false); + if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - var lastVolumeSize = m_database.GetLastWrittenDBlockVolumeSize(rtr.Transaction); - await CompactIfRequired(backendManager, rtr, lastVolumeSize); - } - - if (m_options.UploadVerificationFile && await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - { - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload); - await FilelistProcessor.UploadVerificationFile(backendManager, m_options, m_database, rtr.Transaction); - } - - if (!m_options.Dryrun) - { - rtr.Commit("CommitFinalizingBackup"); - if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - { - if (m_options.NoBackendverification) - await UpdateStorageStatsFromDatabase(m_result, m_database, m_options, backendManager, m_taskReader.ProgressToken).ConfigureAwait(false); - else - await PostBackupVerification(filesetvolume.RemoteFilename, lastTempFilelist.Name, backendManager, rtr).ConfigureAwait(false); - } - rtr.Commit("CommitAfterVerification", restart: false); + if (m_options.NoBackendverification) + await UpdateStorageStatsFromDatabase(m_result, m_database, m_options, backendManager, m_taskReader.ProgressToken) + .ConfigureAwait(false); + else + await PostBackupVerification(filesetvolume.RemoteFilename, lastTempFilelist.Name, backendManager) + .ConfigureAwait(false); } + await database.Transaction + .CommitAsync("CommitAfterVerification") + .ConfigureAwait(false); } m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Complete); diff --git a/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs b/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs index 6dacd07cf..312a79260 100644 --- a/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs +++ b/Duplicati/Library/Main/Operation/Common/DatabaseCommon.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -40,7 +40,6 @@ namespace Duplicati.Library.Main.Operation.Common private static readonly string LOGTAG = Logging.Log.LogTagFromType(); protected readonly LocalDatabase m_db; - private System.Data.IDbTransaction m_transaction; protected readonly Options m_options; public DatabaseCommon(LocalDatabase db, Options options) @@ -48,112 +47,131 @@ namespace Duplicati.Library.Main.Operation.Common { m_db = db; m_options = options; - m_transaction = db.BeginTransaction(); } - protected System.Data.IDbTransaction GetTransaction() + public Task RegisterRemoteVolumeAsync(string name, RemoteVolumeType type, RemoteVolumeState state, CancellationToken cancellationToken) { - m_workerSource.Token.ThrowIfCancellationRequested(); - - if (m_transaction == null) - m_transaction = m_db.BeginTransaction(); - return m_transaction; + return RunOnMain(async () => + await m_db + .RegisterRemoteVolume(name, type, state, cancellationToken) + .ConfigureAwait(false) + ); } - public Task RegisterRemoteVolumeAsync(string name, RemoteVolumeType type, RemoteVolumeState state) + public Task UpdateRemoteVolumeAsync(string name, RemoteVolumeState state, long size, string hash, bool suppressCleanup, TimeSpan deleteGraceTime, bool? setArchived, CancellationToken cancellationToken) { - return RunOnMain(() => m_db.RegisterRemoteVolume(name, type, state, GetTransaction())); + return RunOnMain(async () => + await m_db + .UpdateRemoteVolume(name, state, size, hash, suppressCleanup, deleteGraceTime, setArchived, cancellationToken) + .ConfigureAwait(false) + ); } - public Task UpdateRemoteVolumeAsync(string name, RemoteVolumeState state, long size, string hash, bool suppressCleanup = false, TimeSpan deleteGraceTime = default(TimeSpan), bool? setArchived = null) + public async Task FlushBackendMessagesAndCommitAsync(IBackendManager backendManager, CancellationToken cancellationToken) { - return RunOnMain(() => m_db.UpdateRemoteVolume(name, state, size, hash, suppressCleanup, deleteGraceTime, setArchived, GetTransaction())); + await FlushPendingBackendMessagesAsync(backendManager, cancellationToken).ConfigureAwait(false); + await CommitTransactionAsync("FlushBackendMessagesAndCommitAsync", true, cancellationToken).ConfigureAwait(false); } - public async Task FlushBackendMessagesAndCommitAsync(IBackendManager backendManager) - { - await FlushPendingBackendMessagesAsync(backendManager).ConfigureAwait(false); - await CommitTransactionAsync("FlushBackendMessagesAndCommitAsync").ConfigureAwait(false); - } + private Task FlushPendingBackendMessagesAsync(IBackendManager backendManager, CancellationToken cancellationToken) + => RunOnMain(async () => await backendManager.FlushPendingMessagesAsync(m_db, cancellationToken).ConfigureAwait(false)); - private Task FlushPendingBackendMessagesAsync(IBackendManager backendManager) - => RunOnMain(() => backendManager.FlushPendingMessagesAsync(m_db, GetTransaction(), CancellationToken.None).ConfigureAwait(false)); - - public Task CommitTransactionAsync(string message, bool restart = true) + public Task CommitTransactionAsync(string message, bool restart, CancellationToken cancellationToken) { - return RunOnMain(() => + return RunOnMain(async () => { if (m_options.Dryrun) { - if (!restart) - { - if (m_transaction != null) - m_transaction.Rollback(); - m_transaction = null; - } - return; + await m_db.Transaction + .RollBackAsync(cancellationToken) + .ConfigureAwait(false); } - - if (m_transaction != null) + else { using (new Logging.Timer(LOGTAG, "CommitTransactionAsync", message)) - m_transaction.Commit(); - m_transaction = null; + await m_db.Transaction + .CommitAsync(message, restart, cancellationToken) + .ConfigureAwait(false); } }); } - public Task RollbackTransactionAsync() + public Task RollbackTransactionAsync(CancellationToken cancellationToken) { - return RunOnMain(() => - { - if (m_transaction != null) - { - m_transaction.Rollback(); - m_transaction = null; - } - }); + return RunOnMain(async () => + await m_db.Transaction + .RollBackAsync(cancellationToken) + .ConfigureAwait(false) + ); } - - - public Task RenameRemoteFileAsync(string oldname, string newname) + public Task RenameRemoteFileAsync(string oldname, string newname, CancellationToken cancellationToken) { - return RunOnMain(() => m_db.RenameRemoteFile(oldname, newname, GetTransaction())); + return RunOnMain(async () => + await m_db + .RenameRemoteFile(oldname, newname, cancellationToken) + .ConfigureAwait(false) + ); } - public Task LogRemoteOperationAsync(string operation, string path, string data) + public Task LogRemoteOperationAsync(string operation, string path, string data, CancellationToken cancellationToken) { - return RunOnMain(() => m_db.LogRemoteOperation(operation, path, data, GetTransaction())); + return RunOnMain(async () => + await m_db + .LogRemoteOperation(operation, path, data, cancellationToken) + .ConfigureAwait(false) + ); } - public Task GetBlocksAsync(long volumeid) + public Task GetBlocksAsync(long volumeid, CancellationToken cancellationToken) { // TODO: Figure out how to return the enumerable, while keeping the lock // and not creating the entire result in memory - return RunOnMain(() => m_db.GetBlocks(volumeid, GetTransaction()).ToArray()); + return RunOnMain(async () => + await m_db + .GetBlocks(volumeid, cancellationToken) + .ToArrayAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false) + ); } - public Task GetVolumeInfoAsync(string remotename) + public Task GetVolumeInfoAsync(string remotename, CancellationToken cancellationToken) { - return RunOnMain(() => m_db.GetRemoteVolume(remotename, GetTransaction())); + return RunOnMain(async () => + await m_db + .GetRemoteVolume(remotename, cancellationToken) + .ConfigureAwait(false) + ); } - public Task<(string Hash, byte[] Buffer, int Size)[]> GetBlocklistsAsync(long volumeid, int blocksize, int hashsize) + public Task<(string Hash, byte[] Buffer, int Size)[]> GetBlocklistsAsync(long volumeid, int blocksize, int hashsize, CancellationToken cancellationToken) { // TODO: Figure out how to return the enumerable, while keeping the lock // and not creating the entire result in memory - return RunOnMain(() => m_db.GetBlocklists(volumeid, blocksize, hashsize, GetTransaction()).ToArray()); + return RunOnMain(async () => + await m_db + .GetBlocklists(volumeid, blocksize, hashsize, cancellationToken) + .ToArrayAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false) + ); } - public Task GetRemoteVolumeIDAsync(string remotename) + public Task GetRemoteVolumeIDAsync(string remotename, CancellationToken cancellationToken) { - return RunOnMain(() => m_db.GetRemoteVolumeID(remotename, GetTransaction())); + return RunOnMain(async () => + await m_db + .GetRemoteVolumeID(remotename, cancellationToken) + .ConfigureAwait(false) + ); } - public Task AddIndexBlockLinkAsync(long indexVolumeID, long blockVolumeID) + public Task AddIndexBlockLinkAsync(long indexVolumeID, long blockVolumeID, CancellationToken cancellationToken) { - return RunOnMain(() => m_db.AddIndexBlockLink(indexVolumeID, blockVolumeID, GetTransaction())); + return RunOnMain(async () => + await m_db + .AddIndexBlockLink(indexVolumeID, blockVolumeID, cancellationToken) + .ConfigureAwait(false) + ); } @@ -162,14 +180,7 @@ namespace Duplicati.Library.Main.Operation.Common if (m_workerSource.IsCancellationRequested) return; - RunOnMain(() => - { - base.Dispose(isDisposing); - - var tr = Interlocked.Exchange(ref m_transaction, null); - tr?.Commit(); - tr?.Dispose(); - }).Await(); + base.Dispose(isDisposing); } } diff --git a/Duplicati/Library/Main/Operation/Common/IndexVolumeCreator.cs b/Duplicati/Library/Main/Operation/Common/IndexVolumeCreator.cs index caa6349e1..1360b0b58 100644 --- a/Duplicati/Library/Main/Operation/Common/IndexVolumeCreator.cs +++ b/Duplicati/Library/Main/Operation/Common/IndexVolumeCreator.cs @@ -1,26 +1,27 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; +using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Main.Volumes; using Duplicati.Library.Utility; @@ -67,14 +68,15 @@ namespace Duplicati.Library.Main.Operation.Common /// /// Creates an index volume with the temporary contents /// - /// The index volume. /// The name of the block file. /// The options used in this run. /// The database to use. - public async Task CreateVolume(string blockfilename, Options options, DatabaseCommon database) + /// The cancellation token to use. + /// The index volume. + public async Task CreateVolume(string blockfilename, Options options, DatabaseCommon database, CancellationToken cancellationToken) { var w = new IndexVolumeWriter(options); - w.VolumeID = await database.RegisterRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary); + w.VolumeID = await database.RegisterRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, cancellationToken).ConfigureAwait(false); var enumerator = blockListHashes.GetEnumerator(); while (enumerator.MoveNext()) @@ -158,23 +160,23 @@ namespace Duplicati.Library.Main.Operation.Common /// internal static class IndexVolumeCreator { - public static async Task CreateIndexVolume(string blockname, Options options, Common.DatabaseCommon database) + public static async Task CreateIndexVolume(string blockname, Options options, Common.DatabaseCommon database, CancellationToken cancellationToken) { using (var h = HashFactory.CreateHasher(options.BlockHashAlgorithm)) { var w = new IndexVolumeWriter(options); - w.VolumeID = await database.RegisterRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary); + w.VolumeID = await database.RegisterRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, cancellationToken).ConfigureAwait(false); - var blockvolume = await database.GetVolumeInfoAsync(blockname); + var blockvolume = await database.GetVolumeInfoAsync(blockname, cancellationToken).ConfigureAwait(false); w.StartVolume(blockname); - foreach (var b in await database.GetBlocksAsync(blockvolume.ID)) + foreach (var b in await database.GetBlocksAsync(blockvolume.ID, cancellationToken).ConfigureAwait(false)) w.AddBlock(b.Hash, b.Size); w.FinishVolume(blockvolume.Hash, blockvolume.Size); if (options.IndexfilePolicy == Options.IndexFileStrategy.Full) - foreach (var b in await database.GetBlocklistsAsync(blockvolume.ID, options.Blocksize, options.BlockhashSize)) + foreach (var b in await database.GetBlocklistsAsync(blockvolume.ID, options.Blocksize, options.BlockhashSize, cancellationToken).ConfigureAwait(false)) { var bh = Convert.ToBase64String(h.ComputeHash(b.Item2, 0, b.Item3)); if (bh != b.Item1) @@ -185,7 +187,7 @@ namespace Duplicati.Library.Main.Operation.Common w.Close(); // Register that the index file is tracking the block file - await database.AddIndexBlockLinkAsync(w.VolumeID, blockvolume.ID); + await database.AddIndexBlockLinkAsync(w.VolumeID, blockvolume.ID, cancellationToken).ConfigureAwait(false); return w; } diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index e918c1491..e3b8112f2 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Main.Database; @@ -52,23 +52,28 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new Exception(string.Format("Database file does not exist: {0}", m_options.Dbpath)); - using (var db = new LocalDeleteDatabase(m_options.Dbpath, "Compact", m_options.SqlitePageCache)) - using (var tr = new ReusableTransaction(db)) + await using (var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Compact", null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { - Utility.UpdateOptionsFromDb(db, m_options); - Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); - var changed = await DoCompactAsync(db, false, tr, backendManager).ConfigureAwait(false); + var changed = await DoCompactAsync(db, false, backendManager).ConfigureAwait(false); if (changed && m_options.UploadVerificationFile) - await FilelistProcessor.UploadVerificationFile(backendManager, m_options, db, null); + await FilelistProcessor.UploadVerificationFile(backendManager, m_options, db, m_result.TaskControl.ProgressToken); if (!m_options.Dryrun) { - tr.Commit("CommitCompact", restart: false); + await db.Transaction + .CommitAsync("CommitCompact") + .ConfigureAwait(false); + if (changed) { - db.WriteResults(m_result); + await db.WriteResults(m_result, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); if (m_options.AutoVacuum) { m_result.VacuumResults = new VacuumResults(m_result); @@ -79,31 +84,45 @@ namespace Duplicati.Library.Main.Operation } } - internal async Task DoCompactAsync(LocalDeleteDatabase db, bool hasVerifiedBackend, ReusableTransaction rtr, IBackendManager backendManager) + internal async Task DoCompactAsync(LocalDeleteDatabase db, bool hasVerifiedBackend, IBackendManager backendManager) { - var report = db.GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount, rtr.Transaction); + var report = await db + .GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + report.ReportCompactData(); if (report.ShouldReclaim || report.ShouldCompact) { // Workaround where we allow a running backendmanager to be used if (!hasVerifiedBackend) - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, FilelistProcessor.VerifyMode.VerifyStrict, rtr.Transaction).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, FilelistProcessor.VerifyMode.VerifyStrict, m_result.TaskControl.ProgressToken).ConfigureAwait(false); var newvol = new BlockVolumeWriter(m_options); - newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, rtr.Transaction); + newvol.VolumeID = await db + .RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); IndexVolumeWriter newvolindex = null; if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None) { newvolindex = new IndexVolumeWriter(m_options); - newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, rtr.Transaction); - db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, rtr.Transaction); + newvolindex.VolumeID = await db + .RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + await db + .AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); } var blocksInVolume = 0L; var buffer = new byte[m_options.Blocksize]; - var remoteList = db.GetRemoteVolumes().Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified).ToArray(); + var remoteList = await db + .GetRemoteVolumes(m_result.TaskControl.ProgressToken) + .Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified) + .ToArrayAsync(m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); //These are for bookkeeping var uploadedVolumes = new List>(); @@ -121,7 +140,7 @@ namespace Duplicati.Library.Main.Operation .Cast() .ToList(); } - await foreach (var d in DoDelete(db, backendManager, fullyDeleteable, rtr, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + await foreach (var d in DoDelete(db, backendManager, fullyDeleteable, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) deletedVolumes.Add(d); // This list is used to pick up unused volumes, @@ -133,7 +152,9 @@ namespace Duplicati.Library.Main.Operation { // If we crash now, we may leave partial files if (!m_options.Dryrun) - db.TerminatedWithActiveUploads = true; + await db + .TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, true) + .ConfigureAwait(false); newvolindex?.StartVolume(newvol.RemoteFilename); List volumesToDownload = []; @@ -147,7 +168,7 @@ namespace Duplicati.Library.Main.Operation .ToList(); } - using (var q = db.CreateBlockQueryHelper(rtr.Transaction)) + using (var q = await db.CreateBlockQueryHelper(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(volumesToDownload, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { @@ -156,59 +177,79 @@ namespace Duplicati.Library.Main.Operation var entry = new RemoteVolume(name, hash, size); if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); return false; } downloadedVolumes.Add(new KeyValuePair(entry.Name, entry.Size)); - var volumeid = db.GetRemoteVolumeID(entry.Name, rtr.Transaction); + var volumeid = await db + .GetRemoteVolumeID(entry.Name, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + var inst = VolumeBase.ParseFilename(entry.Name); using (var f = new BlockVolumeReader(inst.CompressionModule, tmpfile, m_options)) { foreach (var e in f.Blocks) { - if (q.UseBlock(e.Key, e.Value, volumeid, rtr.Transaction)) + if (await q.UseBlock(e.Key, e.Value, volumeid, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { //TODO: How do we get the compression hint? Reverse query for filename in db? var s = f.ReadBlock(e.Key, buffer); if (s != e.Value) throw new Exception(string.Format("Size mismatch problem for block {0}, {1} vs {2}", e.Key, s, e.Value)); - newvol.AddBlock(e.Key, buffer, 0, s, Interface.CompressionHint.Compressible); + await newvol + .AddBlock(e.Key, buffer, 0, s, Interface.CompressionHint.Compressible) + .ConfigureAwait(false); + if (newvolindex != null) newvolindex.AddBlock(e.Key, e.Value); - db.RegisterDuplicatedBlock(e.Key, e.Value, newvol.VolumeID, rtr.Transaction); + await db + .RegisterDuplicatedBlock(e.Key, e.Value, newvol.VolumeID, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + blocksInVolume++; if (newvol.Filesize > (m_options.VolumeSize - m_options.Blocksize)) { - await FinishVolumeAndUpload(db, backendManager, newvol, newvolindex, uploadedVolumes, rtr); + await FinishVolumeAndUpload(db, backendManager, newvol, newvolindex, uploadedVolumes) + .ConfigureAwait(false); newvol = new BlockVolumeWriter(m_options); - newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, rtr.Transaction); + newvol.VolumeID = await db + .RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None) { newvolindex = new IndexVolumeWriter(m_options); - newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, rtr.Transaction); - db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, rtr.Transaction); + newvolindex.VolumeID = await db + .RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + await db + .AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + newvolindex.StartVolume(newvol.RemoteFilename); } blocksInVolume = 0; // Wait for the backend to catch up - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); // Commit as we have uploaded a volume if (!m_options.Dryrun) - rtr.Commit("CommitCompact"); + await db.Transaction + .CommitAsync("CommitCompact", true, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); if (deleteableVolumes.Any()) { // Preserve space by deleting the old volume - await foreach (var d in DoDelete(db, backendManager, deleteableVolumes, rtr, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + await foreach (var d in DoDelete(db, backendManager, deleteableVolumes, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) deletedVolumes.Add(d); deleteableVolumes.Clear(); } @@ -223,14 +264,20 @@ namespace Duplicati.Library.Main.Operation if (blocksInVolume > 0) { - await FinishVolumeAndUpload(db, backendManager, newvol, newvolindex, uploadedVolumes, rtr).ConfigureAwait(false); + await FinishVolumeAndUpload(db, backendManager, newvol, newvolindex, uploadedVolumes).ConfigureAwait(false); } else { - db.RemoveRemoteVolume(newvol.RemoteFilename, rtr.Transaction); + await db + .RemoveRemoteVolume(newvol.RemoteFilename, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + if (newvolindex != null) { - db.RemoveRemoteVolume(newvolindex.RemoteFilename, rtr.Transaction); + await db + .RemoveRemoteVolume(newvolindex.RemoteFilename, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + newvolindex.FinishVolume(null, 0); } } @@ -238,7 +285,9 @@ namespace Duplicati.Library.Main.Operation // The remainder of the operation cannot leave partial files if (!m_options.Dryrun) - db.TerminatedWithActiveUploads = false; + await db + .TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, false) + .ConfigureAwait(false); } else { @@ -246,7 +295,7 @@ namespace Duplicati.Library.Main.Operation newvol.Dispose(); } - await foreach (var d in DoDelete(db, backendManager, deleteableVolumes, rtr, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + await foreach (var d in DoDelete(db, backendManager, deleteableVolumes, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) deletedVolumes.Add(d); var downloadSize = downloadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value); @@ -291,7 +340,7 @@ namespace Duplicati.Library.Main.Operation Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize - m_result.UploadedFileSize)); } - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; return (m_result.DeletedFileCount + m_result.UploadedFileCount) > 0; @@ -303,41 +352,53 @@ namespace Duplicati.Library.Main.Operation } } - private async IAsyncEnumerable> DoDelete(LocalDeleteDatabase db, IBackendManager backend, IEnumerable deleteableVolumes, ReusableTransaction rtr, [EnumeratorCancellation] CancellationToken cancellationToken) + private async IAsyncEnumerable> DoDelete(LocalDeleteDatabase db, IBackendManager backend, IEnumerable deleteableVolumes, [EnumeratorCancellation] CancellationToken cancellationToken) { // Find volumes that can be deleted - var remoteFilesToRemove = db.ReOrderDeleteableVolumes(deleteableVolumes, rtr.Transaction).ToList(); + var remoteFilesToRemove = await db + .ReOrderDeleteableVolumes(deleteableVolumes, cancellationToken) + .ToListAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); // Make sure we do not re-assign blocks to any of the volumes we are about to delete - var toRemoveVolumeIds = db.GetRemoteVolumeIDs(remoteFilesToRemove.Select(x => x.Name), rtr.Transaction) + var toRemoveVolumeIds = await db + .GetRemoteVolumeIDs(remoteFilesToRemove.Select(x => x.Name), cancellationToken) .Select(x => x.Value) .Distinct() - .ToList(); + .ToListAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); // Mark all volumes and relevant index files as disposable foreach (var f in remoteFilesToRemove) { - db.PrepareForDelete(f.Name, toRemoveVolumeIds, rtr.Transaction); - db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, rtr.Transaction); + await db + .PrepareForDelete(f.Name, toRemoveVolumeIds, cancellationToken) + .ConfigureAwait(false); + + await db + .UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, cancellationToken) + .ConfigureAwait(false); } // Before we commit the current state, make sure the backend has caught up - await backend.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backend.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - rtr.Commit("CommitDelete"); + await db.Transaction + .CommitAsync("CommitDelete", token: cancellationToken) + .ConfigureAwait(false); await foreach (var d in PerformDelete(backend, remoteFilesToRemove, cancellationToken).ConfigureAwait(false)) yield return d; } - private async Task FinishVolumeAndUpload(LocalDeleteDatabase db, IBackendManager backendManager, BlockVolumeWriter newvol, IndexVolumeWriter newvolindex, List> uploadedVolumes, ReusableTransaction rtr) + private async Task FinishVolumeAndUpload(LocalDeleteDatabase db, IBackendManager backendManager, BlockVolumeWriter newvol, IndexVolumeWriter newvolindex, List> uploadedVolumes) { - Action indexVolumeFinished = null; + Func indexVolumeFinished = null; if (newvolindex != null && m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) - indexVolumeFinished = () => + indexVolumeFinished = async () => { - foreach (var blocklist in db.GetBlocklists(newvol.VolumeID, m_options.Blocksize, m_options.BlockhashSize, rtr.Transaction)) + await foreach (var blocklist in db.GetBlocklists(newvol.VolumeID, m_options.Blocksize, m_options.BlockhashSize, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) newvolindex.WriteBlocklist(blocklist.Hash, blocklist.Buffer, 0, blocklist.Size); }; @@ -347,8 +408,10 @@ namespace Duplicati.Library.Main.Operation // We can handle at most one in-flight upload at a time, // because the transaction is not thread-safe, and shared with the upload - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - db.UpdateRemoteVolume(newvol.RemoteFilename, RemoteVolumeState.Uploading, -1, null, rtr.Transaction); + await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await db + .UpdateRemoteVolume(newvol.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); // TODO: The upload here does not flush the database messages, // and this can leave the database in a state where it does not know of the remote file @@ -356,7 +419,10 @@ namespace Duplicati.Library.Main.Operation // Once fixed, we can perhaps let he backend manager simply call the database directly if (!m_options.Dryrun) { - rtr.Commit("CommitUpload"); + await db.Transaction + .CommitAsync("CommitUpload") + .ConfigureAwait(false); + await backendManager.PutAsync(newvol, newvolindex, indexVolumeFinished, false, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } else diff --git a/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs b/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs index f46ec3701..164c3ad4c 100644 --- a/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs +++ b/Duplicati/Library/Main/Operation/CreateBugReportHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -68,11 +68,11 @@ namespace Duplicati.Library.Main.Operation using (var tmp = new TempFile()) { File.Copy(m_options.Dbpath, tmp, true); - using (var db = new LocalBugReportDatabase(tmp, m_options.SqlitePageCache)) + await using (var db = await LocalBugReportDatabase.CreateAsync(tmp, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { - db.Fix(); + await db.Fix(m_result.TaskControl.ProgressToken).ConfigureAwait(false); if (m_options.AutoVacuum) - db.Vacuum(); + await db.Vacuum(m_result.TaskControl.ProgressToken).ConfigureAwait(false); } // Apply zip64 option if not already set diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs index 6e291fc4e..f45562038 100644 --- a/Duplicati/Library/Main/Operation/DeleteHandler.cs +++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -51,25 +51,33 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseFileMissing"); - using (var db = new LocalDeleteDatabase(m_options.Dbpath, "Delete", m_options.SqlitePageCache)) - using (var tr = new ReusableTransaction(db)) - { - Utility.UpdateOptionsFromDb(db, m_options); - Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + await using var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Delete", null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); - await DoRunAsync(db, tr, false, false, backendManager).ConfigureAwait(false); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); - if (!m_options.Dryrun) - tr.Commit("ComitDelete", restart: false); - } + await DoRunAsync(db, false, false, backendManager).ConfigureAwait(false); + + if (!m_options.Dryrun) + await db.Transaction + .CommitAsync("ComitDelete") + .ConfigureAwait(false); } - public async Task DoRunAsync(LocalDeleteDatabase db, ReusableTransaction rtr, bool hasVerifiedBackend, bool forceCompact, IBackendManager backendManager) + public async Task DoRunAsync(LocalDeleteDatabase db, bool hasVerifiedBackend, bool forceCompact, IBackendManager backendManager) { if (!hasVerifiedBackend) - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, rtr.Transaction).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + + // We collapse the async enumerablo into a array to avoid multiple + // enumerations (and thus multiple database queries) + var filesets = await db + .FilesetsWithBackupVersion(m_result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); - var filesets = db.FilesetsWithBackupVersion.ToArray(); List versionsToDelete = [ .. new SpecificVersionsRemover(m_options).GetFilesetsToDelete(filesets), @@ -97,18 +105,31 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "DeleteRemoteFileset", "Deleting {0} remote fileset(s) ...", versionsToDelete.Count); - var lst = db.DropFilesetsFromTable(versionsToDelete.Select(x => x.Time).ToArray(), rtr.Transaction).ToArray(); + var lst = await db + .DropFilesetsFromTable( + versionsToDelete + .Select(x => x.Time) + .ToArray(), + m_result.TaskControl.ProgressToken + ) + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + foreach (var f in lst) - db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, rtr.Transaction); + await db + .UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); if (!m_options.Dryrun) - rtr.Commit("CommitBeforeDelete"); + await db.Transaction + .CommitAsync("CommitBeforeDelete", true, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); foreach (var f in lst) { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); return; } @@ -118,7 +139,7 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFileset", "Would delete remote fileset: {0}", f.Key); } - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); var count = lst.Length; if (!m_options.Dryrun) @@ -145,7 +166,7 @@ namespace Duplicati.Library.Main.Operation { m_result.CompactResults = new CompactResults(m_result); await new CompactHandler(m_options, (CompactResults)m_result.CompactResults) - .DoCompactAsync(db, true, rtr, backendManager).ConfigureAwait(false); + .DoCompactAsync(db, true, backendManager).ConfigureAwait(false); } m_result.SetResults(versionsToDelete.Select(v => new Tuple(v.Version, v.Time)), m_options.Dryrun); diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index 1f5239faf..5bf01708b 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -63,13 +63,16 @@ namespace Duplicati.Library.Main.Operation /// /// Helper method that verifies uploaded volumes and updates their state in the database. - /// Throws an error if there are issues with the remote storage + /// Throws an error if there are issues with the remote storage. /// - /// The database to compare with - public static async Task VerifyLocalList(IBackendManager backendManager, LocalDatabase database, IDbTransaction transaction, CancellationToken cancellationToken) + /// The backend manager instance to use. + /// The database to compare with. + /// The cancellation token to use. + /// A task that completes when the verification is done. + public static async Task VerifyLocalList(IBackendManager backendManager, LocalDatabase database, CancellationToken cancellationToken) { - var locallist = database.GetRemoteVolumes(transaction); - foreach (var i in locallist) + var locallist = database.GetRemoteVolumes(cancellationToken); + await foreach (var i in locallist.ConfigureAwait(false)) { switch (i.State) { @@ -98,48 +101,55 @@ namespace Duplicati.Library.Main.Operation break; } - await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false); } } /// /// Helper method that verifies uploaded volumes and updates their state in the database. - /// Throws an error if there are issues with the remote storage + /// Throws an error if there are issues with the remote storage. /// - /// The backend instance to use - /// The options used - /// The database to compare with - /// The backend writer to use - /// True if only the latest volumes should be verified - /// The mode to use for verification - /// An optional transaction object - /// An awaitable task - public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter backendWriter, bool latestVolumesOnly, VerifyMode verifyMode, IDbTransaction transaction) + /// The backend instance to use. + /// The options used. + /// The database to compare with. + /// The backend writer to use. + /// True if only the latest volumes should be verified. + /// The mode to use for verification. + /// The cancellation token to use. + /// An awaitable task. + public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter backendWriter, bool latestVolumesOnly, VerifyMode verifyMode, CancellationToken cancellationToken) { if (!options.NoBackendverification) { - LocalBackupDatabase backupDatabase = new LocalBackupDatabase(database, options); - IEnumerable protectedFiles = backupDatabase.GetTemporaryFilelistVolumeNames(latestVolumesOnly, transaction); - await VerifyRemoteList(backend, options, database, transaction, backendWriter, protectedFiles, [], logErrors: true, verifyMode: verifyMode).ConfigureAwait(false); + LocalBackupDatabase backupDatabase = + await LocalBackupDatabase.CreateAsync(database, options, null, cancellationToken) + .ConfigureAwait(false); + + IEnumerable protectedFiles = await backupDatabase + .GetTemporaryFilelistVolumeNames(latestVolumesOnly, cancellationToken) + .ConfigureAwait(false); + + await VerifyRemoteList(backend, options, database, backendWriter, protectedFiles, [], logErrors: true, verifyMode: verifyMode, cancellationToken).ConfigureAwait(false); } } /// /// Helper method that verifies uploaded volumes and updates their state in the database. - /// Throws an error if there are issues with the remote storage + /// Throws an error if there are issues with the remote storage. /// - /// The backend instance to use - /// The options used - /// The database to compare with - /// The log instance to use - /// Filenames that should be exempted from deletion - /// Filenames that should be exempted from strict verification - /// The mode to use for verification - /// Disable the logging of errors to prevent spamming the log; exceptions will be thrown regardless - /// An awaitable task - public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IDbTransaction transaction, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, bool logErrors, VerifyMode verifyMode) + /// The backend instance to use. + /// The options used. + /// The database to compare with. + /// The log instance to use. + /// Filenames that should be exempted from deletion. + /// Filenames that should be exempted from strict verification. + /// Disable the logging of errors to prevent spamming the log; exceptions will be thrown regardless. + /// The mode to use for verification. + /// The cancellation token to use. + /// An awaitable task. + public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, bool logErrors, VerifyMode verifyMode, CancellationToken cancellationToken) { - var tp = await RemoteListAnalysis(backend, options, database, transaction, log, protectedFiles, strictExcemptFiles, verifyMode).ConfigureAwait(false); + var tp = await RemoteListAnalysis(backend, options, database, log, protectedFiles, strictExcemptFiles, verifyMode, cancellationToken).ConfigureAwait(false); long extraCount = 0; long missingCount = 0; @@ -205,26 +215,34 @@ namespace Duplicati.Library.Main.Operation /// The verification file. /// The database instance /// The stream to write to - public static void CreateVerificationFile(LocalDatabase db, System.IO.StreamWriter stream) + /// The cancellation token to use. + public static void CreateVerificationFile(LocalDatabase db, System.IO.StreamWriter stream, CancellationToken cancellationToken) { var s = new Newtonsoft.Json.JsonSerializer(); - s.Serialize(stream, db.GetRemoteVolumes().Where(x => x.State != RemoteVolumeState.Temporary).Cast().ToArray()); + s.Serialize(stream, + db.GetRemoteVolumes(cancellationToken) + .Where(x => x.State != RemoteVolumeState.Temporary) + .ToEnumerable() + .Cast() + .ToArray() + ); } /// /// Uploads the verification file. /// - /// The backend url - /// The options to use - /// The attached database - /// An optional transaction object - public static async Task UploadVerificationFile(IBackendManager backendManager, Options options, LocalDatabase db, IDbTransaction transaction) + /// The backend manager instance to use. + /// The options to use. + /// The attached database. + /// The cancellation token to use. + /// A task that completes when the upload is done. + public static async Task UploadVerificationFile(IBackendManager backendManager, Options options, LocalDatabase db, CancellationToken cancellationToken) { using (var tempfile = new Library.Utility.TempFile()) { var remotename = options.Prefix + "-verification.json"; await using (var stream = new System.IO.StreamWriter(tempfile, false, System.Text.Encoding.UTF8)) - CreateVerificationFile(db, stream); + CreateVerificationFile(db, stream, cancellationToken); if (options.Dryrun) { @@ -232,34 +250,36 @@ namespace Duplicati.Library.Main.Operation } else { - await backendManager.PutVerificationFileAsync(remotename, tempfile, CancellationToken.None).ConfigureAwait(false); - await backendManager.WaitForEmptyAsync(db, transaction, CancellationToken.None).ConfigureAwait(false); + await backendManager.PutVerificationFileAsync(remotename, tempfile, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); } } } /// /// Helper method that verifies uploaded volumes and updates their state in the database. - /// Throws an error if there are issues with the remote storage + /// Throws an error if there are issues with the remote storage. /// - /// The backend instance to use - /// The options used - /// The database to compare with - /// Filenames that should be exempted from deletion - /// Filenames that should be exempted from strict verification - /// The mode to use for verification - /// The result of the analysis - public static async Task RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IDbTransaction transaction, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, VerifyMode verifyMode) + /// The backend manager instance to use. + /// The options used. + /// The database to compare with. + /// The log instance to use. + /// Filenames that should be exempted from deletion. + /// Filenames that should be exempted from strict verification. + /// The mode to use for verification. + /// The cancellation token to use. + /// A task that when awaited contains the analysis result. + public static async Task RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IBackendWriter log, IEnumerable protectedFiles, IEnumerable strictExcemptFiles, VerifyMode verifyMode, CancellationToken cancellationToken) { // If the last operation completed, no cleanup should be required - if (verifyMode == VerifyMode.VerifyAndClean && !database.TerminatedWithActiveUploads) + if (verifyMode == VerifyMode.VerifyAndClean && !await database.TerminatedWithActiveUploads(cancellationToken).ConfigureAwait(false)) verifyMode = VerifyMode.VerifyStrict; // Force cleanup should set the mode to cleanup if (verifyMode == VerifyMode.VerifyAndCleanForced) verifyMode = VerifyMode.VerifyAndClean; - var rawlist = await backendManager.ListAsync(CancellationToken.None).ConfigureAwait(false); + var rawlist = await backendManager.ListAsync(cancellationToken).ConfigureAwait(false); var lookup = new Dictionary(); protectedFiles = protectedFiles ?? []; strictExcemptFiles = strictExcemptFiles ?? []; @@ -290,7 +310,10 @@ namespace Duplicati.Library.Main.Operation log.KnownFileSize = knownFileSize; log.UnknownFileCount = unknownlist.Count; log.UnknownFileSize = unknownlist.Select(x => Math.Max(0, x.Size)).Sum(); - log.BackupListCount = database.FilesetTimes.Count(); + log.BackupListCount = await database + .FilesetTimes(cancellationToken) + .CountAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime(); await CheckQuota(backendManager, options, log, knownFileSize).ConfigureAwait(false); @@ -304,16 +327,18 @@ namespace Duplicati.Library.Main.Operation var missingUploadingVolumes = new HashSet(); var protectedVolumes = new HashSet(); - foreach (var e in database.DuplicateRemoteVolumes(transaction)) + await foreach (var e in database.DuplicateRemoteVolumes(cancellationToken).ConfigureAwait(false)) { if (e.Value == RemoteVolumeState.Uploading || e.Value == RemoteVolumeState.Temporary) - database.UnlinkRemoteVolume(e.Key, e.Value, transaction); + await database + .UnlinkRemoteVolume(e.Key, e.Value, cancellationToken) + .ConfigureAwait(false); else throw new RemoteListVerificationException(string.Format("The remote volume {0} appears in the database with state {1} and a deleted state, cannot continue", e.Key, e.Value.ToString()), "AmbiguousStateRemoteFiles"); } - var locallist = database.GetRemoteVolumes(transaction); - foreach (var i in locallist) + var locallist = database.GetRemoteVolumes(cancellationToken); + await foreach (var i in locallist.ConfigureAwait(false)) { var remoteFound = lookup.TryGetValue(i.Name, out var r); var correctSize = remoteFound && i.Size >= 0 && (i.Size == r.File.Size || r.File.Size < 0); @@ -321,9 +346,13 @@ namespace Duplicati.Library.Main.Operation if (archived && r.File.Size == 0) correctSize = true; if (archived && i.ArchiveTime <= DateTime.UnixEpoch) - database.UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, true, transaction); + await database + .UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, true, cancellationToken) + .ConfigureAwait(false); else if (!archived && i.ArchiveTime > DateTime.UnixEpoch) - database.UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, false, transaction); + await database + .UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, false, cancellationToken) + .ConfigureAwait(false); lookup.Remove(i.Name); @@ -378,7 +407,9 @@ namespace Duplicati.Library.Main.Operation if (remoteFound && correctSize && r.File.Size >= 0) { Logging.Log.WriteInformationMessage(LOGTAG, "PromotingCompleteFile", "Promoting uploaded complete file from {0} to {2}: {1}", i.State, i.Name, RemoteVolumeState.Uploaded); - database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash, transaction); + await database + .UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash, cancellationToken) + .ConfigureAwait(false); } else if (!remoteFound) { @@ -386,7 +417,9 @@ namespace Duplicati.Library.Main.Operation { protectedVolumes.Add(i.Name); Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "Keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name); - database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null, transaction); + await database + .UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null, cancellationToken) + .ConfigureAwait(false); } else if (verifyMode == VerifyMode.VerifyStrict && !strictExcemptFiles.Any(pf => pf == i.Name)) { @@ -397,7 +430,9 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "SchedulingMissingFileForDelete", "Scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name); missingUploadingVolumes.Add(i.Name); - database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null, transaction); + await database + .UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null, cancellationToken) + .ConfigureAwait(false); } } else @@ -435,7 +470,9 @@ namespace Duplicati.Library.Main.Operation if (!remoteFound) missing.Add(i); else if (correctSize) - database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash, transaction); + await database + .UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash, cancellationToken) + .ConfigureAwait(false); else missingHash.Add(new Tuple(r.File.Size, i)); @@ -455,19 +492,28 @@ namespace Duplicati.Library.Main.Operation } } - await backendManager.WaitForEmptyAsync(database, null, CancellationToken.None).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(database, CancellationToken.None).ConfigureAwait(false); // Batch cleanup deleted volumes in DB if (verifyMode == VerifyMode.VerifyStrict && missingUploadingVolumes.Count > 0) throw new RemoteListVerificationException($"The remote volumes {string.Join(", ", missingUploadingVolumes)} are supposed to be deleted, but strict mode is on. Try running the \"repair\" command", "DeleteDuringStrictMode"); else if (verifyMode == VerifyMode.VerifyAndClean || verifyMode == VerifyMode.VerifyStrict) { - database.RemoveRemoteVolumes(missingUploadingVolumes.Concat(temporaryAndDeletedVolumes), transaction); + await database + .RemoveRemoteVolumes( + missingUploadingVolumes + .Concat(temporaryAndDeletedVolumes), + cancellationToken + ) + .ConfigureAwait(false); + // Clear the flag after we have cleaned up if (!options.Dryrun && protectedVolumes.Count == 0) - database.TerminatedWithActiveUploads = false; + await database + .TerminatedWithActiveUploads(cancellationToken, false) + .ConfigureAwait(false); } - else if (verifyMode == VerifyMode.VerifyOnly && database.TerminatedWithActiveUploads) + else if (verifyMode == VerifyMode.VerifyOnly && await database.TerminatedWithActiveUploads(cancellationToken).ConfigureAwait(false)) { Logging.Log.WriteWarningMessage(LOGTAG, "ActiveUploadsDetected", null, "Active uploads detected, but no cleanup was performed. Run the \"repair\" command to clean up the incomplete files"); } diff --git a/Duplicati/Library/Main/Operation/ListAffected.cs b/Duplicati/Library/Main/Operation/ListAffected.cs index 77baf1ca9..571123e8a 100644 --- a/Duplicati/Library/Main/Operation/ListAffected.cs +++ b/Duplicati/Library/Main/Operation/ListAffected.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -39,36 +39,50 @@ namespace Duplicati.Library.Main.Operation m_result = result; } - public Task RunAsync(List args, Action callback = null) + public async Task RunAsync(List args, Action callback = null) { if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using (var db = new Database.LocalListAffectedDatabase(m_options.Dbpath, m_options.SqlitePageCache)) + await using var db = await Database.LocalListAffectedDatabase.CreateAsync(m_options.Dbpath, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + if (callback == null) { - if (callback == null) - { - m_result.SetResult( - db.GetFilesets(args).OrderByDescending(x => x.Time).ToArray(), - db.GetFiles(args).ToArray(), - db.GetLogLines(args).ToArray(), - db.GetVolumes(args).ToArray() - ); - } - else - { - m_result.SetResult( - db.GetFilesets(args).OrderByDescending(x => x.Time), - db.GetFiles(args), - db.GetLogLines(args), - db.GetVolumes(args) - ); - - callback(m_result); - } + m_result.SetResult( + await db.GetFilesets(args, m_result.TaskControl.ProgressToken) + .OrderByDescending(x => x.Time) + .ToArrayAsync() + .ConfigureAwait(false), + await db.GetFiles(args, m_result.TaskControl.ProgressToken) + .ToArrayAsync() + .ConfigureAwait(false), + await db.GetLogLines(args, m_result.TaskControl.ProgressToken) + .ToArrayAsync() + .ConfigureAwait(false), + await db.GetVolumes(args, m_result.TaskControl.ProgressToken) + .ToArrayAsync() + .ConfigureAwait(false) + ); } + else + { + m_result.SetResult( + await db.GetFilesets(args, m_result.TaskControl.ProgressToken) + .OrderByDescending(x => x.Time) + .ToArrayAsync() + .ConfigureAwait(false), + await db.GetFiles(args, m_result.TaskControl.ProgressToken) + .ToArrayAsync() + .ConfigureAwait(false), + await db.GetLogLines(args, m_result.TaskControl.ProgressToken) + .ToArrayAsync() + .ConfigureAwait(false), + await db.GetVolumes(args, m_result.TaskControl.ProgressToken) + .ToArrayAsync() + .ConfigureAwait(false) + ); - return Task.CompletedTask; + callback(m_result); + } } } } diff --git a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs index bbc8be9aa..b554ac130 100644 --- a/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListBrokenFilesHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable @@ -52,28 +52,31 @@ namespace Duplicati.Library.Main.Operation if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using (var db = new Database.LocalListBrokenFilesDatabase(m_options.Dbpath, m_options.SqlitePageCache)) - using (var tr = db.BeginTransaction()) - { - Utility.UpdateOptionsFromDb(db, m_options); - Utility.VerifyOptionsAndUpdateDatabase(db, m_options); - await DoRunAsync(backendManager, db, tr, filter, callbackhandler).ConfigureAwait(false); - } + await using var db = await Database.LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + await DoRunAsync(backendManager, db, filter, callbackhandler).ConfigureAwait(false); + await db.Transaction.RollBackAsync().ConfigureAwait(false); } - public static async Task<((DateTime FilesetTime, long FilesetID, long RemoveCount)[]?, List? Missing)> GetBrokenFilesetsFromRemote(IBackendManager backendManager, BasicResults result, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Options options) + public static async Task<((DateTime FilesetTime, long FilesetID, long RemoveCount)[]?, List? Missing)> GetBrokenFilesetsFromRemote(IBackendManager backendManager, BasicResults result, Database.LocalListBrokenFilesDatabase db, Options options) { List? missing = null; - var brokensets = db.GetBrokenFilesets(options.Time, options.Version, transaction).ToArray(); + var brokensets = await db + .GetBrokenFilesets(options.Time, options.Version, result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken) + .ConfigureAwait(false); if (brokensets.Length == 0) { - if (db.RepairInProgress) + if (await db.RepairInProgress(result.TaskControl.ProgressToken).ConfigureAwait(false)) throw new UserInformationException("Cannot continue because the database is marked as being under repair, but does not have broken files.", "CannotListOnDatabaseInRepair"); Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsInDatabase", "No broken filesets found in database, checking for missing remote files"); - var remotestate = await FilelistProcessor.RemoteListAnalysis(backendManager, options, db, transaction, result.BackendWriter, null, null, FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false); + var remotestate = await FilelistProcessor.RemoteListAnalysis(backendManager, options, db, result.BackendWriter, null, null, FilelistProcessor.VerifyMode.VerifyOnly, result.TaskControl.ProgressToken).ConfigureAwait(false); if (!remotestate.ParsedVolumes.Any()) throw new UserInformationException("No remote volumes were found, refusing purge", "CannotPurgeWithNoRemoteVolumes"); @@ -86,32 +89,41 @@ namespace Duplicati.Library.Main.Operation // Mark all volumes as disposable foreach (var f in missing) - db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, transaction); + await db + .UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, result.TaskControl.ProgressToken) + .ConfigureAwait(false); Logging.Log.WriteInformationMessage(LOGTAG, "MarkedRemoteFilesForDeletion", "Marked {0} remote files for deletion", missing.Count); // Drop all content from tables - db.RemoveMissingBlocks(missing.Select(x => x.Name), transaction); + await db + .RemoveMissingBlocks(missing.Select(x => x.Name), result.TaskControl.ProgressToken) + .ConfigureAwait(false); // Mark all orphaned index files as disposable after removing the missing block files - foreach (var f in db.GetOrphanedIndexFiles(transaction).ToList()) - db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, transaction); + await foreach (var f in db.GetOrphanedIndexFiles(result.TaskControl.ProgressToken).ConfigureAwait(false)) + await db + .UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, result.TaskControl.ProgressToken) + .ConfigureAwait(false); - brokensets = db.GetBrokenFilesets(options.Time, options.Version, transaction).ToArray(); + brokensets = await db + .GetBrokenFilesets(options.Time, options.Version, result.TaskControl.ProgressToken) + .ToArrayAsync() + .ConfigureAwait(false); } return (brokensets, missing); } - private async Task DoRunAsync(IBackendManager backendManager, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, IFilter filter, Func? callbackhandler) + private async Task DoRunAsync(IBackendManager backendManager, Database.LocalListBrokenFilesDatabase db, IFilter filter, Func? callbackhandler) { if (filter != null && !filter.Empty) throw new UserInformationException("Filters are not supported for this operation", "FiltersAreNotSupportedForListBrokenFiles"); - if (db.PartiallyRecreated) + if (await db.PartiallyRecreated(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) throw new UserInformationException("The command does not work on partially recreated databases", "ListBrokenFilesDoesNotWorkOnPartialDatabase"); - (var brokensets, var missing) = await GetBrokenFilesetsFromRemote(backendManager, m_result, db, transaction, m_options).ConfigureAwait(false); + (var brokensets, var missing) = await GetBrokenFilesetsFromRemote(backendManager, m_result, db, m_options).ConfigureAwait(false); if (brokensets == null) return; @@ -129,35 +141,41 @@ namespace Duplicati.Library.Main.Operation return; } - var fstimes = db.FilesetTimes.ToList(); + var fstimes = await db + .FilesetTimes(m_result.TaskControl.ProgressToken) + .ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); var brokenfilesets = brokensets.Select(x => new { - Version = fstimes.FindIndex(y => y.Key == x.Item2), - Timestamp = x.Item1, - FilesetID = x.Item2, - BrokenCount = x.Item3 + Version = fstimes.FindIndex(y => y.Key == x.FilesetID), + Timestamp = x.FilesetTime, + FilesetID = x.FilesetID, + BrokenCount = x.RemoveCount } ) .ToArray(); m_result.BrokenFiles = - brokenfilesets.Select( - x => new Tuple>>( + await Task.WhenAll(brokenfilesets.Select( + async x => new Tuple>>( x.Version, x.Timestamp, callbackhandler == null && !m_options.ListSetsOnly - ? db.GetBrokenFilenames(x.FilesetID, transaction).ToArray().AsEnumerable() + ? await db + .GetBrokenFilenames(x.FilesetID, m_result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) + .ConfigureAwait(false) : new MockList>((int)x.BrokenCount) - )) - .ToArray(); + ))) + .ConfigureAwait(false); if (callbackhandler != null) foreach (var bs in brokenfilesets) - foreach (var fe in db.GetBrokenFilenames(bs.FilesetID, transaction)) + await foreach (var fe in db.GetBrokenFilenames(bs.FilesetID, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) if (!callbackhandler(bs.Version, bs.Timestamp, bs.BrokenCount, fe.Item1, fe.Item2)) break; } diff --git a/Duplicati/Library/Main/Operation/ListChangesHandler.cs b/Duplicati/Library/Main/Operation/ListChangesHandler.cs index 603a6f331..71bfe0fd2 100644 --- a/Duplicati/Library/Main/Operation/ListChangesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListChangesHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -79,103 +79,118 @@ namespace Duplicati.Library.Main.Operation DateTime baseVersionTime; DateTime compareVersionTime; - using (var tmpdb = useLocalDb ? null : new TempFile()) - using (var db = new Database.LocalListChangesDatabase(useLocalDb ? m_options.Dbpath : (string)tmpdb, m_options.SqlitePageCache)) - using (var storageKeeper = db.CreateStorageHelper()) + using var tmpdb = useLocalDb ? null : new TempFile(); + await using var db = await Database.LocalListChangesDatabase.CreateAsync(useLocalDb ? m_options.Dbpath : (string)tmpdb, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await using var storageKeeper = await db.CreateStorageHelper(m_result.TaskControl.ProgressToken).ConfigureAwait(false); + if (useLocalDb) { - if (useLocalDb) - { - var dbtimes = db.FilesetTimes.ToList(); - if (dbtimes.Count < 2) - throw new UserInformationException(string.Format("Need at least two backups to show differences, database contains {0} backups", dbtimes.Count), "NeedTwoBackupsToStartDiff"); + var dbtimes = await db + .FilesetTimes(m_result.TaskControl.ProgressToken) + .ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); - long baseVersionId; - long compareVersionId; + if (dbtimes.Count < 2) + throw new UserInformationException(string.Format("Need at least two backups to show differences, database contains {0} backups", dbtimes.Count), "NeedTwoBackupsToStartDiff"); - var times = dbtimes.Zip(Enumerable.Range(0, dbtimes.Count), (a, b) => new Tuple(b, a.Value, a.Key)).ToList(); - var bt = SelectTime(baseVersion, times, out baseVersionIndex, out baseVersionTime, out baseVersionId); - times.Remove(bt); - SelectTime(compareVersion, times, out compareVersionIndex, out compareVersionTime, out compareVersionId); + long baseVersionId; + long compareVersionId; - storageKeeper.AddFromDb(baseVersionId, false, filter); - storageKeeper.AddFromDb(compareVersionId, true, filter); - } - else - { - Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, accessing remote store"); + var times = dbtimes.Zip(Enumerable.Range(0, dbtimes.Count), (a, b) => new Tuple(b, a.Value, a.Key)).ToList(); + var bt = SelectTime(baseVersion, times, out baseVersionIndex, out baseVersionTime, out baseVersionId); + times.Remove(bt); + SelectTime(compareVersion, times, out compareVersionIndex, out compareVersionTime, out compareVersionId); - var parsedlist = (await backendManager.ListAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) - .Select(n => Volumes.VolumeBase.ParseFilename(n)) - .Where(p => p != null && p.FileType == RemoteVolumeType.Files) - .OrderByDescending(p => p.Time) - .ToArray(); - - var numberedList = parsedlist.Zip(Enumerable.Range(0, parsedlist.Length), (a, b) => new Tuple(b, a.Time, a)).ToList(); - if (numberedList.Count < 2) - throw new UserInformationException(string.Format("Need at least two backups to show differences, database contains {0} backups", numberedList.Count), "NeedTwoBackupsToStartDiff"); - - Volumes.IParsedVolume baseFile; - Volumes.IParsedVolume compareFile; - - var bt = SelectTime(baseVersion, numberedList, out baseVersionIndex, out baseVersionTime, out baseFile); - numberedList.Remove(bt); - SelectTime(compareVersion, numberedList, out compareVersionIndex, out compareVersionTime, out compareFile); - - Func conv = (x) => - { - switch (x) - { - case FilelistEntryType.File: - return ListChangesElementType.File; - case FilelistEntryType.Folder: - return ListChangesElementType.Folder; - case FilelistEntryType.Symlink: - return ListChangesElementType.Symlink; - default: - return (ListChangesElementType)(-1); - } - }; - - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; - - using (var tmpfile = await backendManager.GetAsync(baseFile.File.Name, null, baseFile.File.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) - using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(baseFile.File.Name), tmpfile, m_options)) - foreach (var f in rd.Files) - if (FilterExpression.Matches(filter, f.Path)) - storageKeeper.AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), false); - - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; - - using (var tmpfile = await backendManager.GetAsync(compareFile.File.Name, null, compareFile.File.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) - using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(compareFile.File.Name), tmpfile, m_options)) - foreach (var f in rd.Files) - if (FilterExpression.Matches(filter, f.Path)) - storageKeeper.AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), true); - } - - var changes = storageKeeper.CreateChangeCountReport(); - var sizes = storageKeeper.CreateChangeSizeReport(); - - var lst = (m_options.FullResult || callback != null) ? - (from n in storageKeeper.CreateChangedFileReport() - select n) : null; - - m_result.SetResult( - baseVersionTime, baseVersionIndex, compareVersionTime, compareVersionIndex, - changes.AddedFolders, changes.AddedSymlinks, changes.AddedFiles, - changes.DeletedFolders, changes.DeletedSymlinks, changes.DeletedFiles, - changes.ModifiedFolders, changes.ModifiedSymlinks, changes.ModifiedFiles, - sizes.AddedSize, sizes.DeletedSize, sizes.PreviousSize, sizes.CurrentSize, - (lst == null || callback == null) ? null : lst.ToArray() - ); - - if (callback != null) - callback(m_result, lst); - - return; + await storageKeeper + .AddFromDb(baseVersionId, false, filter, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + await storageKeeper + .AddFromDb(compareVersionId, true, filter, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); } + else + { + Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, accessing remote store"); + + var parsedlist = (await backendManager.ListAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + .Select(n => Volumes.VolumeBase.ParseFilename(n)) + .Where(p => p != null && p.FileType == RemoteVolumeType.Files) + .OrderByDescending(p => p.Time) + .ToArray(); + + var numberedList = parsedlist.Zip(Enumerable.Range(0, parsedlist.Length), (a, b) => new Tuple(b, a.Time, a)).ToList(); + if (numberedList.Count < 2) + throw new UserInformationException(string.Format("Need at least two backups to show differences, database contains {0} backups", numberedList.Count), "NeedTwoBackupsToStartDiff"); + + Volumes.IParsedVolume baseFile; + Volumes.IParsedVolume compareFile; + + var bt = SelectTime(baseVersion, numberedList, out baseVersionIndex, out baseVersionTime, out baseFile); + numberedList.Remove(bt); + SelectTime(compareVersion, numberedList, out compareVersionIndex, out compareVersionTime, out compareFile); + + Func conv = (x) => + { + switch (x) + { + case FilelistEntryType.File: + return ListChangesElementType.File; + case FilelistEntryType.Folder: + return ListChangesElementType.Folder; + case FilelistEntryType.Symlink: + return ListChangesElementType.Symlink; + default: + return (ListChangesElementType)(-1); + } + }; + + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + + using (var tmpfile = await backendManager.GetAsync(baseFile.File.Name, null, baseFile.File.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(baseFile.File.Name), tmpfile, m_options)) + foreach (var f in rd.Files) + if (FilterExpression.Matches(filter, f.Path)) + await storageKeeper + .AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), false, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + + using (var tmpfile = await backendManager.GetAsync(compareFile.File.Name, null, compareFile.File.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(compareFile.File.Name), tmpfile, m_options)) + foreach (var f in rd.Files) + if (FilterExpression.Matches(filter, f.Path)) + await storageKeeper + .AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), true, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + } + + var changes = await storageKeeper + .CreateChangeCountReport(m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + var sizes = await storageKeeper + .CreateChangeSizeReport(m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + var lst = (m_options.FullResult || callback != null) ? + (from n in storageKeeper.CreateChangedFileReport(m_result.TaskControl.ProgressToken) + select n) : null; + + m_result.SetResult( + baseVersionTime, baseVersionIndex, compareVersionTime, compareVersionIndex, + changes.AddedFolders, changes.AddedSymlinks, changes.AddedFiles, + changes.DeletedFolders, changes.DeletedSymlinks, changes.DeletedFiles, + changes.ModifiedFolders, changes.ModifiedSymlinks, changes.ModifiedFiles, + sizes.AddedSize, sizes.DeletedSize, sizes.PreviousSize, sizes.CurrentSize, + (lst == null || callback == null) ? null : await lst.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken).ConfigureAwait(false) + ); + + if (callback != null) + callback(m_result, lst.ToEnumerable()); + + return; } } } diff --git a/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs b/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs index 02c5bf8da..0d21ebf02 100644 --- a/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListControlFilesHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -42,51 +42,51 @@ namespace Duplicati.Library.Main.Operation public async Task RunAsync(IBackendManager backendManager, IEnumerable filterstrings, IFilter compositefilter) { var cancellationToken = CancellationToken.None; - using (var tmpdb = new TempFile()) - using (var db = new LocalDatabase(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true, m_options.SqlitePageCache)) + using var tmpdb = new TempFile(); + await using var db = await LocalDatabase.CreateLocalDatabaseAsync(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + var filter = JoinedFilterExpression.Join(new Library.Utility.FilterExpression(filterstrings), compositefilter); + + try { - var filter = JoinedFilterExpression.Join(new Library.Utility.FilterExpression(filterstrings), compositefilter); + var filteredList = ListFilesHandler.ParseAndFilterFilesets(await backendManager.ListAsync(cancellationToken).ConfigureAwait(false), m_options); + if (filteredList.Count == 0) + throw new Exception("No filesets found on remote target"); - try - { - var filteredList = ListFilesHandler.ParseAndFilterFilesets(await backendManager.ListAsync(cancellationToken).ConfigureAwait(false), m_options); - if (filteredList.Count == 0) - throw new Exception("No filesets found on remote target"); + Exception lastEx = new Exception("No suitable files found on remote target"); - Exception lastEx = new Exception("No suitable files found on remote target"); + foreach (var fileversion in filteredList) + try + { + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; - foreach (var fileversion in filteredList) - try - { - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; + var file = fileversion.Value.File; + var entry = await db + .GetRemoteVolume(file.Name, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); - var file = fileversion.Value.File; - var entry = db.GetRemoteVolume(file.Name); + var files = new List(); + using (var tmpfile = await backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, cancellationToken).ConfigureAwait(false)) + using (var tmp = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(file.Name), tmpfile, m_options)) + foreach (var cf in tmp.ControlFiles) + if (Library.Utility.FilterExpression.Matches(filter, cf.Key)) + files.Add(new ListResultFile(cf.Key, null)); - var files = new List(); - using (var tmpfile = await backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, cancellationToken).ConfigureAwait(false)) - using (var tmp = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(file.Name), tmpfile, m_options)) - foreach (var cf in tmp.ControlFiles) - if (Library.Utility.FilterExpression.Matches(filter, cf.Key)) - files.Add(new ListResultFile(cf.Key, null)); + m_result.SetResult(new Library.Interface.IListResultFileset[] { new ListResultFileset(fileversion.Key, BackupType.PARTIAL_BACKUP, fileversion.Value.Time, -1, -1) }, files); + lastEx = null; + break; + } + catch (Exception ex) when (!ex.IsAbortException()) + { + lastEx = ex; + } - m_result.SetResult(new Library.Interface.IListResultFileset[] { new ListResultFileset(fileversion.Key, BackupType.PARTIAL_BACKUP, fileversion.Value.Time, -1, -1) }, files); - lastEx = null; - break; - } - catch (Exception ex) when (!ex.IsAbortException()) - { - lastEx = ex; - } - - if (lastEx != null) - throw lastEx; - } - finally - { - await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); - } + if (lastEx != null) + throw lastEx; + } + finally + { + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs b/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs index 95e53370b..8829f14cb 100644 --- a/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFileVersionsHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable @@ -42,16 +42,22 @@ internal static class ListFileVersionsHandler /// The offset to start listing from /// The maximum number of results to return /// A task representing the asynchronous operation - public static Task RunAsync(Options options, ListFileVersionsResults result, string[] paths, long offset, long limit) + public async static Task RunAsync(Options options, ListFileVersionsResults result, string[] paths, long offset, long limit) { if (!System.IO.File.Exists(options.Dbpath) || options.NoLocalDb) throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase"); - using var db = new Database.LocalListDatabase(options.Dbpath, options.SqlitePageCache); + await using var db = + await Database.LocalListDatabase.CreateAsync(options.Dbpath, null, result.TaskControl.ProgressToken) + .ConfigureAwait(false); long[]? filesetIds = null; if (!options.AllVersions) { - filesetIds = db.GetFilesetIDs(options.Time, options.Version).ToArray(); + filesetIds = await db + .GetFilesetIDs(options.Time, options.Version, false, result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken) + .ConfigureAwait(false); + if (filesetIds.Length == 0) throw new UserInformationException("No filesets found", "NoFilesetsFound"); } @@ -60,8 +66,8 @@ internal static class ListFileVersionsHandler throw new UserInformationException("No path specified", "NoPathSpecified"); paths = paths.Select(path => Util.AppendDirSeparator(path)).ToArray(); - result.FileVersions = db.ListFileVersions(paths, filesetIds, offset, limit); - - return Task.CompletedTask; + result.FileVersions = await db + .ListFileVersions(paths, filesetIds, offset, limit, result.TaskControl.ProgressToken) + .ConfigureAwait(false); } } diff --git a/Duplicati/Library/Main/Operation/ListFilesHandler.cs b/Duplicati/Library/Main/Operation/ListFilesHandler.cs index ec68ca2b9..8ad83b976 100644 --- a/Duplicati/Library/Main/Operation/ListFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFilesHandler.cs @@ -1,23 +1,24 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. + using System.Collections.Generic; using System.Linq; using System.Threading; @@ -54,59 +55,76 @@ namespace Duplicati.Library.Main.Operation //Use a speedy local query if (!m_options.NoLocalDb && System.IO.File.Exists(m_options.Dbpath)) - using (var db = new Database.LocalListDatabase(m_options.Dbpath, m_options.SqlitePageCache)) + await using (var db = await Database.LocalListDatabase.CreateAsync(m_options.Dbpath, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { - using (var filesets = db.SelectFileSets(m_options.Time, m_options.Version)) + await using var filesets = await db.SelectFileSets(m_options.Time, m_options.Version, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + if (!filter.Empty) { - if (!filter.Empty) + if (simpleList || (m_options.ListFolderContents && !m_options.AllVersions)) { - if (simpleList || (m_options.ListFolderContents && !m_options.AllVersions)) - { - filesets.TakeFirst(); - } + await filesets + .TakeFirst(m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); } - - IEnumerable files; - if (m_options.ListFolderContents) - { - files = filesets.SelectFolderContents(filter); - } - else if (m_options.ListPrefixOnly) - { - files = filesets.GetLargestPrefix(filter); - } - else if (filter.Empty) - { - files = null; - } - else - { - files = filesets.SelectFiles(filter); - } - - if (m_options.ListSetsOnly) - { - m_result.SetResult( - filesets.QuickSets.Select(x => new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes)).ToArray(), - null - ); - } - else - { - m_result.SetResult( - filesets.Sets.Select(x => - new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes)).ToArray(), - files == null - ? null - : (from n in files - select (Duplicati.Library.Interface.IListResultFile)(new ListResultFile(n.Path, - n.Sizes.ToArray()))) - .ToArray() - ); - } - - return; } + + IAsyncEnumerable files; + if (m_options.ListFolderContents) + { + files = filesets.SelectFolderContents(filter, m_result.TaskControl.ProgressToken); + } + else if (m_options.ListPrefixOnly) + { + files = filesets.GetLargestPrefix(filter, m_result.TaskControl.ProgressToken); + } + else if (filter.Empty) + { + files = null; + } + else + { + files = filesets.SelectFiles(filter, m_result.TaskControl.ProgressToken); + } + + if (m_options.ListSetsOnly) + { + m_result.SetResult( + await filesets + .QuickSets(m_result.TaskControl.ProgressToken) + .Select(x => new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes)) + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) + .ConfigureAwait(false), + null + ); + } + else + { + m_result.SetResult( + await filesets + .Sets(m_result.TaskControl.ProgressToken) + .Select(x => new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes)) + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) + .ConfigureAwait(false), + files == null + ? null + : + await files.Select(async n => + new ListResultFile( + n.Path, + await n + .Sizes(m_result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) + .ConfigureAwait(false) + ) + ) + .Select(x => x.Result) + .Cast() + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) + .ConfigureAwait(false) + ); + } + + return; } Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, accessing remote store"); @@ -119,13 +137,14 @@ namespace Duplicati.Library.Main.Operation // Otherwise, grab info from remote location using (var tmpdb = new TempFile()) - using (var db = new LocalDatabase(tmpdb, "List", true, m_options.SqlitePageCache)) + await using (var db = await LocalDatabase.CreateLocalDatabaseAsync(tmpdb, "List", true, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { var filteredList = ParseAndFilterFilesets(await backendManager.ListAsync(cancellationToken).ConfigureAwait(false), m_options); if (filteredList.Count == 0) throw new UserInformationException("No filesets found on remote target", "EmptyRemoteFolder"); - var numberSeq = await CreateResultSequence(filteredList, backendManager, m_options, cancellationToken); + var numberSeq = await CreateResultSequence(filteredList, backendManager, m_options, cancellationToken) + .ConfigureAwait(false); if (filter.Empty) { m_result.SetResult(numberSeq, null); diff --git a/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs b/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs index 81d7c311f..22ce6e35c 100644 --- a/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFilesetsHandler.cs @@ -1,27 +1,28 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable using System.Linq; +using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Interface; @@ -46,11 +47,17 @@ internal static class ListFilesetsHandler /// A task representing the asynchronous operation public static async Task RunAsync(Options options, ListFilesetResults result, IBackendManager backendManager) { - //Use a speedy local query + // Use a speedy local query if (System.IO.File.Exists(options.Dbpath) && !options.NoLocalDb) { - using var db = new Database.LocalListDatabase(options.Dbpath, options.SqlitePageCache); - result.Filesets = db.ListFilesetsExtended().ToArray(); + await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, null, result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + result.Filesets = await db + .ListFilesetsExtended(result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken) + .ConfigureAwait(false); + return; } diff --git a/Duplicati/Library/Main/Operation/ListFolderHandler.cs b/Duplicati/Library/Main/Operation/ListFolderHandler.cs index 400184a5d..0305812f2 100644 --- a/Duplicati/Library/Main/Operation/ListFolderHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFolderHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable @@ -41,13 +41,19 @@ internal static class ListFolderHandler /// The offset to start listing from /// The maximum number of entries to list /// A task representing the asynchronous operation - public static Task RunAsync(Options options, ListFolderResults result, string[] folders, long offset, long limit) + public static async Task RunAsync(Options options, ListFolderResults result, string[] folders, long offset, long limit) { if (!System.IO.File.Exists(options.Dbpath) || options.NoLocalDb) throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase"); - using var db = new Database.LocalListDatabase(options.Dbpath, options.SqlitePageCache); - var filesetIds = db.GetFilesetIDs(options.Time, options.Version, singleTimeMatch: true).ToArray(); + await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, null, result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + var filesetIds = await db + .GetFilesetIDs(options.Time, options.Version, singleTimeMatch: true, result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken) + .ConfigureAwait(false); + if (filesetIds.Length == 0) throw new UserInformationException("No filesets found", "NoFilesetsFound"); if (filesetIds.Length > 1) @@ -57,13 +63,23 @@ internal static class ListFolderHandler { if (folders != null && folders.Length > 1) throw new UserInformationException("When no folder is specified, only one folder can be listed", "MultipleFoldersFound"); - var rootFolders = db.GetMinimalUniquePrefixEntries(filesetIds[0]); + var rootFolders = await db + .GetMinimalUniquePrefixEntries(filesetIds[0], result.TaskControl.ProgressToken) + .ToListAsync(cancellationToken: result.TaskControl.ProgressToken) + .ConfigureAwait(false); result.Entries = new PaginatedResults(0, rootFolders.Count, 1, rootFolders.Count, rootFolders); } else { - result.Entries = db.ListFolder(db.GetPrefixIds(folders), filesetIds[0], offset, limit); + result.Entries = await db + .ListFolder( + db.GetPrefixIds(folders, result.TaskControl.ProgressToken).ToEnumerable(), + filesetIds[0], + offset, + limit, + result.TaskControl.ProgressToken + ) + .ConfigureAwait(false); } - return Task.CompletedTask; } } diff --git a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs index 9eaa42b84..fb13f13c3 100644 --- a/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeBrokenFilesHandler.cs @@ -1,203 +1,225 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -#nullable enable - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Duplicati.Library.Interface; -using Duplicati.Library.Main.Database; - -namespace Duplicati.Library.Main.Operation -{ - internal class PurgeBrokenFilesHandler - { - /// - /// The tag used for logging - /// - private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(PurgeBrokenFilesHandler)); - protected readonly Options m_options; - protected readonly PurgeBrokenFilesResults m_result; - - public PurgeBrokenFilesHandler(Options options, PurgeBrokenFilesResults result) - { - m_options = options; - m_result = result; - } - - public async Task RunAsync(IBackendManager backendManager, Library.Utility.IFilter filter) - { - if (!System.IO.File.Exists(m_options.Dbpath)) - throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - - if (filter != null && !filter.Empty) - throw new UserInformationException("Filters are not supported for this operation", "FiltersNotAllowedOnPurgeBrokenFiles"); - - using (var db = new LocalListBrokenFilesDatabase(m_options.Dbpath, m_options.SqlitePageCache)) - using (var tr = db.BeginTransaction()) - { - if (db.PartiallyRecreated) - throw new UserInformationException("The command does not work on partially recreated databases", "CannotPurgeOnPartialDatabase"); - - Utility.UpdateOptionsFromDb(db, m_options); - Utility.VerifyOptionsAndUpdateDatabase(db, m_options); - - (var sets, var missing) = await ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(backendManager, m_result, db, tr, m_options).ConfigureAwait(false); - if (sets == null) - return; - - if (sets.Length == 0) - { - if (missing == null) - Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesets", "Found no broken filesets"); - else if (missing.Count == 0) - Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsOrMissingFiles", "Found no broken filesets and no missing remote files"); - else - Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenSetsButMissingRemoteFiles", string.Format("Found no broken filesets, but {0} missing remote files. Purging from database.", missing.Count)); - } - else - { - Logging.Log.WriteInformationMessage(LOGTAG, "FoundBrokenFilesets", "Found {0} broken filesets with {1} affected files, purging files", sets.Length, sets.Sum(x => x.RemoveCount)); - - var pgoffset = 0.0f; - var pgspan = 0.95f / sets.Length; - - var filesets = db.FilesetTimes.ToList(); - - var compare_list = sets.Select(x => new - { - FilesetID = x.Item2, - Timestamp = x.Item1, - RemoveCount = x.Item3, - Version = filesets.FindIndex(y => y.Key == x.Item2), - SetCount = db.GetFilesetFileCount(x.Item2, tr) - }).ToArray(); - - var replacementMetadataBlocksetId = -1L; - if (!m_options.DisableReplaceMissingMetadata) - { - var emptymetadata = Utility.WrapMetadata(new Dictionary(), m_options); - replacementMetadataBlocksetId = db.GetEmptyMetadataBlocksetId((missing ?? []).Select(x => x.ID), emptymetadata.FileHash, emptymetadata.Blob.Length, null); - if (replacementMetadataBlocksetId < 0) - throw new UserInformationException($"Failed to locate an empty metadata blockset to replace missing metadata. Set the option --disable-replace-missing-metadata=true to ignore this and drop files with missing metadata.", "FailedToLocateEmptyMetadataBlockset"); - } - - var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray(); - var to_purge = compare_list.Where(x => x.RemoveCount != x.SetCount).ToArray(); - - if (fully_emptied.Length == db.FilesetTimes.Count()) - throw new UserInformationException("All filesets are fully broken and needs to be removed. To avoid unexpected deletions, you must manually remove the remote files and delete the database.", "AllFilesetsBroken"); - - if (!m_options.Dryrun) - tr.Commit(); - - - if (fully_emptied.Length != 0) - { - if (fully_emptied.Length == 1) - Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing entire fileset {1} as all {0} file(s) are broken", fully_emptied.First().Timestamp, fully_emptied.First().RemoveCount); - else - Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing {0} filesets where all file(s) are broken: {1}", fully_emptied.Length, string.Join(", ", fully_emptied.Select(x => x.Timestamp.ToLocalTime().ToString()))); - - m_result.DeleteResults = new DeleteResults(m_result); - using (var rmdb = new LocalDeleteDatabase(db)) - using (var deltr = new ReusableTransaction(rmdb)) - { - var opts = new Options(new Dictionary(m_options.RawOptions)); - opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString())); - opts.RawOptions.Remove("time"); - opts.RawOptions["no-auto-compact"] = "true"; - - await new DeleteHandler(opts, (DeleteResults)m_result.DeleteResults) - .DoRunAsync(rmdb, deltr, true, false, backendManager).ConfigureAwait(false); - - if (!m_options.Dryrun) - deltr.Commit("CommitDelete", restart: false); - } - - pgoffset += (pgspan * fully_emptied.Length); - m_result.OperationProgressUpdater.UpdateProgress(pgoffset); - } - - if (to_purge.Length > 0) - { - m_result.PurgeResults = new PurgeFilesResults(m_result); - - foreach (var bs in to_purge) - { - Logging.Log.WriteInformationMessage(LOGTAG, "PurgingFiles", "Purging {0} file(s) from fileset {1}", bs.RemoveCount, bs.Timestamp.ToLocalTime()); - var opts = new Options(new Dictionary(m_options.RawOptions)); - - using (var pgdb = new Database.LocalPurgeDatabase(db)) - { - // Recompute the version number after we deleted the versions before - filesets = pgdb.FilesetTimes.ToList(); - var thisversion = filesets.FindIndex(y => y.Key == bs.FilesetID); - if (thisversion < 0) - throw new Exception(string.Format("Failed to find match for {0} ({1}) in {2}", bs.FilesetID, bs.Timestamp.ToLocalTime(), string.Join(", ", filesets.Select(x => x.ToString())))); - - opts.RawOptions["version"] = thisversion.ToString(); - opts.RawOptions.Remove("time"); - opts.RawOptions["no-auto-compact"] = "true"; - - await new PurgeFilesHandler(opts, (PurgeFilesResults)m_result.PurgeResults).RunAsync(backendManager, pgdb, pgoffset, pgspan, (cmd, filesetid, tablename) => - { - if (filesetid != bs.FilesetID) - throw new Exception(string.Format("Unexpected filesetid: {0}, expected {1}", filesetid, bs.FilesetID)); - - // Update entries that would be removed because of missing metadata - var updatedEntries = 0; - if (!m_options.DisableReplaceMissingMetadata) - updatedEntries = db.ReplaceMetadata(filesetid, replacementMetadataBlocksetId, cmd.Transaction); - - db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID", cmd.Transaction); - return updatedEntries; - }).ConfigureAwait(false); - } - - pgoffset += pgspan; - m_result.OperationProgressUpdater.UpdateProgress(pgoffset); - } - } - } - - m_result.OperationProgressUpdater.UpdateProgress(0.95f); - - if (!m_options.Dryrun && db.RepairInProgress) - { - Logging.Log.WriteInformationMessage(LOGTAG, "ValidatingDatabase", "Database was previously marked as in-progress, checking if it is valid after purging files"); - db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null); - Logging.Log.WriteInformationMessage(LOGTAG, "UpdatingDatabase", "Purge completed, and consistency checks completed, marking database as complete"); - db.RepairInProgress = false; - } - else - { - db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null); - } - - m_result.OperationProgressUpdater.UpdateProgress(1.0f); - - } - } - } -} +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Duplicati.Library.Interface; +using Duplicati.Library.Main.Database; +using Duplicati.Library.Utility; + +namespace Duplicati.Library.Main.Operation +{ + internal class PurgeBrokenFilesHandler + { + /// + /// The tag used for logging + /// + private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(PurgeBrokenFilesHandler)); + protected readonly Options m_options; + protected readonly PurgeBrokenFilesResults m_result; + + public PurgeBrokenFilesHandler(Options options, PurgeBrokenFilesResults result) + { + m_options = options; + m_result = result; + } + + public async Task RunAsync(IBackendManager backendManager, Library.Utility.IFilter filter) + { + if (!System.IO.File.Exists(m_options.Dbpath)) + throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); + + if (filter != null && !filter.Empty) + throw new UserInformationException("Filters are not supported for this operation", "FiltersNotAllowedOnPurgeBrokenFiles"); + + await using var db = await LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + if (await db.PartiallyRecreated(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + throw new UserInformationException("The command does not work on partially recreated databases", "CannotPurgeOnPartialDatabase"); + + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + var (sets, missing) = await ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(backendManager, m_result, db, m_options).ConfigureAwait(false); + if (sets == null) + return; + + if (sets.Length == 0) + { + if (missing == null) + Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesets", "Found no broken filesets"); + else if (missing.Count == 0) + Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsOrMissingFiles", "Found no broken filesets and no missing remote files"); + else + Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenSetsButMissingRemoteFiles", string.Format("Found no broken filesets, but {0} missing remote files. Purging from database.", missing.Count)); + } + else + { + Logging.Log.WriteInformationMessage(LOGTAG, "FoundBrokenFilesets", "Found {0} broken filesets with {1} affected files, purging files", sets.Length, sets.Sum(x => x.RemoveCount)); + + var pgoffset = 0.0f; + var pgspan = 0.95f / sets.Length; + + var filesets = await db + .FilesetTimes(m_result.TaskControl.ProgressToken) + .ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + var compare_list = sets.Select(async x => new + { + FilesetID = x.FilesetID, + Timestamp = x.FilesetTime, + RemoveCount = x.RemoveCount, + Version = filesets.FindIndex(y => y.Key == x.FilesetID), + SetCount = await db + .GetFilesetFileCount(x.FilesetID, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false) + }) + .Select(x => x.Result) + .ToArray(); + + var replacementMetadataBlocksetId = -1L; + if (!m_options.DisableReplaceMissingMetadata) + { + var emptymetadata = Utility.WrapMetadata(new Dictionary(), m_options); + replacementMetadataBlocksetId = await db + .GetEmptyMetadataBlocksetId( + (missing ?? []).Select(x => x.ID), + emptymetadata.FileHash, + emptymetadata.Blob.Length, + m_result.TaskControl.ProgressToken + ) + .ConfigureAwait(false); + if (replacementMetadataBlocksetId < 0) + throw new UserInformationException($"Failed to locate an empty metadata blockset to replace missing metadata. Set the option --disable-replace-missing-metadata=true to ignore this and drop files with missing metadata.", "FailedToLocateEmptyMetadataBlockset"); + } + + var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray(); + var to_purge = compare_list.Where(x => x.RemoveCount != x.SetCount).ToArray(); + + if (fully_emptied.Length == await db.FilesetTimes(m_result.TaskControl.ProgressToken).CountAsync(cancellationToken: m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + throw new UserInformationException("All filesets are fully broken and needs to be removed. To avoid unexpected deletions, you must manually remove the remote files and delete the database.", "AllFilesetsBroken"); + + if (!m_options.Dryrun) + await db.Transaction.CommitAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false); + + + if (fully_emptied.Length != 0) + { + if (fully_emptied.Length == 1) + Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing entire fileset {1} as all {0} file(s) are broken", fully_emptied.First().Timestamp, fully_emptied.First().RemoveCount); + else + Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing {0} filesets where all file(s) are broken: {1}", fully_emptied.Length, string.Join(", ", fully_emptied.Select(x => x.Timestamp.ToLocalTime().ToString()))); + + m_result.DeleteResults = new DeleteResults(m_result); + await using (var rmdb = await LocalDeleteDatabase.CreateAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + { + var opts = new Options(new Dictionary(m_options.RawOptions)); + opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString())); + opts.RawOptions.Remove("time"); + opts.RawOptions["no-auto-compact"] = "true"; + + await new DeleteHandler(opts, (DeleteResults)m_result.DeleteResults) + .DoRunAsync(rmdb, true, false, backendManager).ConfigureAwait(false); + + if (!m_options.Dryrun) + await rmdb.Transaction + .CommitAsync("CommitDelete", true, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + } + + pgoffset += (pgspan * fully_emptied.Length); + m_result.OperationProgressUpdater.UpdateProgress(pgoffset); + } + + if (to_purge.Length > 0) + { + m_result.PurgeResults = new PurgeFilesResults(m_result); + + foreach (var bs in to_purge) + { + Logging.Log.WriteInformationMessage(LOGTAG, "PurgingFiles", "Purging {0} file(s) from fileset {1}", bs.RemoveCount, bs.Timestamp.ToLocalTime()); + var opts = new Options(new Dictionary(m_options.RawOptions)); + + await using (var pgdb = await Database.LocalPurgeDatabase.CreateAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + { + // Recompute the version number after we deleted the versions before + filesets = await pgdb + .FilesetTimes(m_result.TaskControl.ProgressToken) + .ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + var thisversion = filesets.FindIndex(y => y.Key == bs.FilesetID); + if (thisversion < 0) + throw new Exception(string.Format("Failed to find match for {0} ({1}) in {2}", bs.FilesetID, bs.Timestamp.ToLocalTime(), string.Join(", ", filesets.Select(x => x.ToString())))); + + opts.RawOptions["version"] = thisversion.ToString(); + opts.RawOptions.Remove("time"); + opts.RawOptions["no-auto-compact"] = "true"; + + await new PurgeFilesHandler(opts, (PurgeFilesResults)m_result.PurgeResults).RunAsync(backendManager, pgdb, pgoffset, pgspan, async (cmd, filesetid, tablename) => + { + if (filesetid != bs.FilesetID) + throw new Exception(string.Format("Unexpected filesetid: {0}, expected {1}", filesetid, bs.FilesetID)); + + // Update entries that would be removed because of missing metadata + var updatedEntries = 0; + if (!m_options.DisableReplaceMissingMetadata) + updatedEntries = await db.ReplaceMetadata(filesetid, replacementMetadataBlocksetId, m_result.TaskControl.ProgressToken); + + await db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID", m_result.TaskControl.ProgressToken); + return updatedEntries; + }).ConfigureAwait(false); + } + + pgoffset += pgspan; + m_result.OperationProgressUpdater.UpdateProgress(pgoffset); + } + } + } + + m_result.OperationProgressUpdater.UpdateProgress(0.95f); + + if (!m_options.Dryrun && await db.RepairInProgress(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + { + Logging.Log.WriteInformationMessage(LOGTAG, "ValidatingDatabase", "Database was previously marked as in-progress, checking if it is valid after purging files"); + await db + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + Logging.Log.WriteInformationMessage(LOGTAG, "UpdatingDatabase", "Purge completed, and consistency checks completed, marking database as complete"); + await db.RepairInProgress(m_result.TaskControl.ProgressToken, false).ConfigureAwait(false); + } + else + { + await db.Transaction.RollBackAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await db + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + } + + m_result.OperationProgressUpdater.UpdateProgress(1.0f); + } + } +} diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index dafbdbfa9..11650c4a1 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -25,6 +25,7 @@ using System.Threading.Tasks; using Duplicati.Library.Interface; using Duplicati.Library.Main.Volumes; using Duplicati.Library.Utility; +using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Operation { @@ -51,51 +52,62 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using (var db = new Database.LocalPurgeDatabase(m_options.Dbpath, m_options.SqlitePageCache)) - { - await DoRunAsync(backendManager, db, filter, null, 0, 1).ConfigureAwait(false); - db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null); - } + await using var db = await Database.LocalPurgeDatabase.CreateAsync(m_options.Dbpath, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await DoRunAsync(backendManager, db, filter, null, 0, 1).ConfigureAwait(false); + await db + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); } - public Task RunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, float pgoffset, float pgspan, Func filtercommand) + public Task RunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, float pgoffset, float pgspan, Func> filtercommand) => DoRunAsync(backendManager, db, null, filtercommand, pgoffset, pgspan); - private async Task DoRunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, IFilter filter, Func filtercommand, float pgoffset, float pgspan) + private async Task DoRunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, IFilter filter, Func> filtercommand, float pgoffset, float pgspan) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Begin); Logging.Log.WriteInformationMessage(LOGTAG, "StartingPurge", "Starting purge operation"); var doCompactStep = !m_options.NoAutoCompact && filtercommand == null; - if (db.PartiallyRecreated) + if (await db.PartiallyRecreated(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) throw new UserInformationException("The purge command does not work on partially recreated databases", "PurgeNotAllowedOnPartialDatabase"); - if (db.RepairInProgress && filtercommand == null) + if (await db.RepairInProgress(m_result.TaskControl.ProgressToken).ConfigureAwait(false) && filtercommand == null) throw new UserInformationException(string.Format("The purge command does not work on an incomplete database, try the {0} operation.", "purge-broken-files"), "PurgeNotAllowedOnIncompleteDatabase"); - var versions = db.GetFilesetIDs(m_options.Time, m_options.Version).OrderByDescending(x => x).ToArray(); + var versions = await db + .GetFilesetIDs(m_options.Time, m_options.Version, false, m_result.TaskControl.ProgressToken) + .OrderByDescending(x => x) + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); if (versions.Length <= 0) throw new UserInformationException("No filesets matched the supplied time or versions", "NoFilesetFoundForTimeOrVersion"); - var orphans = db.CountOrphanFiles(null); + var orphans = await db.CountOrphanFiles(m_result.TaskControl.ProgressToken).ConfigureAwait(false); if (orphans != 0) throw new UserInformationException(string.Format("Unable to start the purge process as there are {0} orphan file(s)", orphans), "CannotPurgeWithOrphans"); - Utility.UpdateOptionsFromDb(db, m_options); - Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); if (filtercommand == null) { - db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, false, null); + await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, false, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); if (m_options.NoBackendverification) - await FilelistProcessor.VerifyLocalList(backendManager, db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await FilelistProcessor.VerifyLocalList(backendManager, db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); else - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, null, m_result.BackendWriter, null, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, null, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } - var filesets = db.FilesetTimes.OrderByDescending(x => x.Value).ToArray(); + var filesets = await db + .FilesetTimes(m_result.TaskControl.ProgressToken) + .OrderByDescending(x => x.Value) + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); var versionprogress = ((doCompactStep ? 0.75f : 1.0f) / versions.Length) * pgspan; var currentprogress = pgoffset; @@ -106,7 +118,7 @@ namespace Duplicati.Library.Main.Operation // If we crash now, it is possible that the remote storage contains partial files if (!m_options.Dryrun) - db.TerminatedWithActiveUploads = true; + await db.TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, true).ConfigureAwait(false); // Reverse makes sure we re-write the old versions first foreach (var versionid in versions.Reverse()) @@ -114,94 +126,120 @@ namespace Duplicati.Library.Main.Operation progress++; Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingFilelistVolumes", "Processing filelist volume {0} of {1}", progress, versions.Length); - using (var tr = db.BeginTransaction()) + (var _, var tsOriginal, var ix) = filesets.Select((x, i) => (x.Key, x.Value, i)).FirstOrDefault(x => x.Key == versionid); + if (ix < 0 || tsOriginal.Ticks == 0) + throw new InvalidProgramException(string.Format("Fileset was reported with id {0}, but could not be found?", versionid)); + + var ts = await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, tsOriginal, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + var prevfilename = await db.GetRemoteVolumeNameForFileset(filesets[ix].Key, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + if (ix != 0 && filesets[ix - 1].Value <= ts) + throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is larger than the next timestamp {2}", prevfilename, ts, filesets[ix - 1].Value)); + + await using (var tempset = await db.CreateTemporaryFileset(versionid, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { - (var _, var tsOriginal, var ix) = filesets.Select((x, i) => (x.Key, x.Value, i)).FirstOrDefault(x => x.Key == versionid); - if (ix < 0 || tsOriginal.Ticks == 0) - throw new InvalidProgramException(string.Format("Fileset was reported with id {0}, but could not be found?", versionid)); + if (filtercommand == null) + await tempset.ApplyFilter(filter, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + else + await tempset.ApplyFilter(filtercommand, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); - var ts = FilesetVolumeWriter.ProbeUnusedFilenameName(db, tr, m_options, tsOriginal); - var prevfilename = db.GetRemoteVolumeNameForFileset(filesets[ix].Key, tr); - - if (ix != 0 && filesets[ix - 1].Value <= ts) - throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is larger than the next timestamp {2}", prevfilename, ts, filesets[ix - 1].Value)); - - using (var tempset = db.CreateTemporaryFileset(versionid, tr)) + if (tempset.RemovedFileCount + tempset.UpdatedFileCount == 0) { - if (filtercommand == null) - tempset.ApplyFilter(filter); - else - tempset.ApplyFilter(filtercommand); + Logging.Log.WriteInformationMessage(LOGTAG, "NotWritingNewFileset", "Not writing a new fileset for {0} as it was not changed", prevfilename); + currentprogress += versionprogress; + await db.Transaction + .RollBackAsync() + .ConfigureAwait(false); + continue; + } + else + { + using (var tf = new TempFile()) + using (var vol = new FilesetVolumeWriter(m_options, ts)) + { + var isOriginalFilesetFullBackup = await db + .IsFilesetFullBackup(tsOriginal, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + var newids = await tempset + .ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + vol.VolumeID = newids.Item1; + vol.CreateFilesetFile(isOriginalFilesetFullBackup); - if (tempset.RemovedFileCount + tempset.UpdatedFileCount == 0) - { - Logging.Log.WriteInformationMessage(LOGTAG, "NotWritingNewFileset", "Not writing a new fileset for {0} as it was not changed", prevfilename); - currentprogress += versionprogress; - tr.Rollback(); - continue; - } - else - { - using (var tf = new TempFile()) - using (var vol = new FilesetVolumeWriter(m_options, ts)) + Logging.Log.WriteInformationMessage(LOGTAG, "ReplacingFileset", "Replacing fileset {0} with {1} which has with {2} fewer file(s) ({3} reduction)", prevfilename, vol.RemoteFilename, tempset.RemovedFileCount, Library.Utility.Utility.FormatSizeString(tempset.RemovedFileSize)); + + await db + .WriteFileset(vol, newids.Item2, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + m_result.RemovedFileSize += tempset.RemovedFileSize; + m_result.RemovedFileCount += tempset.RemovedFileCount; + m_result.UpdatedFileCount += tempset.UpdatedFileCount; + m_result.RewrittenFileLists++; + + currentprogress += (versionprogress / 2); + m_result.OperationProgressUpdater.UpdateProgress(currentprogress); + + if (m_options.Dryrun || m_options.FullResult) { - var isOriginalFilesetFullBackup = db.IsFilesetFullBackup(tsOriginal, tr); - var newids = tempset.ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup); - vol.VolumeID = newids.Item1; - vol.CreateFilesetFile(isOriginalFilesetFullBackup); - - Logging.Log.WriteInformationMessage(LOGTAG, "ReplacingFileset", "Replacing fileset {0} with {1} which has with {2} fewer file(s) ({3} reduction)", prevfilename, vol.RemoteFilename, tempset.RemovedFileCount, Library.Utility.Utility.FormatSizeString(tempset.RemovedFileSize)); - - db.WriteFileset(vol, newids.Item2, tr); - - m_result.RemovedFileSize += tempset.RemovedFileSize; - m_result.RemovedFileCount += tempset.RemovedFileCount; - m_result.UpdatedFileCount += tempset.UpdatedFileCount; - m_result.RewrittenFileLists++; - - currentprogress += (versionprogress / 2); - m_result.OperationProgressUpdater.UpdateProgress(currentprogress); - - if (m_options.Dryrun || m_options.FullResult) + await foreach (var fe in tempset.ListAllDeletedFiles(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { - foreach (var fe in tempset.ListAllDeletedFiles()) - { - var msg = string.Format(" Purging file {0} ({1})", fe.Key, Library.Utility.Utility.FormatSizeString(fe.Value)); + var msg = string.Format(" Purging file {0} ({1})", fe.Key, Library.Utility.Utility.FormatSizeString(fe.Value)); - Logging.Log.WriteProfilingMessage(LOGTAG, "PurgeFile", msg); - Logging.Log.WriteVerboseMessage(LOGTAG, "PurgeFile", msg); - - if (m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPurgeFile", msg); - } + Logging.Log.WriteProfilingMessage(LOGTAG, "PurgeFile", msg); + Logging.Log.WriteVerboseMessage(LOGTAG, "PurgeFile", msg); if (m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldWriteRemoteFiles", "Would write files to remote storage"); - - Logging.Log.WriteVerboseMessage(LOGTAG, "WritingRemoteFiles", "Writing files to remote storage"); + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPurgeFile", msg); } if (m_options.Dryrun) - { - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadAndDelete", "Would upload file {0} ({1}) and delete file {2}, removing {3} files", vol.RemoteFilename, Library.Utility.Utility.FormatSizeString(vol.Filesize), prevfilename, tempset.RemovedFileCount); - tr.Rollback(); - } - else - { - db.UpdateRemoteVolume(vol.RemoteFilename, RemoteVolumeState.Uploading, -1, null, tr); - var lst = db.DropFilesetsFromTable(new[] { tsOriginal }, tr).ToArray(); - foreach (var f in lst) - db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, tr); + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldWriteRemoteFiles", "Would write files to remote storage"); - tr.Commit(); + Logging.Log.WriteVerboseMessage(LOGTAG, "WritingRemoteFiles", "Writing files to remote storage"); + } - await backendManager.PutAsync(vol, null, null, true, async () => - { - await backendManager.FlushPendingMessagesAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - }, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - await backendManager.DeleteAsync(prevfilename, -1, true, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - await backendManager.WaitForEmptyAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - } + if (m_options.Dryrun) + { + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadAndDelete", "Would upload file {0} ({1}) and delete file {2}, removing {3} files", vol.RemoteFilename, Library.Utility.Utility.FormatSizeString(vol.Filesize), prevfilename, tempset.RemovedFileCount); + await db.Transaction + .RollBackAsync() + .ConfigureAwait(false); + } + else + { + await db + .UpdateRemoteVolume(vol.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + var lst = await db + .DropFilesetsFromTable(new[] { tsOriginal }, m_result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + foreach (var f in lst) + await db + .UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + await db.Transaction + .CommitAsync(m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + await backendManager.PutAsync(vol, null, null, true, async () => + { + await backendManager.FlushPendingMessagesAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + // This commit mimics the pre- + // Microsoft.Data.Sqlite backend way of not + // passing down a transaction, but maybe is + // not needed anymore. + await db.Transaction + .CommitAsync(m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + }, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.DeleteAsync(prevfilename, -1, true, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } } } @@ -212,7 +250,9 @@ namespace Duplicati.Library.Main.Operation } if (!m_options.Dryrun) - db.TerminatedWithActiveUploads = false; + await db + .TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, false) + .ConfigureAwait(false); if (doCompactStep) { @@ -225,21 +265,18 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(pgoffset + (0.75f * pgspan)); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Compact); m_result.CompactResults = new CompactResults(m_result); - using (var cdb = new Database.LocalDeleteDatabase(db)) - using (var ctr = new Database.ReusableTransaction(cdb)) - { - await new CompactHandler(m_options, (CompactResults)m_result.CompactResults) - .DoCompactAsync(cdb, true, ctr, backendManager) - .ConfigureAwait(false); + await using var cdb = await Database.LocalDeleteDatabase.CreateAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await new CompactHandler(m_options, (CompactResults)m_result.CompactResults) + .DoCompactAsync(cdb, true, backendManager) + .ConfigureAwait(false); - ctr.Commit("PostCompact", restart: false); - } + await cdb.Transaction.CommitAsync("PostCompact", true, m_result.TaskControl.ProgressToken); } m_result.OperationProgressUpdater.UpdateProgress(pgoffset + pgspan); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Complete); - await backendManager.WaitForEmptyAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index e410d7deb..f6f9945c1 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -1,33 +1,35 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; -using System.Data; +//using System.Data; using System.IO; using System.Linq; +using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; using Duplicati.Library.Main.Volumes; using Duplicati.Library.Utility; +using Microsoft.Data.Sqlite; namespace Duplicati.Library.Main.Operation { @@ -51,523 +53,641 @@ namespace Duplicati.Library.Main.Operation } /// - /// Run the recreate procedure + /// Run the recreate procedure. /// - /// Path to the database that will be created - /// The backend manager to use for downloading files - /// A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist - /// Filters the files in a filelist to prevent downloading unwanted data - /// A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists + /// Path to the database that will be created. + /// The backend manager to use for downloading files. + /// Filters the files in a filelist to prevent downloading unwanted data. + /// A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist. + /// A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists. public async Task RunAsync(string path, IBackendManager backendManager, IFilter filter, NumberedFilterFilelistDelegate filelistfilter, BlockVolumePostProcessor blockprocessor) { if (File.Exists(path)) throw new UserInformationException(string.Format("Cannot recreate database because file already exists: {0}", path), "RecreateTargetDatabaseExists"); - using (var db = new LocalDatabase(path, "Recreate", true, m_options.SqlitePageCache)) - { - await DoRunAsync(backendManager, db, false, filter, filelistfilter, blockprocessor).ConfigureAwait(false); + await using var db = + await LocalDatabase.CreateLocalDatabaseAsync(path, "Recreate", true, null, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); - // Ensure database is consistent after the recreate - db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null); - } + await DoRunAsync(backendManager, db, false, filter, filelistfilter, blockprocessor).ConfigureAwait(false); + + // Ensure database is consistent after the recreate + await db + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); } /// - /// Updates a database with new path information from a remote fileset + /// Updates a database with new path information from a remote fileset. /// - /// A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist - /// Filters the files in a filelist to prevent downloading unwanted data - /// A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists + /// The backend manager to use for downloading files. + /// Filters the files in a filelist to prevent downloading unwanted data. + /// A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist. + /// A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists. public async Task RunUpdateAsync(IBackendManager backendManager, Library.Utility.IFilter filter, NumberedFilterFilelistDelegate filelistfilter, BlockVolumePostProcessor blockprocessor) { if (!m_options.RepairOnlyPaths) throw new UserInformationException(string.Format("Can only update with paths, try setting --{0}", "repair-only-paths"), "RepairUpdateRequiresPathsOnly"); - using (var db = new LocalDatabase(m_options.Dbpath, "Recreate", true, m_options.SqlitePageCache)) + await using var db = + await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Recreate", true, null, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + if ((await db.FindMatchingFilesets(m_options.Time, m_options.Version, m_result.TaskControl.ProgressToken).ConfigureAwait(false)).Any()) { - if (db.FindMatchingFilesets(m_options.Time, m_options.Version).Any()) + if (m_options.IgnoreUpdateIfVersionExists) { - if (m_options.IgnoreUpdateIfVersionExists) - { - Logging.Log.WriteInformationMessage(LOGTAG, "UpdateVersionAlreadyExists", "The version(s) being updated to already exists, ignoring update request"); - return; - } - - throw new UserInformationException("The version(s) being updated to, already exists", "UpdateVersionAlreadyExists"); + Logging.Log.WriteInformationMessage(LOGTAG, "UpdateVersionAlreadyExists", "The version(s) being updated to already exists, ignoring update request"); + return; } - // Mark as incomplete - db.PartiallyRecreated = true; - - var preexistingOptionsInDatabase = Utility.ContainsOptionsForVerification(db); - Utility.UpdateOptionsFromDb(db, m_options, null); - - // Make sure the options have not changed between calls, unless there are no previous options - if (preexistingOptionsInDatabase) - Utility.VerifyOptionsAndUpdateDatabase(db, m_options, null); - - await DoRunAsync(backendManager, db, true, filter, filelistfilter, blockprocessor).ConfigureAwait(false); + throw new UserInformationException("The version(s) being updated to, already exists", "UpdateVersionAlreadyExists"); } + + // Mark as incomplete + await db.PartiallyRecreated(m_result.TaskControl.ProgressToken, true).ConfigureAwait(false); + + var preexistingOptionsInDatabase = + await Utility.ContainsOptionsForVerification(db, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + // Make sure the options have not changed between calls, unless there are no previous options + if (preexistingOptionsInDatabase) + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + await DoRunAsync(backendManager, db, true, filter, filelistfilter, blockprocessor).ConfigureAwait(false); } /// - /// Run the recreate procedure + /// Run the recreate procedure. /// - /// The database to restore into - /// True if this is an update call, false otherwise - /// A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist - /// Filters the files in a filelist to prevent downloading unwanted data - /// A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists + /// The backend manager to use for downloading files. + /// The database to restore into. + /// True if this is an update call, false otherwise. + /// A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist. + /// Filters the files in a filelist to prevent downloading unwanted data. + /// A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists. + /// A task that completes when the operation is done. internal async Task DoRunAsync(IBackendManager backendManager, LocalDatabase dbparent, bool updating, IFilter filter = null, NumberedFilterFilelistDelegate filelistfilter = null, BlockVolumePostProcessor blockprocessor = null) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Recreate_Running); //We build a local database in steps. - using (var restoredb = new LocalRecreateDatabase(dbparent, m_options)) + await using var restoredb = + await LocalRecreateDatabase.CreateAsync(dbparent, m_options, null, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + await restoredb.RepairInProgress(m_result.TaskControl.ProgressToken, true).ConfigureAwait(false); + var expRecreateDb = false; // experimental recreate db code flag + var volumeIds = new Dictionary(); + + if (string.Equals(Environment.GetEnvironmentVariable("EXPERIMENTAL_RECREATEDB_DUPLICATI") ?? string.Empty, "1")) { - restoredb.RepairInProgress = true; - var expRecreateDb = false; // experimental recreate db code flag - var volumeIds = new Dictionary(); + expRecreateDb = true; + } - if (string.Equals(Environment.GetEnvironmentVariable("EXPERIMENTAL_RECREATEDB_DUPLICATI") ?? string.Empty, "1")) + var rawlist = await backendManager.ListAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false); + + //First step is to examine the remote storage to see what + // kind of data we can find + var remotefiles = + (from x in rawlist + let n = VolumeBase.ParseFilename(x) + where + n != null + && + n.Prefix == m_options.Prefix + select n).ToArray(); //ToArray() ensures that we do not remote-request it multiple times + + if (remotefiles.Length == 0) + { + if (!rawlist.Any()) + throw new UserInformationException("No files were found at the remote location, perhaps the target url is incorrect?", "EmptyRemoteLocation"); + else { - expRecreateDb = true; - } - - var rawlist = await backendManager.ListAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false); - - //First step is to examine the remote storage to see what - // kind of data we can find - var remotefiles = + var tmp = (from x in rawlist let n = VolumeBase.ParseFilename(x) where n != null - && - n.Prefix == m_options.Prefix - select n).ToArray(); //ToArray() ensures that we do not remote-request it multiple times + select n.Prefix).ToArray(); - if (remotefiles.Length == 0) + var types = tmp.Distinct().ToArray(); + if (tmp.Length == 0) + throw new UserInformationException(string.Format("Found {0} files at the remote storage, but none that could be parsed", rawlist.Count()), "EmptyRemoteLocation"); + else if (types.Length == 1) + throw new UserInformationException(string.Format("Found {0} parse-able files with the prefix {1}, did you forget to set the backup prefix?", tmp.Length, types[0]), "EmptyRemoteLocationWithPrefix"); + else + throw new UserInformationException(string.Format("Found {0} parse-able files (of {1} files) with different prefixes: {2}, did you forget to set the backup prefix?", tmp.Length, rawlist.Count(), string.Join(", ", types)), "EmptyRemoteLocationWithPrefix"); + } + } + + if (string.IsNullOrWhiteSpace(m_options.Passphrase) && remotefiles.Any(x => !string.IsNullOrWhiteSpace(x.EncryptionModule))) + throw new UserInformationException("The remote files are encrypted, but no passphrase was provided", "MissingPassphrase"); + + //Then we select the filelist we should work with, + // and create the filelist table to fit + IEnumerable filelists = + from n in remotefiles + where n.FileType == RemoteVolumeType.Files + orderby n.Time descending + select n; + + if (!filelists.Any()) + throw new UserInformationException("No filelists found on the remote destination", "EmptyRemoteLocation"); + + if (filelistfilter != null) + filelists = filelistfilter(filelists).Select(x => x.Value).ToArray(); + + if (!filelists.Any()) + throw new UserInformationException("No filelists", "NoMatchingRemoteFilelists"); + + // If we are updating, all files should be accounted for + foreach (var fl in remotefiles) + volumeIds[fl.File.Name] = updating + ? await restoredb.GetRemoteVolumeID(fl.File.Name, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false) + : await restoredb.RegisterRemoteVolume(fl.File.Name, fl.FileType, fl.File.Size, RemoteVolumeState.Uploaded, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + var hasUpdatedOptions = false; + + // Record all blocksets and files needed + var filelistWork = (from n in filelists orderby n.Time select new RemoteVolume(n.File) as IRemoteVolume).ToList(); + Logging.Log.WriteInformationMessage(LOGTAG, "RebuildStarted", "Rebuild database started, downloading {0} filelists", filelistWork.Count); + + var progress = 0; + + // Register the files we are working with, if not already updated + if (updating) + { + foreach (var n in filelists) + if (volumeIds[n.File.Name] == -1) + volumeIds[n.File.Name] = await restoredb + .RegisterRemoteVolume(n.File.Name, n.FileType, RemoteVolumeState.Uploaded, n.File.Size, new TimeSpan(0), m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + } + + var isFirstFilelist = true; + await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(filelistWork, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + { + var entry = new RemoteVolume(name, hash, size); + try { - if (rawlist.Count() == 0) - throw new UserInformationException("No files were found at the remote location, perhaps the target url is incorrect?", "EmptyRemoteLocation"); + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + { + await backendManager.WaitForEmptyAsync(restoredb, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + m_result.EndTime = DateTime.UtcNow; + // Implicit rollback + await restoredb.Transaction + .RollBackAsync(m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + return; + } + + progress++; + if (filelistWork.Count == 1 && m_options.RepairOnlyPaths) + { + m_result.OperationProgressUpdater.UpdateProgress(0.5f); + } else { - var tmp = - (from x in rawlist - let n = VolumeBase.ParseFilename(x) - where - n != null - select n.Prefix).ToArray(); + m_result.OperationProgressUpdater.UpdateProgress(((float)progress / filelistWork.Count()) * (m_options.RepairOnlyPaths ? 1f : 0.2f)); + Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingFilelistVolumes", "Processing filelist volume {0} of {1}", progress, filelistWork.Count); + } - var types = tmp.Distinct().ToArray(); - if (tmp.Length == 0) - throw new UserInformationException(string.Format("Found {0} files at the remote storage, but none that could be parsed", rawlist.Count()), "EmptyRemoteLocation"); - else if (types.Length == 1) - throw new UserInformationException(string.Format("Found {0} parse-able files with the prefix {1}, did you forget to set the backup prefix?", tmp.Length, types[0]), "EmptyRemoteLocationWithPrefix"); - else - throw new UserInformationException(string.Format("Found {0} parse-able files (of {1} files) with different prefixes: {2}, did you forget to set the backup prefix?", tmp.Length, rawlist.Count(), string.Join(", ", types)), "EmptyRemoteLocationWithPrefix"); + using (tmpfile) + { + isFirstFilelist = false; + + if (!string.IsNullOrWhiteSpace(hash) && size > 0) + await restoredb + .UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, size, hash, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + var parsed = VolumeBase.ParseFilename(entry.Name); + + using var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read); + using var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions); + if (compressor == null) + throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); + + if (!hasUpdatedOptions) + { + VolumeReaderBase.UpdateOptionsFromManifest(compressor, m_options); + hasUpdatedOptions = true; + } + + // Create timestamped operations based on the file timestamp + var filesetid = await restoredb + .CreateFileset(volumeIds[entry.Name], parsed.Time, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + await RecreateFilesetFromRemoteList(restoredb, compressor, filesetid, m_options, filter, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); } } - - if (string.IsNullOrWhiteSpace(m_options.Passphrase) && remotefiles.Any(x => !string.IsNullOrWhiteSpace(x.EncryptionModule))) - throw new UserInformationException("The remote files are encrypted, but no passphrase was provided", "MissingPassphrase"); - - //Then we select the filelist we should work with, - // and create the filelist table to fit - IEnumerable filelists = - from n in remotefiles - where n.FileType == RemoteVolumeType.Files - orderby n.Time descending - select n; - - if (!filelists.Any()) - throw new UserInformationException("No filelists found on the remote destination", "EmptyRemoteLocation"); - - if (filelistfilter != null) - filelists = filelistfilter(filelists).Select(x => x.Value).ToArray(); - - if (!filelists.Any()) - throw new UserInformationException("No filelists", "NoMatchingRemoteFilelists"); - - // If we are updating, all files should be accounted for - foreach (var fl in remotefiles) - volumeIds[fl.File.Name] = updating ? restoredb.GetRemoteVolumeID(fl.File.Name) : restoredb.RegisterRemoteVolume(fl.File.Name, fl.FileType, fl.File.Size, RemoteVolumeState.Uploaded); - - var hasUpdatedOptions = false; - - //Record all blocksets and files needed - using (var tr = restoredb.BeginTransaction()) + catch (Exception ex) { - var filelistWork = (from n in filelists orderby n.Time select new RemoteVolume(n.File) as IRemoteVolume).ToList(); - Logging.Log.WriteInformationMessage(LOGTAG, "RebuildStarted", "Rebuild database started, downloading {0} filelists", filelistWork.Count); - - var progress = 0; - - // Register the files we are working with, if not already updated - if (updating) + Logging.Log.WriteWarningMessage(LOGTAG, "FileProcessingFailed", ex, "Failed to process file: {0}", entry.Name); + if (ex.IsAbortException()) { - foreach (var n in filelists) - if (volumeIds[n.File.Name] == -1) - volumeIds[n.File.Name] = restoredb.RegisterRemoteVolume(n.File.Name, n.FileType, RemoteVolumeState.Uploaded, n.File.Size, new TimeSpan(0), tr); + m_result.EndTime = DateTime.UtcNow; + // Implicit rollback + await restoredb.Transaction + .RollBackAsync(m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + throw; } - var isFirstFilelist = true; - await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(filelistWork, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + if (isFirstFilelist && ex is System.Security.Cryptography.CryptographicException) + { + m_result.EndTime = DateTime.UtcNow; + // Implicit rollback + await restoredb.Transaction + .RollBackAsync(m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + throw; + } + + if (m_options.UnittestMode) + { + // Implicit rollback + await restoredb.Transaction + .RollBackAsync(m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + throw; + } + } + } + + //Make sure we write the config if it has been read from a manifest + if (hasUpdatedOptions) + await Utility.VerifyOptionsAndUpdateDatabase(restoredb, m_options, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + using (new Logging.Timer(LOGTAG, "CommitUpdateFilesetFromRemote", "CommitUpdateFilesetFromRemote")) + await restoredb.Transaction + .CommitAsync(m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + // do we stop after just handling the dlist files ? + // (if yes, we never will be able to do a backup !) + if (!m_options.RepairOnlyPaths) + { + var hashsize = 0; + //Grab all index files, and update the block table + + using (var hashalg = HashFactory.CreateHasher(m_options.BlockHashAlgorithm)) + { + hashsize = hashalg.HashSize / 8; + + var indexfiles = ( + from n in remotefiles + where n.FileType == RemoteVolumeType.Index + select new RemoteVolume(n.File) as IRemoteVolume).ToList(); + + Logging.Log.WriteInformationMessage(LOGTAG, "FilelistsRestored", "Filelists restored, downloading {0} index files", indexfiles.Count); + + progress = 0; + + await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(indexfiles, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { - var entry = new RemoteVolume(name, hash, size); try { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(restoredb, tr, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(restoredb, m_result.TaskControl.ProgressToken).ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; + // Implicit rollback + await restoredb.Transaction + .RollBackAsync(m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); return; } progress++; - if (filelistWork.Count == 1 && m_options.RepairOnlyPaths) - { - m_result.OperationProgressUpdater.UpdateProgress(0.5f); - } - else - { - m_result.OperationProgressUpdater.UpdateProgress(((float)progress / filelistWork.Count()) * (m_options.RepairOnlyPaths ? 1f : 0.2f)); - Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingFilelistVolumes", "Processing filelist volume {0} of {1}", progress, filelistWork.Count); - } + m_result.OperationProgressUpdater.UpdateProgress((((float)progress / indexfiles.Count) * 0.5f) + 0.2f); + Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingIndexlistVolumes", "Processing indexlist volume {0} of {1}", progress, indexfiles.Count); using (tmpfile) { - isFirstFilelist = false; - if (!string.IsNullOrWhiteSpace(hash) && size > 0) - restoredb.UpdateRemoteVolume(entry.Name, RemoteVolumeState.Verified, size, hash, tr); + await restoredb + .UpdateRemoteVolume(name, RemoteVolumeState.Verified, size, hash, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); - var parsed = VolumeBase.ParseFilename(entry.Name); - - using var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read); - using var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions); - if (compressor == null) - throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); - - if (!hasUpdatedOptions) + using (var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options, hashsize)) { - VolumeReaderBase.UpdateOptionsFromManifest(compressor, m_options); - hasUpdatedOptions = true; + foreach (var a in svr.Volumes) + { + var filename = a.Filename; + var volumeID = await restoredb + .GetRemoteVolumeID(filename, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + // No such file + if (volumeID < 0) + (volumeID, filename) = + await ProbeForMatchingFilename(filename, restoredb, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + var missing = false; + // Still broken, register a missing item + if (volumeID < 0) + { + var p = VolumeBase.ParseFilename(filename); + if (p == null) + throw new Exception(string.Format("Unable to parse filename: {0}", filename)); + Logging.Log.WriteWarningMessage(LOGTAG, "MissingFileDetected", null, "Remote file referenced as {0} by {1}, but not found in list, registering a missing remote file", filename, name); + missing = true; + volumeID = await restoredb + .RegisterRemoteVolume(filename, p.FileType, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + } + + bool anyChange = false; + //Add all block/volume mappings + foreach (var b in a.Blocks) + { + anyChange |= (await restoredb.UpdateBlock(b.Key, b.Value, volumeID, m_result.TaskControl.ProgressToken).ConfigureAwait(false)).Item1; + } + + await restoredb + .UpdateRemoteVolume(filename, missing ? RemoteVolumeState.Temporary : RemoteVolumeState.Verified, a.Length, a.Hash, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + await restoredb + .AddIndexBlockLink( + await restoredb.GetRemoteVolumeID(name, m_result.TaskControl.ProgressToken).ConfigureAwait(false), + volumeID, + m_result.TaskControl.ProgressToken + ) + .ConfigureAwait(false); + } + + //If there are blocklists in the index file, add them to the temp blocklist hashes table + int wrongHashes = 0; + foreach (var b in svr.BlockLists) + { + // Compact might have created undetected invalid blocklist entries in index files due to broken LocalDatabase.GetBlocklists + // If the hash is wrong, recreate will download the dblock volume with the correct file + try + { + // We need to instantiate the list to ensure the verification is + // done before we add it to the database, since we do not have nested transactions + var list = b.Blocklist.ToList(); + await restoredb + .AddTempBlockListHash(b.Hash, list, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + } + catch (System.IO.InvalidDataException e) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "InvalidDataBlocklist", e, "Exception while processing blocklists in {0}", name); + ++wrongHashes; + } + } + if (wrongHashes != 0) + { + Logging.Log.WriteWarningMessage(LOGTAG, "WrongBlocklistHashes", null, "{0} had invalid blocklists which could not be used. Consider deleting this index file and run repair to recreate it.", name); + } } - - // Create timestamped operations based on the file timestamp - var filesetid = restoredb.CreateFileset(volumeIds[entry.Name], parsed.Time, tr); - - RecreateFilesetFromRemoteList(restoredb, tr, compressor, filesetid, m_options, filter); } } catch (Exception ex) { - Logging.Log.WriteWarningMessage(LOGTAG, "FileProcessingFailed", ex, "Failed to process file: {0}", entry.Name); + //Not fatal + Logging.Log.WriteErrorMessage(LOGTAG, "IndexFileProcessingFailed", ex, "Failed to process index file: {0}", name); if (ex.IsAbortException()) { m_result.EndTime = DateTime.UtcNow; - throw; - } - - if (isFirstFilelist && ex is System.Security.Cryptography.CryptographicException) - { - m_result.EndTime = DateTime.UtcNow; + // Implicit rollback + await restoredb.Transaction + .RollBackAsync(m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); throw; } if (m_options.UnittestMode) + { + // Implicit rollback + await restoredb.Transaction + .RollBackAsync(m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); throw; - + } } } - //Make sure we write the config if it has been read from a manifest - if (hasUpdatedOptions) - Utility.VerifyOptionsAndUpdateDatabase(restoredb, m_options, tr); + using (new Logging.Timer(LOGTAG, "CommitRecreateDb", "CommitRecreatedDb")) + await restoredb.Transaction + .CommitAsync(m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); - using (new Logging.Timer(LOGTAG, "CommitUpdateFilesetFromRemote", "CommitUpdateFilesetFromRemote")) - tr.Commit(); + // TODO: In some cases, we can avoid downloading all index files, + // if we are lucky and pick the right ones } - // do we stop after just handling the dlist files ? - // (if yes, we never will be able to do a backup !) - if (!m_options.RepairOnlyPaths) + await restoredb.CleanupMissingVolumes(m_result.TaskControl.ProgressToken).ConfigureAwait(false); + + // Update the real tables from the temp tables + if (expRecreateDb) + // add missing blocks and blocksetentry data (at this point + // we have not yet anything in the blocksetentry table) + await restoredb + .AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, false, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + else + await restoredb + .FindMissingBlocklistHashes(hashsize, m_options.Blocksize, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + // We have now grabbed as much information as possible, + // if we are still missing data, we must now fetch block files + //We do this in three passes + for (var i = 0; i < 3; i++) { - var hashsize = 0; - //Grab all index files, and update the block table + // Grab the list matching the pass type + var lst = await restoredb + .GetMissingBlockListVolumes(i, m_options.Blocksize, hashsize, m_options.RepairForceBlockUse, m_result.TaskControl.ProgressToken) + .ToListAsync() + .ConfigureAwait(false); - using (var hashalg = HashFactory.CreateHasher(m_options.BlockHashAlgorithm)) - using (var tr = restoredb.BeginTransaction()) + if (lst.Count == 0) + continue; + + var fullist = ": " + string.Join(", ", lst.Select(x => x.Name)); + switch (i) { - hashsize = hashalg.HashSize / 8; + case 0: + Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingRequiredBlocklistVolumes", "Processing required {0} blocklist volumes{1}", lst.Count, fullist); + Logging.Log.WriteInformationMessage(LOGTAG, "ProcessingRequiredBlocklistVolumes", "Processing required {0} blocklist volumes{1}", lst.Count, m_options.FullResult ? fullist : string.Empty); + break; + case 1: + Logging.Log.WriteVerboseMessage(LOGTAG, "ProbingCandicateBlocklistVolumes", "Probing {0} candidate blocklist volumes{1}", lst.Count, fullist); + Logging.Log.WriteInformationMessage(LOGTAG, "ProbingCandicateBlocklistVolumes", "Probing {0} candidate blocklist volumes{1}", lst.Count, m_options.FullResult ? fullist : string.Empty); + break; + default: + Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingAllBlocklistVolumes", "Processing all of the {0} volumes for blocklists{1}", lst.Count, fullist); + Logging.Log.WriteInformationMessage(LOGTAG, "ProcessingAllBlocklistVolumes", "Processing all of the {0} volumes for blocklists{1}", lst.Count, m_options.FullResult ? fullist : string.Empty); + break; + } - var indexfiles = ( - from n in remotefiles - where n.FileType == RemoteVolumeType.Index - select new RemoteVolume(n.File) as IRemoteVolume).ToList(); - - Logging.Log.WriteInformationMessage(LOGTAG, "FilelistsRestored", "Filelists restored, downloading {0} index files", indexfiles.Count); - - var progress = 0; - - await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(indexfiles, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + progress = 0; + await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(lst, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + { + try { - try + using (tmpfile) + using (var rd = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options)) { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(restoredb, tr, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(restoredb, m_result.TaskControl.ProgressToken).ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; + // Implicit rollback + await restoredb.Transaction + .RollBackAsync(m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); return; } progress++; - m_result.OperationProgressUpdater.UpdateProgress((((float)progress / indexfiles.Count) * 0.5f) + 0.2f); - Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingIndexlistVolumes", "Processing indexlist volume {0} of {1}", progress, indexfiles.Count); + m_result.OperationProgressUpdater.UpdateProgress((((float)progress / lst.Count) * 0.1f) + 0.7f + (i * 0.1f)); + Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingBlocklistVolumes", "Pass {0} of 3, processing blocklist volume {1} of {2}", (i + 1), progress, lst.Count); - using (tmpfile) + var volumeid = await restoredb + .GetRemoteVolumeID(name, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + await restoredb + .UpdateRemoteVolume(name, RemoteVolumeState.Uploaded, size, hash, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + bool anyChange = false; + // Update the block table so we know about the block/volume map + foreach (var h in rd.Blocks) { - if (!string.IsNullOrWhiteSpace(hash) && size > 0) - restoredb.UpdateRemoteVolume(name, RemoteVolumeState.Verified, size, hash, tr); + anyChange |= (await restoredb.UpdateBlock(h.Key, h.Value, volumeid, m_result.TaskControl.ProgressToken).ConfigureAwait(false)).Item1; + } - using (var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options, hashsize)) + // now that we have the blocks/volume relationships, we can go from the (already known from dlist step) blocklisthashes + // to the needed list blocks in the volume, so grab them from the database + // read the blocks list hashes from the volume data (the handled file) and insert them into the temp blocklisthash table + await foreach (var blocklisthash in restoredb.GetBlockLists(volumeid, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + { + if (await restoredb.AddTempBlockListHash(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize), m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { - foreach (var a in svr.Volumes) - { - var filename = a.Filename; - var volumeID = restoredb.GetRemoteVolumeID(filename); - - // No such file - if (volumeID < 0) - volumeID = ProbeForMatchingFilename(ref filename, restoredb); - - var missing = false; - // Still broken, register a missing item - if (volumeID < 0) - { - var p = VolumeBase.ParseFilename(filename); - if (p == null) - throw new Exception(string.Format("Unable to parse filename: {0}", filename)); - Logging.Log.WriteWarningMessage(LOGTAG, "MissingFileDetected", null, "Remote file referenced as {0} by {1}, but not found in list, registering a missing remote file", filename, name); - missing = true; - volumeID = restoredb.RegisterRemoteVolume(filename, p.FileType, RemoteVolumeState.Temporary, tr); - } - - bool anyChange = false; - //Add all block/volume mappings - foreach (var b in a.Blocks) - restoredb.UpdateBlock(b.Key, b.Value, volumeID, tr, ref anyChange); - - restoredb.UpdateRemoteVolume(filename, missing ? RemoteVolumeState.Temporary : RemoteVolumeState.Verified, a.Length, a.Hash, tr); - restoredb.AddIndexBlockLink(restoredb.GetRemoteVolumeID(name), volumeID, tr); - } - - //If there are blocklists in the index file, add them to the temp blocklist hashes table - int wrongHashes = 0; - foreach (var b in svr.BlockLists) - { - // Compact might have created undetected invalid blocklist entries in index files due to broken LocalDatabase.GetBlocklists - // If the hash is wrong, recreate will download the dblock volume with the correct file - try - { - // We need to instantiate the list to ensure the verification is - // done before we add it to the database, since we do not have nested transactions - var list = b.Blocklist.ToList(); - restoredb.AddTempBlockListHash(b.Hash, list, tr); - } - catch (System.IO.InvalidDataException e) - { - Logging.Log.WriteVerboseMessage(LOGTAG, "InvalidDataBlocklist", e, "Exception while processing blocklists in {0}", name); - ++wrongHashes; - } - } - if (wrongHashes != 0) - { - Logging.Log.WriteWarningMessage(LOGTAG, "WrongBlocklistHashes", null, "{0} had invalid blocklists which could not be used. Consider deleting this index file and run repair to recreate it.", name); - } + anyChange = true; } } - } - catch (Exception ex) - { - //Not fatal - Logging.Log.WriteErrorMessage(LOGTAG, "IndexFileProcessingFailed", ex, "Failed to process index file: {0}", name); - if (ex.IsAbortException()) + + // Update tables if necessary (if no block or hash have been changed by a data volume + // there is no need to run expensive queries - most data volumes have been + // managed successfully by correct index volumes), so we know if we are done + // if any change, add to the block and blocksetentry tables the references found in + // the block lists of the volume saved in the temp blocklisthash table by AddTempBLockListHash + if (anyChange) { - m_result.EndTime = DateTime.UtcNow; - throw; + if (i == 2) + { + Logging.Log.WriteWarningMessage(LOGTAG, "UpdatingTables", null, "Unexpected changes caused by block {0}", name); + } + if (expRecreateDb) + await restoredb + .AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, false, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + else + await restoredb + .FindMissingBlocklistHashes(hashsize, m_options.Blocksize, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); } - if (m_options.UnittestMode) - throw; + using (new Logging.Timer(LOGTAG, "CommitRestoredBlocklist", "CommitRestoredBlocklist")) + await restoredb.Transaction + .CommitAsync(m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + //At this point we can patch files with data from the block volume + if (blockprocessor != null) + blockprocessor(name, rd); } } - - using (new Logging.Timer(LOGTAG, "CommitRecreateDb", "CommitRecreatedDb")) - tr.Commit(); - - // TODO: In some cases, we can avoid downloading all index files, - // if we are lucky and pick the right ones - } - - restoredb.CleanupMissingVolumes(); - - // Update the real tables from the temp tables - if (expRecreateDb) - // add missing blocks and blocksetentry data (at this point - // we have not yet anything in the blocksetentry table) - restoredb.AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, null); - else - restoredb.FindMissingBlocklistHashes(hashsize, m_options.Blocksize, null); - - // We have now grabbed as much information as possible, - // if we are still missing data, we must now fetch block files - //We do this in three passes - for (var i = 0; i < 3; i++) - { - // Grab the list matching the pass type - var lst = restoredb.GetMissingBlockListVolumes(i, m_options.Blocksize, hashsize, m_options.RepairForceBlockUse).ToList(); - if (lst.Count == 0) - continue; - - var fullist = ": " + string.Join(", ", lst.Select(x => x.Name)); - switch (i) + catch (Exception ex) { - case 0: - Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingRequiredBlocklistVolumes", "Processing required {0} blocklist volumes{1}", lst.Count, fullist); - Logging.Log.WriteInformationMessage(LOGTAG, "ProcessingRequiredBlocklistVolumes", "Processing required {0} blocklist volumes{1}", lst.Count, m_options.FullResult ? fullist : string.Empty); - break; - case 1: - Logging.Log.WriteVerboseMessage(LOGTAG, "ProbingCandicateBlocklistVolumes", "Probing {0} candidate blocklist volumes{1}", lst.Count, fullist); - Logging.Log.WriteInformationMessage(LOGTAG, "ProbingCandicateBlocklistVolumes", "Probing {0} candidate blocklist volumes{1}", lst.Count, m_options.FullResult ? fullist : string.Empty); - break; - default: - Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingAllBlocklistVolumes", "Processing all of the {0} volumes for blocklists{1}", lst.Count, fullist); - Logging.Log.WriteInformationMessage(LOGTAG, "ProcessingAllBlocklistVolumes", "Processing all of the {0} volumes for blocklists{1}", lst.Count, m_options.FullResult ? fullist : string.Empty); - break; - } - - var progress = 0; - await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(lst, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) - { - try + Logging.Log.WriteWarningMessage(LOGTAG, "FailedRebuildingWithFile", ex, "Failed to use information from {0} to rebuild database: {1}", name, ex.Message); + if (m_options.UnittestMode) { - using (tmpfile) - using (var rd = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options)) - using (var tr = restoredb.BeginTransaction()) - { - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - { - await backendManager.WaitForEmptyAsync(restoredb, tr, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - m_result.EndTime = DateTime.UtcNow; - return; - } - - progress++; - m_result.OperationProgressUpdater.UpdateProgress((((float)progress / lst.Count) * 0.1f) + 0.7f + (i * 0.1f)); - Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingBlocklistVolumes", "Pass {0} of 3, processing blocklist volume {1} of {2}", (i + 1), progress, lst.Count); - - var volumeid = restoredb.GetRemoteVolumeID(name); - - restoredb.UpdateRemoteVolume(name, RemoteVolumeState.Uploaded, size, hash, tr); - - bool anyChange = false; - // Update the block table so we know about the block/volume map - foreach (var h in rd.Blocks) - restoredb.UpdateBlock(h.Key, h.Value, volumeid, tr, ref anyChange); - - // now that we have the blocks/volume relationships, we can go from the (already known from dlist step) blocklisthashes - // to the needed list blocks in the volume, so grab them from the database - // read the blocks list hashes from the volume data (the handled file) and insert them into the temp blocklisthash table - foreach (var blocklisthash in restoredb.GetBlockLists(volumeid)) - { - if (restoredb.AddTempBlockListHash(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize), tr)) - { - anyChange = true; - } - } - - // Update tables if necessary (if no block or hash have been changed by a data volume - // there is no need to run expensive queries - most data volumes have been - // managed successfully by correct index volumes), so we know if we are done - // if any change, add to the block and blocksetentry tables the references found in - // the block lists of the volume saved in the temp blocklisthash table by AddTempBLockListHash - if (anyChange) - { - if (i == 2) - { - Logging.Log.WriteWarningMessage(LOGTAG, "UpdatingTables", null, "Unexpected changes caused by block {0}", name); - } - if (expRecreateDb) - restoredb.AddBlockAndBlockSetEntryFromTemp(hashsize, m_options.Blocksize, tr, false); - else - restoredb.FindMissingBlocklistHashes(hashsize, m_options.Blocksize, tr); - } - - using (new Logging.Timer(LOGTAG, "CommitRestoredBlocklist", "CommitRestoredBlocklist")) - tr.Commit(); - - //At this point we can patch files with data from the block volume - if (blockprocessor != null) - blockprocessor(name, rd); - } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "FailedRebuildingWithFile", ex, "Failed to use information from {0} to rebuild database: {1}", name, ex.Message); - if (m_options.UnittestMode) - throw; + // Implicit rollback + await restoredb.Transaction + .RollBackAsync(m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + throw; } } } } - - await backendManager.WaitForEmptyAsync(restoredb, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - - if (!m_options.RepairOnlyPaths) - { - // All blocks are collected and added into the Block table - // Find out which blocks are deleted and move them into DeletedBlock, - // so that compact can calculate the unused space - restoredb.CleanupDeletedBlocks(null); - } - - restoredb.CleanupMissingVolumes(); - - if (m_options.RepairOnlyPaths) - { - Logging.Log.WriteInformationMessage(LOGTAG, "RecreateOrUpdateOnly", "Recreate/path-update completed, not running consistency checks"); - } - else - { - Logging.Log.WriteInformationMessage(LOGTAG, "RecreateCompletedCheckingDatabase", "Recreate completed, verifying the database consistency"); - - //All done, we must verify that we have all blocklist fully intact - // if this fails, the db will not be deleted, so it can be used, - // except to continue a backup - m_result.EndTime = DateTime.UtcNow; - - using (var lbfdb = new LocalListBrokenFilesDatabase(restoredb)) - { - var broken = lbfdb.GetBrokenFilesets(new DateTime(0), null, null).Count(); - if (broken != 0) - throw new UserInformationException(string.Format("Recreated database has missing blocks and {0} broken filelists. Consider using \"{1}\" and \"{2}\" to purge broken data from the remote store and the database.", broken, "list-broken-files", "purge-broken-files"), "DatabaseIsBrokenConsiderPurge"); - } - - restoredb.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null); - - Logging.Log.WriteInformationMessage(LOGTAG, "RecreateCompleted", "Recreate completed, and consistency checks completed, marking database as complete"); - - restoredb.RepairInProgress = false; - } - - m_result.EndTime = DateTime.UtcNow; } + + await backendManager.WaitForEmptyAsync(restoredb, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + + if (!m_options.RepairOnlyPaths) + { + // All blocks are collected and added into the Block table + // Find out which blocks are deleted and move them into DeletedBlock, + // so that compact can calculate the unused space + await restoredb.CleanupDeletedBlocks(m_result.TaskControl.ProgressToken).ConfigureAwait(false); + } + + await restoredb.CleanupMissingVolumes(m_result.TaskControl.ProgressToken).ConfigureAwait(false); + + if (m_options.RepairOnlyPaths) + { + Logging.Log.WriteInformationMessage(LOGTAG, "RecreateOrUpdateOnly", "Recreate/path-update completed, not running consistency checks"); + } + else + { + Logging.Log.WriteInformationMessage(LOGTAG, "RecreateCompletedCheckingDatabase", "Recreate completed, verifying the database consistency"); + + //All done, we must verify that we have all blocklist fully intact + // if this fails, the db will not be deleted, so it can be used, + // except to continue a backup + m_result.EndTime = DateTime.UtcNow; + + await using (var lbfdb = await LocalListBrokenFilesDatabase.CreateAsync(restoredb, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + { + var broken = await lbfdb + .GetBrokenFilesets(new DateTime(0), null, m_result.TaskControl.ProgressToken) + .CountAsync(cancellationToken: m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + if (broken != 0) + throw new UserInformationException(string.Format("Recreated database has missing blocks and {0} broken filelists. Consider using \"{1}\" and \"{2}\" to purge broken data from the remote store and the database.", broken, "list-broken-files", "purge-broken-files"), "DatabaseIsBrokenConsiderPurge"); + } + + await restoredb + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + Logging.Log.WriteInformationMessage(LOGTAG, "RecreateCompleted", "Recreate completed, and consistency checks completed, marking database as complete"); + + await restoredb.RepairInProgress(m_result.TaskControl.ProgressToken, false).ConfigureAwait(false); + } + + m_result.EndTime = DateTime.UtcNow; } - public static void RecreateFilesetFromRemoteList(LocalRecreateDatabase restoredb, IDbTransaction transaction, ICompression compressor, long filesetid, Options options, IFilter filter) + /// + /// Recreate a fileset from the remote filelist. + /// + /// The database to restore to. + /// The compression module to use. + /// The ID of the fileset to recreate. + /// The options to use for the operation. + /// The filter to apply to the files. + /// The cancellation token to monitor for cancellation requests. + /// A task that completes when the fileset has been recreated. + public static async Task RecreateFilesetFromRemoteList(LocalRecreateDatabase restoredb, ICompression compressor, long filesetid, Options options, IFilter filter, CancellationToken cancellationToken) { var blocksize = options.Blocksize; var hashes_pr_block = blocksize / options.BlockhashSize; @@ -576,10 +696,12 @@ namespace Duplicati.Library.Main.Operation var filesetData = VolumeReaderBase.GetFilesetData(compressor, options); // update fileset using filesetData - restoredb.UpdateFullBackupStateInFileset(filesetid, filesetData.IsFullBackup, transaction); + await restoredb + .UpdateFullBackupStateInFileset(filesetid, filesetData.IsFullBackup, cancellationToken) + .ConfigureAwait(false); // clear any existing fileset entries - restoredb.ClearFilesetEntries(filesetid, transaction); + await restoredb.ClearFilesetEntries(filesetid, cancellationToken).ConfigureAwait(false); using (var filelistreader = new FilesetVolumeReader(compressor, options)) foreach (var fe in filelistreader.Files.Where(x => Library.Utility.FilterExpression.Matches(filter, x.Path))) @@ -592,37 +714,58 @@ namespace Duplicati.Library.Main.Operation var metadataid = long.MinValue; var split = LocalDatabase.SplitIntoPrefixAndName(fe.Path); - var prefixid = restoredb.GetOrCreatePathPrefix(split.Key, transaction); + var prefixid = await restoredb + .GetOrCreatePathPrefix(split.Key, cancellationToken) + .ConfigureAwait(false); switch (fe.Type) { case FilelistEntryType.Folder: - metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, transaction); - restoredb.AddDirectoryEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, transaction); + metadataid = await restoredb + .AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, cancellationToken) + .ConfigureAwait(false); + await restoredb + .AddDirectoryEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, cancellationToken) + .ConfigureAwait(false); + break; case FilelistEntryType.File: var expectedblocks = (fe.Size + blocksize - 1) / blocksize; var expectedblocklisthashes = (expectedblocks + hashes_pr_block - 1) / hashes_pr_block; if (expectedblocks <= 1) expectedblocklisthashes = 0; - var blocksetid = restoredb.AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes, transaction); - metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, transaction); - restoredb.AddFileEntry(filesetid, prefixid, split.Value, fe.Time, blocksetid, metadataid, transaction); + var blocksetid = await restoredb + .AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes, cancellationToken) + .ConfigureAwait(false); + metadataid = await restoredb + .AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, cancellationToken) + .ConfigureAwait(false); + await restoredb + .AddFileEntry(filesetid, prefixid, split.Value, fe.Time, blocksetid, metadataid, cancellationToken) + .ConfigureAwait(false); if (fe.Size <= blocksize) { if (!string.IsNullOrWhiteSpace(fe.Blockhash)) - restoredb.AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize, transaction); + await restoredb + .AddSmallBlocksetLink(fe.Hash, fe.Blockhash, fe.Blocksize, cancellationToken) + .ConfigureAwait(false); else if (options.BlockHashAlgorithm == options.FileHashAlgorithm) - restoredb.AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size, transaction); + await restoredb + .AddSmallBlocksetLink(fe.Hash, fe.Hash, fe.Size, cancellationToken) + .ConfigureAwait(false); else if (fe.Size > 0) Logging.Log.WriteWarningMessage(LOGTAG, "MissingBlockHash", null, "No block hash found for file: {0}", fe.Path); } break; case FilelistEntryType.Symlink: - metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, transaction); - restoredb.AddSymlinkEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, transaction); + metadataid = await restoredb + .AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, cancellationToken) + .ConfigureAwait(false); + await restoredb + .AddSymlinkEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, cancellationToken) + .ConfigureAwait(false); break; default: Logging.Log.WriteWarningMessage(LOGTAG, "SkippingUnknownFileEntry", null, "Skipping file-entry with unknown type {0}: {1} ", fe.Type, fe.Path); @@ -632,9 +775,13 @@ namespace Duplicati.Library.Main.Operation if (fe.Metasize <= blocksize && (fe.Type == FilelistEntryType.Folder || fe.Type == FilelistEntryType.File || fe.Type == FilelistEntryType.Symlink)) { if (!string.IsNullOrWhiteSpace(fe.Metablockhash)) - restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize, transaction); + await restoredb + .AddSmallBlocksetLink(fe.Metahash, fe.Metablockhash, fe.Metasize, cancellationToken) + .ConfigureAwait(false); else if (options.BlockHashAlgorithm == options.FileHashAlgorithm) - restoredb.AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize, transaction); + await restoredb + .AddSmallBlocksetLink(fe.Metahash, fe.Metahash, fe.Metasize, cancellationToken) + .ConfigureAwait(false); else Logging.Log.WriteWarningMessage(LOGTAG, "MissingMetadataBlockHash", null, "No block hash found for file metadata: {0}", fe.Path); } @@ -644,34 +791,40 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteWarningMessage(LOGTAG, "FileEntryProcessingFailed", ex, "Failed to process file-entry: {0}", fe.Path); } } + + await restoredb.Transaction.CommitAsync(cancellationToken).ConfigureAwait(false); } /// - /// Look in the database for filenames similar to the current filename, but with a different compression and encryption module + /// Look in the database for filenames similar to the current filename, but with a different compression and encryption module. /// - /// The volume id of the item - /// The filename read and written - /// The database to query - public long ProbeForMatchingFilename(ref string filename, LocalRestoreDatabase restoredb) + /// The filename read and written. + /// The database to query. + /// A cancellation token to cancel the operation. + /// The volume id of the item. + public static async Task<(long, string)> ProbeForMatchingFilename(string filename, LocalRecreateDatabase restoredb, CancellationToken cancellationToken) { var p = VolumeBase.ParseFilename(filename); if (p != null) { foreach (var compmodule in Library.DynamicLoader.CompressionLoader.Keys) - foreach (var encmodule in Library.DynamicLoader.EncryptionLoader.Keys.Union(new string[] { "" })) + foreach (var encmodule in Library.DynamicLoader.EncryptionLoader.Keys.Union([""])) { var testfilename = VolumeBase.GenerateFilename(p.FileType, p.Prefix, p.Guid, p.Time, compmodule, encmodule); - var tvid = restoredb.GetRemoteVolumeID(testfilename); + var tvid = await restoredb + .GetRemoteVolumeID(testfilename, cancellationToken) + .ConfigureAwait(false); + if (tvid >= 0) { Logging.Log.WriteWarningMessage(LOGTAG, "RewritingFilenameMapping", null, "Unable to find volume {0}, but mapping to matching file {1}", filename, testfilename); filename = testfilename; - return tvid; + return (tvid, filename); } } } - return -1; + return (-1, filename); } public void Dispose() diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 1e36d7e94..803436c0e 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -1,23 +1,24 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. + using System; using System.Collections.Generic; using System.Data; @@ -56,7 +57,7 @@ namespace Duplicati.Library.Main.Operation if (!File.Exists(m_options.Dbpath)) { await RunRepairLocalAsync(backendManager, filter).ConfigureAwait(false); - RunRepairCommon(); + await RunRepairCommon().ConfigureAwait(false); m_result.EndTime = DateTime.UtcNow; return; } @@ -64,8 +65,13 @@ namespace Duplicati.Library.Main.Operation long knownRemotes = -1; try { - using (var db = new LocalRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache)) - knownRemotes = db.GetRemoteVolumes().Count(); + await using var db = + await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, null, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + knownRemotes = await db + .GetRemoteVolumes(m_result.TaskControl.ProgressToken) + .CountAsync(cancellationToken: m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); } catch (Exception ex) { @@ -90,11 +96,11 @@ namespace Duplicati.Library.Main.Operation } await RunRepairLocalAsync(backendManager, filter).ConfigureAwait(false); - RunRepairCommon(); + await RunRepairCommon().ConfigureAwait(false); } else { - RunRepairCommon(); + await RunRepairCommon().ConfigureAwait(false); await RunRepairBrokenFilesets(backendManager).ConfigureAwait(false); await RunRepairRemoteAsync(backendManager, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } @@ -127,42 +133,72 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdateProgress(0); - using (var db = new LocalRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache)) - using (var rtr = new ReusableTransaction(db)) - { - Utility.UpdateOptionsFromDb(db, m_options); - Utility.VerifyOptionsAndUpdateDatabase(db, m_options); + await using var db = + await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, null, cancellationToken) + .ConfigureAwait(false); - if (db.PartiallyRecreated) + try + { + + await Utility.UpdateOptionsFromDb(db, m_options, cancellationToken) + .ConfigureAwait(false); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, cancellationToken) + .ConfigureAwait(false); + + if (await db.PartiallyRecreated(cancellationToken).ConfigureAwait(false)) throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated"); - if (db.RepairInProgress) + if (await db.RepairInProgress(cancellationToken).ConfigureAwait(false)) throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsInRepairState"); // Ensure the database is consistent before we start fixing the remote - db.VerifyConsistencyForRepair(m_options.Blocksize, m_options.BlockhashSize, true, rtr.Transaction); + await db + .VerifyConsistencyForRepair(m_options.Blocksize, m_options.BlockhashSize, true, cancellationToken) + .ConfigureAwait(false); // If the last backup failed, guard the incomplete fileset, so we can create a synthetic filelist - var lastTempFilelist = db.GetLastIncompleteFilesetVolume(rtr.Transaction); - var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, rtr.Transaction, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced).ConfigureAwait(false); - rtr.Commit("CommitRemoteListAnalysisTransaction"); + var lastTempFilelist = await db + .GetLastIncompleteFilesetVolume(cancellationToken) + .ConfigureAwait(false); + var tp = await FilelistProcessor.RemoteListAnalysis(backendManager, m_options, db, m_result.BackendWriter, [lastTempFilelist.Name], null, FilelistProcessor.VerifyMode.VerifyAndCleanForced, cancellationToken).ConfigureAwait(false); + await db.Transaction + .CommitAsync("CommitRemoteListAnalysisTransaction", token: cancellationToken) + .ConfigureAwait(false); var buffer = new byte[m_options.Blocksize]; var hashsize = m_options.BlockhashSize; - var missingRemoteFilesets = db.MissingRemoteFilesets().ToList(); - var missingLocalFilesets = db.MissingLocalFilesets().ToList(); - var emptyIndexFiles = db.EmptyIndexFiles().ToList(); + var missingRemoteFilesets = await db + .MissingRemoteFilesets(cancellationToken) + .ToListAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); + var missingLocalFilesets = await db + .MissingLocalFilesets(cancellationToken) + .ToListAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); + var emptyIndexFiles = await db + .EmptyIndexFiles(cancellationToken) + .ToListAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); var progress = 0; - var targetProgess = tp.ExtraVolumes.Count() + tp.MissingVolumes.Count() + tp.VerificationRequiredVolumes.Count() + missingRemoteFilesets.Count + missingLocalFilesets.Count + emptyIndexFiles.Count; + var targetProgess = + tp.ExtraVolumes.Count() + + tp.MissingVolumes.Count() + + tp.VerificationRequiredVolumes.Count() + + missingRemoteFilesets.Count + + missingLocalFilesets.Count + + emptyIndexFiles.Count; // Find the most recent timestamp from either a fileset or a remote volume - var mostRecentLocal = db.GetRemoteVolumes(rtr.Transaction) + var mostRecentLocal = await db + .GetRemoteVolumes(cancellationToken) .Where(x => x.Type == RemoteVolumeType.Files) .Select(x => VolumeBase.ParseFilename(x.Name).Time.ToLocalTime()) - .Concat(db.FilesetTimes.Select(x => x.Value.ToLocalTime())) - .Append(DateTime.MinValue).Max(); + .Concat(db.FilesetTimes(cancellationToken).Select(x => x.Value.ToLocalTime())) + .Append(DateTime.MinValue) + .MaxAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); var mostRecentRemote = tp.ParsedVolumes.Select(x => x.Time.ToLocalTime()).Append(DateTime.MinValue).Max(); if (mostRecentLocal < DateTime.UnixEpoch) @@ -194,46 +230,59 @@ namespace Duplicati.Library.Main.Operation { if (tp.VerificationRequiredVolumes.Any()) { - using (var testdb = new LocalTestDatabase(db)) - { - foreach (var n in tp.VerificationRequiredVolumes) - try + await using var testdb = + await LocalTestDatabase.CreateAsync(db, null, cancellationToken) + .ConfigureAwait(false); + + foreach (var n in tp.VerificationRequiredVolumes) + try + { + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - { - await backendManager.WaitForEmptyAsync(testdb, rtr.Transaction, cancellationToken).ConfigureAwait(false); - if (!m_options.Dryrun) - rtr.Commit("CommitEarlyExit", false); - return; - } - - progress++; - m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); - - KeyValuePair>> res; - (var tf, var hash, var size) = await backendManager.GetWithInfoAsync(n.Name, n.Hash, n.Size, cancellationToken).ConfigureAwait(false); - using (tf) - res = TestHandler.TestVolumeInternals(testdb, rtr, n, tf, m_options, 1); - - if (res.Value.Any()) - throw new Exception(string.Format("Remote verification failure: {0}", res.Value.First())); - + await backendManager.WaitForEmptyAsync(testdb, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - { - Logging.Log.WriteInformationMessage(LOGTAG, "CapturedRemoteFileHash", "Sucessfully captured hash for {0}, updating database", n.Name); - db.UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash); - } - + await testdb.Transaction + .CommitAsync("CommitEarlyExit", false, cancellationToken) + .ConfigureAwait(false); + else + await testdb.Transaction + .RollBackAsync(cancellationToken) + .ConfigureAwait(false); + return; } - catch (Exception ex) + + progress++; + m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); + + KeyValuePair>> res; + var (tf, hash, size) = await backendManager.GetWithInfoAsync(n.Name, n.Hash, n.Size, cancellationToken).ConfigureAwait(false); + using (tf) + res = + await TestHandler.TestVolumeInternals(testdb, n, tf, m_options, 1, cancellationToken) + .ConfigureAwait(false); + + if (res.Value.Any()) + throw new Exception(string.Format("Remote verification failure: {0}", res.Value.First())); + + if (!m_options.Dryrun) { - Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileVerificationError", ex, "Failed to perform verification for file: {0}, please run verify; message: {1}", n.Name, ex.Message); - if (ex.IsAbortException()) - throw; + Logging.Log.WriteInformationMessage(LOGTAG, "CapturedRemoteFileHash", "Sucessfully captured hash for {0}, updating database", n.Name); + await db + .UpdateRemoteVolume(n.Name, RemoteVolumeState.Verified, size, hash, cancellationToken) + .ConfigureAwait(false); } - rtr.Commit("CommitVerificationTransaction"); - } + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileVerificationError", ex, "Failed to perform verification for file: {0}, please run verify; message: {1}", n.Name, ex.Message); + if (ex.IsAbortException()) + throw; + } + + await db.Transaction + .CommitAsync("CommitVerificationTransaction", true, cancellationToken) + .ConfigureAwait(false); } // TODO: It is actually possible to use the extra files if we parse them @@ -242,9 +291,15 @@ namespace Duplicati.Library.Main.Operation { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - rtr.Commit("CommitEarlyExit", false); + await db.Transaction + .CommitAsync("CommitEarlyExit", false, cancellationToken) + .ConfigureAwait(false); + else + await db.Transaction + .RollBackAsync(cancellationToken) + .ConfigureAwait(false); return; } @@ -263,7 +318,9 @@ namespace Duplicati.Library.Main.Operation { foreach (var rv in ifr.Volumes) { - var entry = db.GetRemoteVolume(rv.Filename); + var entry = await db + .GetRemoteVolume(rv.Filename, cancellationToken) + .ConfigureAwait(false); if (entry.ID < 0) throw new Exception(string.Format("Unknown remote file {0} detected", rv.Filename)); @@ -273,31 +330,47 @@ namespace Duplicati.Library.Main.Operation if (entry.Hash != rv.Hash || entry.Size != rv.Length || !new[] { RemoteVolumeState.Uploading, RemoteVolumeState.Uploaded, RemoteVolumeState.Verified }.Contains(entry.State)) throw new Exception(string.Format("Volume {0} hash/size mismatch ({1} - {2}) vs ({3} - {4})", rv.Filename, entry.Hash, entry.Size, rv.Hash, rv.Length)); - db.CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks, rtr.Transaction); + await db + .CheckAllBlocksAreInVolume(rv.Filename, rv.Blocks, cancellationToken) + .ConfigureAwait(false); } var blocksize = m_options.Blocksize; foreach (var ixb in ifr.BlockLists) - db.CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize, rtr.Transaction); + await db + .CheckBlocklistCorrect(ixb.Hash, ixb.Length, ixb.Blocklist, blocksize, hashsize, cancellationToken) + .ConfigureAwait(false); // Register the new index file and link it to the block files - var selfid = db.RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, RemoteVolumeState.Uploading, size, new TimeSpan(0), rtr.Transaction); + var selfid = await db + .RegisterRemoteVolume(n.File.Name, RemoteVolumeType.Index, RemoteVolumeState.Uploading, size, new TimeSpan(0), cancellationToken) + .ConfigureAwait(false); foreach (var rv in ifr.Volumes) { // Guard against unknown block files - long id = db.GetRemoteVolumeID(rv.Filename, rtr.Transaction); + long id = await db + .GetRemoteVolumeID(rv.Filename, cancellationToken) + .ConfigureAwait(false); if (id == -1) Logging.Log.WriteWarningMessage(LOGTAG, "UnknownBlockFile", null, "Index file {0} references unknown block file: {1}", n.File.Name, rv.Filename); else - db.AddIndexBlockLink(selfid, id, rtr.Transaction); + await db + .AddIndexBlockLink(selfid, id, cancellationToken) + .ConfigureAwait(false); } + if (!m_options.Dryrun) - rtr.Commit("CommitIndexFileTransaction"); + await db.Transaction + .CommitAsync("CommitIndexFileTransaction", true, cancellationToken) + .ConfigureAwait(false); } // All checks fine, we accept the new index file Logging.Log.WriteInformationMessage(LOGTAG, "AcceptNewIndexFile", "Accepting new index file {0}", n.File.Name); - db.UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash, rtr.Transaction); + await db + .UpdateRemoteVolume(n.File.Name, RemoteVolumeState.Verified, size, hash, cancellationToken) + .ConfigureAwait(false); + continue; } catch (Exception rex) @@ -308,7 +381,9 @@ namespace Duplicati.Library.Main.Operation if (!m_options.Dryrun) { - db.RegisterRemoteVolume(n.File.Name, n.FileType, n.File.Size, RemoteVolumeState.Deleting); + await db + .RegisterRemoteVolume(n.File.Name, n.FileType, n.File.Size, RemoteVolumeState.Deleting, cancellationToken) + .ConfigureAwait(false); await backendManager.DeleteAsync(n.File.Name, n.File.Size, false, cancellationToken).ConfigureAwait(false); } else @@ -330,15 +405,23 @@ namespace Duplicati.Library.Main.Operation { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - rtr.Commit("CommitEarlyExit", false); + await db.Transaction + .CommitAsync("CommitEarlyExit", false, cancellationToken) + .ConfigureAwait(false); + else + await db.Transaction + .RollBackAsync(cancellationToken) + .ConfigureAwait(false); return; } progress++; m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); - var fileTime = FilesetVolumeWriter.ProbeUnusedFilenameName(db, rtr.Transaction, m_options, timestamp); + var fileTime = + await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, timestamp, cancellationToken) + .ConfigureAwait(false); var fsw = new FilesetVolumeWriter(m_options, fileTime); Logging.Log.WriteInformationMessage(LOGTAG, "ReuploadingFileset", "Re-uploading fileset {0} from {1} as remote volume registration is missing, new filename: {2}", filesetId, timestamp, fsw.RemoteFilename); @@ -348,12 +431,20 @@ namespace Duplicati.Library.Main.Operation fsw.AddControlFile(p, m_options.GetCompressionHintFromFilename(p)); fsw.CreateFilesetFile(isfull); - db.WriteFileset(fsw, filesetId, null); + await db + .WriteFileset(fsw, filesetId, cancellationToken) + .ConfigureAwait(false); fsw.Close(); - fsw.VolumeID = db.RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, -1, RemoteVolumeState.Temporary); - db.LinkFilesetToVolume(filesetId, fsw.VolumeID, null); + fsw.VolumeID = await db + .RegisterRemoteVolume(fsw.RemoteFilename, RemoteVolumeType.Files, -1, RemoteVolumeState.Temporary, cancellationToken) + .ConfigureAwait(false); + + await db + .LinkFilesetToVolume(filesetId, fsw.VolumeID, cancellationToken) + .ConfigureAwait(false); + if (m_options.Dryrun) { fsw.Dispose(); @@ -365,35 +456,44 @@ namespace Duplicati.Library.Main.Operation if (anyDlistUploads) { - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); - await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); + await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - rtr.Commit("CommitFilesetTransaction"); + await db.Transaction + .CommitAsync("CommitFilesetTransaction", token: cancellationToken) + .ConfigureAwait(false); } foreach (var volumename in missingLocalFilesets) { - var remoteVolume = db.GetRemoteVolume(volumename); - using (var tmpfile = await backendManager.GetAsync(remoteVolume.Name, remoteVolume.Hash, remoteVolume.Size, cancellationToken).ConfigureAwait(false)) - { - var parsed = VolumeBase.ParseFilename(remoteVolume.Name); - using (var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read)) - using (var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions)) - using (var recreatedb = new LocalRecreateDatabase(db, m_options)) - { - if (compressor == null) - throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); + var remoteVolume = await db + .GetRemoteVolume(volumename, cancellationToken) + .ConfigureAwait(false); + using var tmpfile = await backendManager.GetAsync(remoteVolume.Name, remoteVolume.Hash, remoteVolume.Size, cancellationToken).ConfigureAwait(false); + var parsed = VolumeBase.ParseFilename(remoteVolume.Name); + using var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read); + using var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions); + await using var recreatedb = await LocalRecreateDatabase.CreateAsync(db, m_options, null, cancellationToken).ConfigureAwait(false); + if (compressor == null) + throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); - var filesetid = db.CreateFileset(remoteVolume.ID, parsed.Time, rtr.Transaction); - RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, rtr.Transaction, compressor, filesetid, m_options, new FilterExpression()); - if (!m_options.Dryrun) - rtr.Commit("CommitRecreateFilesetTransaction"); - } - } + var filesetid = await db + .CreateFileset(remoteVolume.ID, parsed.Time, cancellationToken) + .ConfigureAwait(false); + + await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(recreatedb, compressor, filesetid, m_options, new FilterExpression(), cancellationToken) + .ConfigureAwait(false); + + if (!m_options.Dryrun) + await recreatedb.Transaction + .CommitAsync("CommitRecreateFilesetTransaction", token: cancellationToken) + .ConfigureAwait(false); } if (!m_options.Dryrun && tp.MissingVolumes.Any()) - db.TerminatedWithActiveUploads = true; + await db + .TerminatedWithActiveUploads(cancellationToken, true) + .ConfigureAwait(false); if (tp.MissingVolumes.Any(x => x.Type != RemoteVolumeType.Index && x.Type != RemoteVolumeType.Files && x.Type != RemoteVolumeType.Blocks)) throw new InvalidOperationException(string.Format("Unknown volume type {0} detected", tp.MissingVolumes.First(x => x.Type != RemoteVolumeType.Index && x.Type != RemoteVolumeType.Files && x.Type != RemoteVolumeType.Blocks).Type)); @@ -401,14 +501,14 @@ namespace Duplicati.Library.Main.Operation // Process each of the missing volumes in the order of blocks, files and index // It is important that we process the blocks first, as the index files are derived from the blocks - Action incrementProgress = () => + void incrementProgress() { progress++; m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess); - }; + } // Blocks are recreated with the entire list of missing files to handle cases where partial recreation is needed - await RunRepairDblocks(backendManager, db, rtr, missingDblocks, incrementProgress, cancellationToken).ConfigureAwait(false); + await RunRepairDblocks(backendManager, db, missingDblocks, incrementProgress, cancellationToken).ConfigureAwait(false); // Filesets are recreated one at a time foreach (var n in tp.MissingVolumes.Where(x => x.Type == RemoteVolumeType.Files)) @@ -419,10 +519,12 @@ namespace Duplicati.Library.Main.Operation try { var timestamp = VolumeBase.ParseFilename(n.Name).Time; - var fileTime = FilesetVolumeWriter.ProbeUnusedFilenameName(db, rtr.Transaction, m_options, timestamp); + var fileTime = + await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, timestamp, cancellationToken) + .ConfigureAwait(false); var volumeWriter = newEntry = new FilesetVolumeWriter(m_options, fileTime); - await RunRepairDlist(backendManager, db, rtr, volumeWriter, n, fileTime, cancellationToken).ConfigureAwait(false); + await RunRepairDlist(backendManager, db, volumeWriter, n, fileTime, cancellationToken).ConfigureAwait(false); } catch (Exception ex) { @@ -446,12 +548,12 @@ namespace Duplicati.Library.Main.Operation try { // Check if the index file has already been deleted, beause the dblock was recreated - var currentState = db.GetRemoteVolume(n.Name, rtr.Transaction).State; + var currentState = (await db.GetRemoteVolume(n.Name, cancellationToken).ConfigureAwait(false)).State; if (currentState == RemoteVolumeState.Deleted || currentState == RemoteVolumeState.Temporary || currentState == RemoteVolumeState.Deleting) continue; var w = newEntry = new IndexVolumeWriter(m_options); - await RunRepairDindex(backendManager, db, rtr, w, n, m_options, cancellationToken).ConfigureAwait(false); + await RunRepairDindex(backendManager, db, w, n, m_options, cancellationToken).ConfigureAwait(false); } catch (Exception ex) { @@ -472,7 +574,11 @@ namespace Duplicati.Library.Main.Operation { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); + // Implicit rollback + await db.Transaction + .RollBackAsync(cancellationToken) + .ConfigureAwait(false); return; } @@ -491,7 +597,15 @@ namespace Duplicati.Library.Main.Operation { Logging.Log.WriteInformationMessage(LOGTAG, "DeletingEmptyIndexFile", "Deleting empty index file {0}", emptyIndexFile.Name); await backendManager.DeleteAsync(emptyIndexFile.Name, emptyIndexFile.Size, false, cancellationToken).ConfigureAwait(false); - await backendManager.FlushPendingMessagesAsync(db, null, cancellationToken).ConfigureAwait(false); + await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); + // This commit mimics the pre + // Microsoft.Data.Sqlite backend way of not + // passing down a transaction to + // FlushPendingMessagesAsync, but maybe it + // is not needed anymore. + await db.Transaction + .CommitAsync(cancellationToken) + .ConfigureAwait(false); } } } @@ -510,94 +624,131 @@ namespace Duplicati.Library.Main.Operation } m_result.OperationProgressUpdater.UpdateProgress(1); - await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) { - db.TerminatedWithActiveUploads = false; - rtr.Commit("CommitRepairTransaction", false); + await db + .TerminatedWithActiveUploads(cancellationToken, false) + .ConfigureAwait(false); + await db.Transaction + .CommitAsync("CommitRepairTransaction", true, cancellationToken) + .ConfigureAwait(false); } + else + { + await db.Transaction + .RollBackAsync("RollbackRepairTransaction", true, cancellationToken) + .ConfigureAwait(false); + } + } + catch (Exception) + { + // Implicit rollback + await db.Transaction.RollBackAsync(cancellationToken).ConfigureAwait(false); + throw; } } /// - /// Repairs a single fileset by recreating it from the database content and uploading it + /// Repairs a single fileset by recreating it from the database content and uploading it. /// - /// The backend manager to use for uploading - /// The database to use for the repair - /// The transaction to use for the repair - /// The volume writer to use for the repair - /// The remote volume entry to repair - /// The time of the new fileset to create - /// Cancellation token to cancel the operation - /// A task representing the asynchronous operation - private async Task RunRepairDlist(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, FilesetVolumeWriter volumeWriter, RemoteVolumeEntry originalVolume, DateTime filesetTime, CancellationToken cancellationToken) + /// The backend manager to use for uploading. + /// The database to use for the repair. + /// The volume writer to use for the repair. + /// The remote volume entry to repair. + /// The time of the new fileset to create. + /// Cancellation token to cancel the operation. + /// A task representing the asynchronous operation. + private async Task RunRepairDlist(IBackendManager backendManager, LocalRepairDatabase db, FilesetVolumeWriter volumeWriter, RemoteVolumeEntry originalVolume, DateTime filesetTime, CancellationToken cancellationToken) { - volumeWriter.VolumeID = db.RegisterRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, rtr.Transaction); - (var prevFilesetId, var _, var isPrevFull) = db.GetFilesetFromRemotename(originalVolume.Name, rtr.Transaction); + volumeWriter.VolumeID = await db + .RegisterRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, cancellationToken) + .ConfigureAwait(false); + (var prevFilesetId, var _, var isPrevFull) = await db + .GetFilesetFromRemotename(originalVolume.Name, cancellationToken) + .ConfigureAwait(false); if (!string.IsNullOrEmpty(m_options.ControlFiles)) foreach (var p in m_options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) volumeWriter.AddControlFile(p, m_options.GetCompressionHintFromFilename(p)); volumeWriter.CreateFilesetFile(isPrevFull); - var newFilesetID = db.CreateFileset(volumeWriter.VolumeID, filesetTime, rtr.Transaction); - db.UpdateFullBackupStateInFileset(newFilesetID, isPrevFull); + var newFilesetID = await db + .CreateFileset(volumeWriter.VolumeID, filesetTime, cancellationToken) + .ConfigureAwait(false); + await db + .UpdateFullBackupStateInFileset(newFilesetID, isPrevFull, cancellationToken) + .ConfigureAwait(false); + await db + .LinkFilesetToVolume(newFilesetID, volumeWriter.VolumeID, cancellationToken) + .ConfigureAwait(false); + await db + .MoveFilesFromFileset(newFilesetID, prevFilesetId, cancellationToken) + .ConfigureAwait(false); + await db + .WriteFileset(volumeWriter, newFilesetID, cancellationToken) + .ConfigureAwait(false); - db.LinkFilesetToVolume(newFilesetID, volumeWriter.VolumeID, rtr.Transaction); - db.MoveFilesFromFileset(newFilesetID, prevFilesetId, rtr.Transaction); - - db.WriteFileset(volumeWriter, newFilesetID, rtr.Transaction); volumeWriter.Close(); - db.UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null, rtr.Transaction); - db.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); - await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await db + .UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null, cancellationToken) + .ConfigureAwait(false); + await db + .UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, cancellationToken) + .ConfigureAwait(false); + await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); if (m_options.Dryrun) Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would upload fileset {0}, with size {1}, previous size {2}", originalVolume.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(volumeWriter.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(originalVolume.Size)); else { - rtr.Commit("CommitPriorToFilesetUpload"); + await db.Transaction + .CommitAsync("CommitPriorToFilesetUpload", true, cancellationToken) + .ConfigureAwait(false); await backendManager.PutAsync(volumeWriter, null, null, false, null, cancellationToken).ConfigureAwait(false); } - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - rtr.Commit("CommitFilesetTransaction"); + await db.Transaction + .CommitAsync("CommitFilesetTransaction", true, cancellationToken) + .ConfigureAwait(false); } /// - /// Repairs a single index file by recreating it from the database content and uploading it + /// Repairs a single index file by recreating it from the database content and uploading it. /// - /// The backend manager to use for uploading - /// The database to use for the repair - /// The transaction to use for the repair - /// The volume writer to use for the repair - /// The remote volume entry to repair - /// The time of the new fileset to create - /// The options for the current operation - /// Cancellation token to cancel the operation - /// A task representing the asynchronous operation - - public static async Task RunRepairDindex(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, IndexVolumeWriter indexWriter, IRemoteVolume originalVolume, Options options, CancellationToken cancellationToken) + /// The backend manager to use for uploading. + /// The database to use for the repair. + /// The volume writer to use for the repair. + /// The remote volume entry to repair. + /// The options for the current operation. + /// Cancellation token to cancel the operation. + /// A task representing the asynchronous operation. + public static async Task RunRepairDindex(IBackendManager backendManager, LocalRepairDatabase db, IndexVolumeWriter indexWriter, IRemoteVolume originalVolume, Options options, CancellationToken cancellationToken) { - indexWriter.VolumeID = db.RegisterRemoteVolume(indexWriter.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Uploading, -1, TimeSpan.Zero, rtr.Transaction); + indexWriter.VolumeID = await db + .RegisterRemoteVolume(indexWriter.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Uploading, -1, TimeSpan.Zero, cancellationToken) + .ConfigureAwait(false); var blockvolumeids = new List(); using var h = HashFactory.CreateHasher(options.BlockHashAlgorithm); - foreach (var blockvolume in db.GetBlockVolumesFromIndexName(originalVolume.Name)) + await foreach (var blockvolume in db.GetBlockVolumesFromIndexName(originalVolume.Name, cancellationToken).ConfigureAwait(false)) { indexWriter.StartVolume(blockvolume.Name); - var volumeid = db.GetRemoteVolumeID(blockvolume.Name, rtr.Transaction); + var volumeid = await db + .GetRemoteVolumeID(blockvolume.Name, cancellationToken) + .ConfigureAwait(false); - foreach (var b in db.GetBlocks(volumeid)) + await foreach (var b in db.GetBlocks(volumeid, cancellationToken).ConfigureAwait(false)) indexWriter.AddBlock(b.Hash, b.Size); indexWriter.FinishVolume(blockvolume.Hash, blockvolume.Size); blockvolumeids.Add(volumeid); if (options.IndexfilePolicy == Options.IndexFileStrategy.Full) - foreach (var b in db.GetBlocklists(volumeid, options.Blocksize, options.BlockhashSize)) + await foreach (var b in db.GetBlocklists(volumeid, options.Blocksize, options.BlockhashSize, cancellationToken).ConfigureAwait(false)) { var bh = Convert.ToBase64String(h.ComputeHash(b.Buffer, 0, b.Size)); if (bh != b.Hash) @@ -608,7 +759,9 @@ namespace Duplicati.Library.Main.Operation } foreach (var blockvolumeid in blockvolumeids) - db.AddIndexBlockLink(indexWriter.VolumeID, blockvolumeid, rtr.Transaction); + await db + .AddIndexBlockLink(indexWriter.VolumeID, blockvolumeid, cancellationToken) + .ConfigureAwait(false); indexWriter.Close(); @@ -616,116 +769,135 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadIndexFile", "would re-upload index file {0}, with size {1}, previous size {2}", originalVolume.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(indexWriter.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(originalVolume.Size)); else { - db.UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); - await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await db + .UpdateRemoteVolume(originalVolume.Name, RemoteVolumeState.Deleted, originalVolume.Size, originalVolume.Hash, false, TimeSpan.FromHours(2), null, cancellationToken) + .ConfigureAwait(false); + await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); await backendManager.PutAsync(indexWriter, null, null, false, null, cancellationToken).ConfigureAwait(false); } - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!options.Dryrun) - rtr.Commit("CommitRepairTransaction"); + await db.Transaction + .CommitAsync("CommitRepairTransaction", true, cancellationToken) + .ConfigureAwait(false); } /// - /// Class for holding the state of a dblock volume that is being constructed + /// Class for holding the state of a dblock volume that is being constructed. /// private sealed class InProgressDblockVolumes { /// - /// Database to use for the operation + /// Database to use for the operation. /// private readonly LocalDatabase m_database; /// - /// Transaction to use for the operation - /// - private readonly ReusableTransaction m_transaction; - /// - /// List of volumes to delete after the upload is complete + /// List of volumes to delete after the upload is complete. /// private readonly List m_toDelete = new List(); /// - /// Options for the current operation + /// Options for the current operation. /// private readonly Options m_options; /// - /// Maximum size of the volume before it is considered full + /// Maximum size of the volume before it is considered full. /// private readonly long m_maxVolumeSize; /// - /// The writers that are completed and ready to be uploaded + /// The writers that are completed and ready to be uploaded. /// private readonly List m_completedWriters = new List(); /// - /// Writer for the current in-progress volume + /// Writer for the current in-progress volume. /// private BlockVolumeWriter m_activeWriter; /// - /// Flag indicating if any data has been added to the volume + /// Flag indicating if any data has been added to the volume. /// private bool m_anyData = false; /// - /// Constructor for the InProgressDblockVolume class + /// Constructor for the InProgressDblockVolume class. /// - /// The options for the current operation - /// The database to use for registering a volume in progress - /// The transaction to use for the operation - public InProgressDblockVolumes(Options options, LocalDatabase db, ReusableTransaction transaction) + /// The options for the current operation. + /// The database to use for registering a volume in progress. + private InProgressDblockVolumes(Options options, LocalDatabase db) { m_database = db; - m_transaction = transaction; m_options = options; m_maxVolumeSize = options.VolumeSize - m_options.Blocksize; m_activeWriter = new BlockVolumeWriter(options); - m_activeWriter.VolumeID = m_database.RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, m_transaction.Transaction); } /// - /// Checks if any data has been added to the current volume + /// Asynchronously creates a new instance of and registers the initial volume. + /// + /// The options for the current operation. + /// The database to use for registering the initial volume. + /// Cancellation token to cancel the operation. + /// A task that completes with the new instance of . + public static async Task CreateAsync(Options options, LocalDatabase db, CancellationToken cancellationToken) + { + var ipdv = new InProgressDblockVolumes(options, db); + ipdv.m_activeWriter.VolumeID = await db + .RegisterRemoteVolume(ipdv.m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, cancellationToken) + .ConfigureAwait(false); + return ipdv; + } + + /// + /// Checks if any data has been added to the current volume. /// public bool AnyData => m_anyData; /// - /// Checks if the current volume is full + /// Checks if the current volume is full. /// private bool IsFull => m_activeWriter.Filesize > m_maxVolumeSize; /// - /// Gets the volume ID of the current volume + /// Gets the volume ID of the current volume. /// public long VolumeID => m_activeWriter.VolumeID; /// - /// Gets the remote filename of the current volume + /// Gets the remote filename of the current volume. /// public string RemoteFilename => m_activeWriter.RemoteFilename; /// - /// Gets the list of volumes to delete after the upload is complete + /// Gets the list of volumes to delete after the upload is complete. /// public List VolumesToDelete => m_toDelete; /// - /// Gets the list of completed writers + /// Gets the list of completed writers. /// public List CompletedWriters => m_completedWriters; /// - /// Adds a block to the current volume, starting a new volume if the current one is full + /// Adds a block to the current volume, starting a new volume if the current one is full. /// - /// The hash of the block - /// The buffer containing the block data - /// The offset in the buffer where the block data starts - /// The size of the block data - /// The compression hint for the block - public void AddBlock(string hash, byte[] buffer, int offset, int size, CompressionHint hint) + /// The hash of the block. + /// The buffer containing the block data. + /// The offset in the buffer where the block data starts. + /// The size of the block data. + /// The compression hint for the block. + /// Cancellation token to cancel the operation. + /// A task that completes when the block is added. + public async Task AddBlock(string hash, byte[] buffer, int offset, int size, CompressionHint hint, CancellationToken cancellationToken) { - m_activeWriter.AddBlock(hash, buffer, offset, size, hint); + await m_activeWriter + .AddBlock(hash, buffer, offset, size, hint) + .ConfigureAwait(false); m_anyData = true; if (IsFull) - StartNewVolume(); + await StartNewVolume(cancellationToken).ConfigureAwait(false); } /// - /// Starts a new volume for the current operation + /// Starts a new volume for the current operation. /// - public void StartNewVolume() + /// Cancellation token to cancel the operation. + /// A task that completes when the new volume is started. + public async Task StartNewVolume(CancellationToken cancellationToken) { if (!m_anyData) return; @@ -733,24 +905,27 @@ namespace Duplicati.Library.Main.Operation m_completedWriters.Add(m_activeWriter); m_activeWriter = new BlockVolumeWriter(m_options); - m_activeWriter.VolumeID = m_database.RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, m_transaction.Transaction); + m_activeWriter.VolumeID = await m_database + .RegisterRemoteVolume(m_activeWriter.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, -1, TimeSpan.Zero, cancellationToken) + .ConfigureAwait(false); m_anyData = false; } } /// - /// Runs the repair process for missing dblock files + /// Runs the repair process for missing dblock files. /// - /// The backend manager to use for uploading - /// The database to use for the repair - /// The transaction to use for the repair - /// The list of missing dblock files - /// - /// Cancellation token to cancel the operation - /// >A task representing the asynchronous operation - private async Task RunRepairDblocks(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, IEnumerable missingDblockFiles, Action incrementProgress, CancellationToken cancellationToken) + /// The backend manager to use for uploading. + /// The database to use for the repair. + /// The list of missing dblock files. + /// + /// Cancellation token to cancel the operation. + /// >A task that completes when the repair is finished. + private async Task RunRepairDblocks(IBackendManager backendManager, LocalRepairDatabase db, IEnumerable missingDblockFiles, Action incrementProgress, CancellationToken cancellationToken) { - var currentVolume = new InProgressDblockVolumes(m_options, db, rtr); + var currentVolume = + await InProgressDblockVolumes.CreateAsync(m_options, db, cancellationToken) + .ConfigureAwait(false); foreach (var n in missingDblockFiles) { @@ -758,17 +933,19 @@ namespace Duplicati.Library.Main.Operation { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - rtr.Commit("CommitEarlyExit", false); + await db.Transaction + .CommitAsync("CommitEarlyExit", false, cancellationToken) + .ConfigureAwait(false); return; } incrementProgress(); - await RunRepairDblock(backendManager, db, rtr, n, currentVolume, cancellationToken).ConfigureAwait(false); + await RunRepairDblock(backendManager, db, n, currentVolume, cancellationToken).ConfigureAwait(false); if (currentVolume.CompletedWriters.Count > 0) - await UploadCompletedBlockVolumes(backendManager, currentVolume, db, rtr, cancellationToken).ConfigureAwait(false); + await UploadCompletedBlockVolumes(backendManager, currentVolume, db, cancellationToken).ConfigureAwait(false); } catch (Exception ex) { @@ -780,70 +957,83 @@ namespace Duplicati.Library.Main.Operation // If we have any data in the volume, complete it so it will be uploaded if (currentVolume.AnyData) - currentVolume.StartNewVolume(); + await currentVolume.StartNewVolume(cancellationToken).ConfigureAwait(false); // Complete any pending uploads - await UploadCompletedBlockVolumes(backendManager, currentVolume, db, rtr, cancellationToken).ConfigureAwait(false); + await UploadCompletedBlockVolumes(backendManager, currentVolume, db, cancellationToken).ConfigureAwait(false); // After reset, we need to remove the temporary volume from the database - db.RemoveRemoteVolume(currentVolume.RemoteFilename, rtr.Transaction); + await db + .RemoveRemoteVolume(currentVolume.RemoteFilename, cancellationToken) + .ConfigureAwait(false); } /// - /// Uploads all completed block volumes to the backend and deletes the old files + /// Uploads all completed block volumes to the backend and deletes the old files. /// - /// The backend manager to use for uploading - /// The completed volume state to upload - /// The database to use for the upload - /// >The transaction to use for the upload - /// Cancellation token to cancel the operation - /// >A task representing the asynchronous operation - async Task UploadCompletedBlockVolumes(IBackendManager backendManager, InProgressDblockVolumes activeVolume, LocalRepairDatabase db, ReusableTransaction rtr, CancellationToken cancellationToken) + /// The backend manager to use for uploading. + /// The completed volume state to upload. + /// The database to use for the upload. + /// Cancellation token to cancel the operation. + /// >A task that completes when the upload is finished. + async Task UploadCompletedBlockVolumes(IBackendManager backendManager, InProgressDblockVolumes activeVolume, LocalRepairDatabase db, CancellationToken cancellationToken) { foreach (var completedVolume in activeVolume.CompletedWriters) { - db.UpdateRemoteVolume(completedVolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, rtr.Transaction); + await db + .UpdateRemoteVolume(completedVolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, cancellationToken) + .ConfigureAwait(false); // Create a new index file that points to the new volume IndexVolumeWriter newvolindex = null; - Action indexVolumeFinished = null; + Func indexVolumeFinished = null; if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None) { newvolindex = new IndexVolumeWriter(m_options); - newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, rtr.Transaction); + newvolindex.VolumeID = await db + .RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, cancellationToken) + .ConfigureAwait(false); newvolindex.StartVolume(completedVolume.RemoteFilename); - foreach (var b in db.GetBlocks(completedVolume.VolumeID)) + await foreach (var b in db.GetBlocks(completedVolume.VolumeID, cancellationToken).ConfigureAwait(false)) newvolindex.AddBlock(b.Hash, b.Size); - db.AddIndexBlockLink(newvolindex.VolumeID, completedVolume.VolumeID, rtr.Transaction); + await db + .AddIndexBlockLink(newvolindex.VolumeID, completedVolume.VolumeID, cancellationToken) + .ConfigureAwait(false); if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) - indexVolumeFinished = () => + indexVolumeFinished = async () => { - foreach (var blocklist in db.GetBlocklists(completedVolume.VolumeID, m_options.Blocksize, m_options.BlockhashSize)) + await foreach (var blocklist in db.GetBlocklists(completedVolume.VolumeID, m_options.Blocksize, m_options.BlockhashSize, cancellationToken).ConfigureAwait(false)) newvolindex.WriteBlocklist(blocklist.Item1, blocklist.Item2, 0, blocklist.Item3); }; } // All information is in place, we can now upload the new volume - await backendManager.FlushPendingMessagesAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backendManager.FlushPendingMessagesAsync(db, cancellationToken).ConfigureAwait(false); if (m_options.Dryrun) Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReplaceBlockFile", "would upload new block file {0}", completedVolume.RemoteFilename); else { - rtr.Commit("PostRepairPreUploadBlockVolume"); + await db.Transaction + .CommitAsync("PostRepairPreUploadBlockVolume", token: cancellationToken) + .ConfigureAwait(false); await backendManager.PutAsync(completedVolume, newvolindex, indexVolumeFinished, false, null, cancellationToken).ConfigureAwait(false); } } // Flush the database as we have new volumes uploaded - await backendManager.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); // Prepare for deleting the old stuff foreach (var vol in activeVolume.VolumesToDelete) - db.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Deleting, vol.Size, vol.Hash, false, TimeSpan.FromHours(2), null, rtr.Transaction); + await db + .UpdateRemoteVolume(vol.Name, RemoteVolumeState.Deleting, vol.Size, vol.Hash, false, TimeSpan.FromHours(2), null, cancellationToken) + .ConfigureAwait(false); // Persist desired state prior to deleting the old files if (!m_options.Dryrun) - rtr.Commit("PostRepairBlockVolume"); + await db.Transaction + .CommitAsync("PostRepairBlockVolume", token: cancellationToken) + .ConfigureAwait(false); // Delete the old files foreach (var vol in activeVolume.VolumesToDelete) @@ -853,312 +1043,366 @@ namespace Duplicati.Library.Main.Operation await backendManager.DeleteAsync(vol.Name, vol.Size, false, cancellationToken).ConfigureAwait(false); // All done, the new dblocks are now in place - await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false); if (!m_options.Dryrun) - rtr.Commit("PostRepairBlockVolume"); + await db.Transaction + .CommitAsync("PostRepairBlockVolume", token: cancellationToken) + .ConfigureAwait(false); activeVolume.CompletedWriters.Clear(); activeVolume.VolumesToDelete.Clear(); } /// - /// Repairs a single dblock file by recreating it from available data + /// Repairs a single dblock file by recreating it from available data. /// - /// The backend manager to use for uploading - /// The database to use for the repair - /// The transaction to use for the repair - /// The remote volume entry to recreate - /// The in-progress dblock volume to use for the recreate - /// Cancellation token to cancel the operation - /// >A task representing the asynchronous operation - private async Task RunRepairDblock(IBackendManager backendManager, LocalRepairDatabase db, ReusableTransaction rtr, RemoteVolumeEntry originalVolume, InProgressDblockVolumes pendingVolume, CancellationToken cancellationToken) + /// The backend manager to use for uploading. + /// The database to use for the repair. + /// The remote volume entry to recreate. + /// The in-progress dblock volume to use for the recreate. + /// Cancellation token to cancel the operation. + /// >A task that completes when the repair is finished. + private async Task RunRepairDblock(IBackendManager backendManager, LocalRepairDatabase db, RemoteVolumeEntry originalVolume, InProgressDblockVolumes pendingVolume, CancellationToken cancellationToken) { // The dblock files are the most complex to recreate // as data can be either file contents, metadata or blocklist hashes // We attempt to recover all three source parts in the steps below - using (var mbl = db.CreateBlockList(originalVolume.Name, rtr)) + await using var mbl = await db + .CreateBlockList(originalVolume.Name, cancellationToken) + .ConfigureAwait(false); + + var originalMissingBlockCount = await mbl + .GetMissingBlockCount(cancellationToken) + .ConfigureAwait(false); + var recoveredSourceBlocks = 0L; + + // First we grab all known blocks from local files + string lastRestoredHash = null; + long lastRestoredSize = -1; + await foreach (var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize, cancellationToken).ConfigureAwait(false)) { - var originalMissingBlockCount = mbl.GetMissingBlockCount(); - var recoveredSourceBlocks = 0L; + if (block.Hash == lastRestoredHash && block.Size == lastRestoredSize) + continue; - // First we grab all known blocks from local files - string lastRestoredHash = null; - long lastRestoredSize = -1; - foreach (var block in mbl.GetSourceFilesWithBlocks(m_options.Blocksize)) + var size = (int)block.Size; + + try { - if (block.Hash == lastRestoredHash && block.Size == lastRestoredSize) + if (!File.Exists(block.File)) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "FileNotFound", null, "File not found: {0}", block.File); continue; + } - var size = (int)block.Size; + using var f = File.OpenRead(block.File); + f.Position = block.Offset; + var buffer = new byte[m_options.Blocksize]; + if (size != Library.Utility.Utility.ForceStreamRead(f, buffer, size)) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "FileLengthMismatch", null, "Block length mismatch on {0}, expected {1} but got {2}", block.File, size, f.Length); + continue; + } + + using var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm); + var newhash = Convert.ToBase64String(blockhasher.ComputeHash(buffer, 0, size)); + if (newhash != block.Hash) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "FileHashMismatch", null, "Block hash mismatch on {0}, expected {1} but got {2}", block.File, block.Hash, newhash); + continue; + } + + // Found it, no need to look again + lastRestoredHash = block.Hash; + lastRestoredSize = block.Size; + if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID, cancellationToken).ConfigureAwait(false)) + { + recoveredSourceBlocks++; + await pendingVolume + .AddBlock(block.Hash, buffer, 0, size, CompressionHint.Default, cancellationToken) + .ConfigureAwait(false); + } + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", ex, "Failed to access file: {0}", block.File); + } + } + + // Then grab all blocks with metadata + await foreach (var block in mbl.GetSourceItemsWithMetadataBlocks(cancellationToken).ConfigureAwait(false)) + { + if (block.Hash == lastRestoredHash && block.Size == lastRestoredSize) + continue; + + var size = (int)block.Size; + + try + { + var isFile = File.Exists(block.Path); + var isDir = Directory.Exists(block.Path); + if (!isFile && !isDir) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "EntryNotFound", null, "Entry not found: {0}", block.Path); + continue; + } + + using var snapshot = Snapshots.SnapshotUtility.CreateNoSnapshot([block.Path], true, true); + var entry = snapshot.GetFilesystemEntry(block.Path, isDir); + if (entry == null) + { + Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", null, "Entry not found: {0}", block.Path); + continue; + } + + var metadata = MetadataGenerator.GenerateMetadata(entry, entry.Attributes, m_options); + var metahash = Utility.WrapMetadata(metadata, m_options); + if (metahash.FileHash != block.Hash || metahash.Blob.Length != size) + { + Logging.Log.WriteVerboseMessage(LOGTAG, "MetadataHashMismatch", null, "Metadata block hash mismatch on {0}, expected {1} but got {2}", block.Path, block.Hash, metahash.FileHash); + continue; + } + + // Found it, no need to look again + lastRestoredHash = block.Hash; + lastRestoredSize = block.Size; + if (await mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID, cancellationToken).ConfigureAwait(false)) + { + recoveredSourceBlocks++; + await pendingVolume + .AddBlock(block.Hash, metahash.Blob, 0, metahash.Blob.Length, CompressionHint.Default, cancellationToken) + .ConfigureAwait(false); + } + } + catch (Exception ex) + { + Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", ex, "Failed to access file: {0}", block.Path); + } + } + + // If we did not find any source blocks, + // we only have the database contents and duplicated blocks to work with + // so no point in creating a new dblock volume for storing this data + if (recoveredSourceBlocks != 0) + { + // Then restore any blocklists that are missing + using var blocklistHash = new MemoryStream(); + LocalRepairDatabase.BlocklistHashesEntry lastBlocklist = null; + var blockhashsize = m_options.BlockhashSize; + var hashesPerBlock = m_options.Blocksize / m_options.BlockhashSize; + + // Helper function to emit a blocklist hash + async Task EmitBlockListBlock() + { + blocklistHash.Position = 0; + using var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm); + var resulthash = Convert.ToBase64String(blockhasher.ComputeHash(blocklistHash)); + var totalHashes = (lastBlocklist.BlocklistHashLength + m_options.Blocksize - 1) / m_options.Blocksize; + var hashesInLastBlock = totalHashes % hashesPerBlock; + var targetsize = lastBlocklist.Index >= totalHashes - 1 + ? hashesInLastBlock * blockhashsize + : hashesPerBlock * blockhashsize; + + if (resulthash == lastBlocklist.BlocklistHash && targetsize == blocklistHash.Length) + { + if (await mbl.SetBlockRestored(resulthash, blocklistHash.Length, pendingVolume.VolumeID, cancellationToken).ConfigureAwait(false)) + await pendingVolume + .AddBlock(resulthash, blocklistHash.ToArray(), 0, (int)blocklistHash.Length, CompressionHint.Default, cancellationToken) + .ConfigureAwait(false); + } + else + { + Logging.Log.WriteWarningMessage(LOGTAG, "BlocklistHashMismatch", null, "Internal consistency issue: blocklist hash mismatch on {0} / {1}, expected {2} but got {3}", lastBlocklist.BlocklistHash, lastBlocklist.Index, lastBlocklist.Hash, resulthash); + } + + blocklistHash.SetLength(0); + } + + await foreach (var blocklist in mbl.GetBlocklistHashes(hashesPerBlock, cancellationToken).ConfigureAwait(false)) + { + lastBlocklist ??= blocklist; + + if (lastBlocklist.BlocksetId != blocklist.BlocksetId || lastBlocklist.BlocklistHashIndex != blocklist.BlocklistHashIndex) + await EmitBlockListBlock().ConfigureAwait(false); + + var data = Convert.FromBase64String(blocklist.Hash); + blocklistHash.Write(data, 0, data.Length); + lastBlocklist = blocklist; + } + + // Handle any trailing blocklist hash + if (blocklistHash.Length > 0) + await EmitBlockListBlock().ConfigureAwait(false); + + //Then we grab all remote volumes that have the missing blocks + await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(await mbl.GetMissingBlockSources(cancellationToken).ToListAsync(cancellationToken: cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false)) + { try { - if (!File.Exists(block.File)) - { - Logging.Log.WriteVerboseMessage(LOGTAG, "FileNotFound", null, "File not found: {0}", block.File); - continue; - } - - using var f = File.OpenRead(block.File); - f.Position = block.Offset; - var buffer = new byte[m_options.Blocksize]; - if (size != Library.Utility.Utility.ForceStreamRead(f, buffer, size)) - { - Logging.Log.WriteVerboseMessage(LOGTAG, "FileLengthMismatch", null, "Block length mismatch on {0}, expected {1} but got {2}", block.File, size, f.Length); - continue; - } - - using var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm); - var newhash = Convert.ToBase64String(blockhasher.ComputeHash(buffer, 0, size)); - if (newhash != block.Hash) - { - Logging.Log.WriteVerboseMessage(LOGTAG, "FileHashMismatch", null, "Block hash mismatch on {0}, expected {1} but got {2}", block.File, block.Hash, newhash); - continue; - } - - // Found it, no need to look again - lastRestoredHash = block.Hash; - lastRestoredSize = block.Size; - if (mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID)) - { - recoveredSourceBlocks++; - pendingVolume.AddBlock(block.Hash, buffer, 0, size, CompressionHint.Default); - } + using (tmpfile) + using (var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options)) + foreach (var b in f.Blocks) + if (await mbl.SetBlockRestored(b.Key, b.Value, pendingVolume.VolumeID, cancellationToken).ConfigureAwait(false)) + if (f.ReadBlock(b.Key, buffer) == b.Value) + await pendingVolume + .AddBlock(b.Key, buffer, 0, (int)b.Value, CompressionHint.Default, cancellationToken) + .ConfigureAwait(false); } catch (Exception ex) { - Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", ex, "Failed to access file: {0}", block.File); + Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileAccessError", ex, "Failed to access remote file: {0}", name); } } + } - // Then grab all blocks with metadata - foreach (var block in mbl.GetSourceItemsWithMetadataBlocks()) + var missingBlocks = await mbl + .GetMissingBlockCount(cancellationToken) + .ConfigureAwait(false); + var recoveredBlocks = originalMissingBlockCount - missingBlocks; + if (recoveredBlocks == 0 || (m_options.DisablePartialDblockRecovery && missingBlocks > 0)) + { + Logging.Log.WriteInformationMessage(LOGTAG, "RepairMissingBlocks", "Repair cannot acquire {0} required blocks for volume {1}, which are required by the following filesets: ", missingBlocks, originalVolume.Name); + await foreach (var f in mbl.GetFilesetsUsingMissingBlocks(cancellationToken).ConfigureAwait(false)) + Logging.Log.WriteInformationMessage(LOGTAG, "AffectedFilesetName", f.Name); + + var recoverymsg = string.Format("If you want to continue working with the database, you can use the \"{0}\" and \"{1}\" commands to purge the missing data from the database and the remote storage.", "list-broken-files", "purge-broken-files"); + var logmsg = string.Format("Repair not possible, missing {0} blocks.\n" + recoverymsg, missingBlocks); + + Logging.Log.WriteInformationMessage(LOGTAG, "RecoverySuggestion", null, logmsg); + throw new UserInformationException(logmsg, "RepairIsNotPossible"); + } + else if (recoveredBlocks > 0) + { + if (missingBlocks > 0) + Logging.Log.WriteWarningMessage(LOGTAG, "RepairMissingBlocks", null, "Repair acquired {0} blocks for volume {1}, but {2} blocks are still missing. If you want to continue working with the database, you can use the \"{3}\" and \"{4}\" commands to purge the missing data from the database and the remote storage.", recoveredBlocks, originalVolume.Name, missingBlocks, "list-broken-files", "purge-broken-files"); + + // If we cannot fully recover the blocks, we will not be able to delete the old files. + // They will instead be registered with duplicated blocks, so the purge-broken-files command + // can remove them cleanly later + if (missingBlocks == 0) { - if (block.Hash == lastRestoredHash && block.Size == lastRestoredSize) - continue; + pendingVolume.VolumesToDelete.Add(originalVolume); + var oldIndexFiles = await db + .GetIndexFilesReferencingBlockFile(originalVolume.ID, cancellationToken) + .ToListAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); - var size = (int)block.Size; - - try + // Find all index files that point to the old volume + foreach (var oldIndexFile in oldIndexFiles) { - var isFile = File.Exists(block.Path); - var isDir = Directory.Exists(block.Path); - if (!isFile && !isDir) + var oldVolume = await db + .GetRemoteVolume(oldIndexFile, cancellationToken) + .ConfigureAwait(false); + if (oldVolume.State == RemoteVolumeState.Uploading || oldVolume.State == RemoteVolumeState.Uploaded || oldVolume.State == RemoteVolumeState.Verified) { - Logging.Log.WriteVerboseMessage(LOGTAG, "EntryNotFound", null, "Entry not found: {0}", block.Path); - continue; - } - - using var snapshot = Snapshots.SnapshotUtility.CreateNoSnapshot([block.Path], true, true); - var entry = snapshot.GetFilesystemEntry(block.Path, isDir); - if (entry == null) - { - Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", null, "Entry not found: {0}", block.Path); - continue; - } - - var metadata = MetadataGenerator.GenerateMetadata(entry, entry.Attributes, m_options); - var metahash = Utility.WrapMetadata(metadata, m_options); - if (metahash.FileHash != block.Hash || metahash.Blob.Length != size) - { - Logging.Log.WriteVerboseMessage(LOGTAG, "MetadataHashMismatch", null, "Metadata block hash mismatch on {0}, expected {1} but got {2}", block.Path, block.Hash, metahash.FileHash); - continue; - } - - // Found it, no need to look again - lastRestoredHash = block.Hash; - lastRestoredSize = block.Size; - if (mbl.SetBlockRestored(block.Hash, block.Size, pendingVolume.VolumeID)) - { - recoveredSourceBlocks++; - pendingVolume.AddBlock(block.Hash, metahash.Blob, 0, metahash.Blob.Length, CompressionHint.Default); - } - } - catch (Exception ex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "FileAccessError", ex, "Failed to access file: {0}", block.Path); - } - } - - // If we did not find any source blocks, - // we only have the database contents and duplicated blocks to work with - // so no point in creating a new dblock volume for storing this data - if (recoveredSourceBlocks != 0) - { - // Then restore any blocklists that are missing - using var blocklistHash = new MemoryStream(); - LocalRepairDatabase.BlocklistHashesEntry lastBlocklist = null; - var blockhashsize = m_options.BlockhashSize; - var hashesPerBlock = m_options.Blocksize / m_options.BlockhashSize; - - // Helper function to emit a blocklist hash - void EmitBlockListBlock() - { - blocklistHash.Position = 0; - using var blockhasher = HashFactory.CreateHasher(m_options.BlockHashAlgorithm); - var resulthash = Convert.ToBase64String(blockhasher.ComputeHash(blocklistHash)); - var totalHashes = (lastBlocklist.BlocklistHashLength + m_options.Blocksize - 1) / m_options.Blocksize; - var hashesInLastBlock = totalHashes % hashesPerBlock; - var targetsize = lastBlocklist.Index >= totalHashes - 1 - ? hashesInLastBlock * blockhashsize - : hashesPerBlock * blockhashsize; - - if (resulthash == lastBlocklist.BlocklistHash && targetsize == blocklistHash.Length) - { - if (mbl.SetBlockRestored(resulthash, blocklistHash.Length, pendingVolume.VolumeID)) - pendingVolume.AddBlock(resulthash, blocklistHash.ToArray(), 0, (int)blocklistHash.Length, CompressionHint.Default); + var blockVolumesReferenced = await db + .GetBlockVolumesFromIndexName(oldIndexFile, cancellationToken) + .ToListAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); + if (blockVolumesReferenced.Any(x => x.Name != originalVolume.Name)) + Logging.Log.WriteVerboseMessage(LOGTAG, "IndexFileNotDeleted", null, "Index file {0} references multiple remote volumes, skipping", oldVolume.Name); + else + pendingVolume.VolumesToDelete.Add(oldVolume); } else { - Logging.Log.WriteWarningMessage(LOGTAG, "BlocklistHashMismatch", null, "Internal consistency issue: blocklist hash mismatch on {0} / {1}, expected {2} but got {3}", lastBlocklist.BlocklistHash, lastBlocklist.Index, lastBlocklist.Hash, resulthash); - } - - blocklistHash.SetLength(0); - } - - foreach (var blocklist in mbl.GetBlocklistHashes(hashesPerBlock)) - { - lastBlocklist ??= blocklist; - - if (lastBlocklist.BlocksetId != blocklist.BlocksetId || lastBlocklist.BlocklistHashIndex != blocklist.BlocklistHashIndex) - EmitBlockListBlock(); - - var data = Convert.FromBase64String(blocklist.Hash); - blocklistHash.Write(data, 0, data.Length); - lastBlocklist = blocklist; - } - - // Handle any trailing blocklist hash - if (blocklistHash.Length > 0) - EmitBlockListBlock(); - - //Then we grab all remote volumes that have the missing blocks - await foreach (var (tmpfile, _, _, name) in backendManager.GetFilesOverlappedAsync(mbl.GetMissingBlockSources().ToList(), cancellationToken).ConfigureAwait(false)) - { - try - { - var buffer = new byte[m_options.Blocksize]; - using (tmpfile) - using (var f = new BlockVolumeReader(RestoreHandler.GetCompressionModule(name), tmpfile, m_options)) - foreach (var b in f.Blocks) - if (mbl.SetBlockRestored(b.Key, b.Value, pendingVolume.VolumeID)) - if (f.ReadBlock(b.Key, buffer) == b.Value) - pendingVolume.AddBlock(b.Key, buffer, 0, (int)b.Value, CompressionHint.Default); - } - catch (Exception ex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileAccessError", ex, "Failed to access remote file: {0}", name); - } - } - } - - var missingBlocks = mbl.GetMissingBlockCount(); - var recoveredBlocks = originalMissingBlockCount - missingBlocks; - if (recoveredBlocks == 0 || (m_options.DisablePartialDblockRecovery && missingBlocks > 0)) - { - Logging.Log.WriteInformationMessage(LOGTAG, "RepairMissingBlocks", "Repair cannot acquire {0} required blocks for volume {1}, which are required by the following filesets: ", missingBlocks, originalVolume.Name); - foreach (var f in mbl.GetFilesetsUsingMissingBlocks()) - Logging.Log.WriteInformationMessage(LOGTAG, "AffectedFilesetName", f.Name); - - var recoverymsg = string.Format("If you want to continue working with the database, you can use the \"{0}\" and \"{1}\" commands to purge the missing data from the database and the remote storage.", "list-broken-files", "purge-broken-files"); - var logmsg = string.Format("Repair not possible, missing {0} blocks.\n" + recoverymsg, missingBlocks); - - Logging.Log.WriteInformationMessage(LOGTAG, "RecoverySuggestion", null, logmsg); - throw new UserInformationException(logmsg, "RepairIsNotPossible"); - } - else if (recoveredBlocks > 0) - { - if (missingBlocks > 0) - Logging.Log.WriteWarningMessage(LOGTAG, "RepairMissingBlocks", null, "Repair acquired {0} blocks for volume {1}, but {2} blocks are still missing. If you want to continue working with the database, you can use the \"{3}\" and \"{4}\" commands to purge the missing data from the database and the remote storage.", recoveredBlocks, originalVolume.Name, missingBlocks, "list-broken-files", "purge-broken-files"); - - // If we cannot fully recover the blocks, we will not be able to delete the old files. - // They will instead be registered with duplicated blocks, so the purge-broken-files command - // can remove them cleanly later - if (missingBlocks == 0) - { - pendingVolume.VolumesToDelete.Add(originalVolume); - var oldIndexFiles = db.GetIndexFilesReferencingBlockFile(originalVolume.ID, rtr.Transaction).ToList(); - - // Find all index files that point to the old volume - foreach (var oldIndexFile in oldIndexFiles) - { - var oldVolume = db.GetRemoteVolume(oldIndexFile, rtr.Transaction); - if (oldVolume.State == RemoteVolumeState.Uploading || oldVolume.State == RemoteVolumeState.Uploaded || oldVolume.State == RemoteVolumeState.Verified) - { - var blockVolumesReferenced = db.GetBlockVolumesFromIndexName(oldIndexFile).ToList(); - if (blockVolumesReferenced.Any(x => x.Name != originalVolume.Name)) - Logging.Log.WriteVerboseMessage(LOGTAG, "IndexFileNotDeleted", null, "Index file {0} references multiple remote volumes, skipping", oldVolume.Name); - else - pendingVolume.VolumesToDelete.Add(oldVolume); - } - else - { - Logging.Log.WriteVerboseMessage(LOGTAG, "IndexFileNotDeleted", null, "Index file {0} is not in a state to be deleted, skipping", oldVolume.Name); - } + Logging.Log.WriteVerboseMessage(LOGTAG, "IndexFileNotDeleted", null, "Index file {0} is not in a state to be deleted, skipping", oldVolume.Name); } } } } } + /// + /// Runs the repair process for broken filesets in the database. + /// + /// The backend manager to use for accessing remote files. + /// >A task that completes when the repair is finished. + /// If the database doesn't exist. public async Task RunRepairBrokenFilesets(IBackendManager backendManager) { if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using (var db = new LocalRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache)) - using (var tr = new ReusableTransaction(db)) + await using var db = + await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, null, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + var sets = await db + .GetFilesetsWithMissingFiles(m_result.TaskControl.ProgressToken) + .ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + if (sets.Count == 0) + return; + + Logging.Log.WriteInformationMessage(LOGTAG, "RepairingBrokenFilesets", "Repairing {0} broken filesets", sets.Count); + var ix = 0; + foreach (var entry in sets) { - var sets = db.GetFilesetsWithMissingFiles(null).ToList(); - if (sets.Count == 0) - return; + ix++; + Logging.Log.WriteInformationMessage(LOGTAG, "RepairingBrokenFileset", "Repairing broken fileset {0} of {1}: {2}", ix, sets.Count, entry.Value); + var volume = await db + .GetRemoteVolumeFromFilesetID(entry.Key, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + var parsed = VolumeBase.ParseFilename(volume.Name); + using var tmpfile = await backendManager.GetAsync(volume.Name, volume.Hash, volume.Size, CancellationToken.None).ConfigureAwait(false); + using var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read); + using var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions); + if (compressor == null) + throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); - Logging.Log.WriteInformationMessage(LOGTAG, "RepairingBrokenFilesets", "Repairing {0} broken filesets", sets.Count); - var ix = 0; - foreach (var entry in sets) + // Clear out the old fileset + await db + .DeleteFilesetEntries(entry.Key, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + await using (var rdb = await LocalRecreateDatabase.CreateAsync(db, m_options, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { - ix++; - Logging.Log.WriteInformationMessage(LOGTAG, "RepairingBrokenFileset", "Repairing broken fileset {0} of {1}: {2}", ix, sets.Count, entry.Value); - var volume = db.GetRemoteVolumeFromFilesetID(entry.Key, tr.Transaction); - var parsed = VolumeBase.ParseFilename(volume.Name); - using var tmpfile = await backendManager.GetAsync(volume.Name, volume.Hash, volume.Size, CancellationToken.None).ConfigureAwait(false); - using var stream = new FileStream(tmpfile, FileMode.Open, FileAccess.Read, FileShare.Read); - using var compressor = DynamicLoader.CompressionLoader.GetModule(parsed.CompressionModule, stream, ArchiveMode.Read, m_options.RawOptions); - if (compressor == null) - throw new UserInformationException(string.Format("Failed to load compression module: {0}", parsed.CompressionModule), "FailedToLoadCompressionModule"); - - // Clear out the old fileset - db.DeleteFilesetEntries(entry.Key, tr.Transaction); - using (var rdb = new LocalRecreateDatabase(db, m_options)) - RecreateDatabaseHandler.RecreateFilesetFromRemoteList(rdb, tr.Transaction, compressor, entry.Key, m_options, new FilterExpression()); - - tr.Commit("PostRepairFileset"); + await RecreateDatabaseHandler.RecreateFilesetFromRemoteList(rdb, compressor, entry.Key, m_options, new FilterExpression(), m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); } + await db.Transaction + .CommitAsync("PostRepairFileset", true, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); } } - public void RunRepairCommon() + /// + /// Runs the common repair operations on the database, such as fixing duplicate entries and missing blocklist hashes. + /// This method does not require any remote access and can be run without a backend manager. + /// + /// A task that completes when the repair is finished. + /// If the database file does not exist. + public async Task RunRepairCommon() { if (!File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); m_result.OperationProgressUpdater.UpdateProgress(0); - using (var db = new LocalRepairDatabase(m_options.Dbpath, m_options.SqlitePageCache)) - { - Utility.UpdateOptionsFromDb(db, m_options); + await using var db = + await LocalRepairDatabase.CreateRepairDatabase(m_options.Dbpath, null, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); - if (db.RepairInProgress || db.PartiallyRecreated) - Logging.Log.WriteWarningMessage(LOGTAG, "InProgressDatabase", null, "The database is marked as \"in-progress\" and may be incomplete."); + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); - db.FixDuplicateMetahash(); - db.FixDuplicateFileentries(); - db.FixDuplicateBlocklistHashes(m_options.Blocksize, m_options.BlockhashSize); - db.FixMissingBlocklistHashes(m_options.BlockHashAlgorithm, m_options.Blocksize); - db.FixEmptyMetadatasets(m_options); - } + if (await db.RepairInProgress(m_result.TaskControl.ProgressToken).ConfigureAwait(false) || await db.PartiallyRecreated(m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + Logging.Log.WriteWarningMessage(LOGTAG, "InProgressDatabase", null, "The database is marked as \"in-progress\" and may be incomplete."); + + await db.FixDuplicateMetahash(m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await db.FixDuplicateFileentries(m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await db + .FixDuplicateBlocklistHashes(m_options.Blocksize, m_options.BlockhashSize, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + await db + .FixMissingBlocklistHashes(m_options.BlockHashAlgorithm, m_options.Blocksize, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + await db + .FixEmptyMetadatasets(m_options, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs index b1793e866..12f424c73 100644 --- a/Duplicati/Library/Main/Operation/Restore/BlockManager.cs +++ b/Duplicati/Library/Main/Operation/Restore/BlockManager.cs @@ -131,10 +131,9 @@ namespace Duplicati.Library.Main.Operation.Restore /// /// Initializes a new instance of the class. /// - /// The database holding information about how many of each block this restore requires. /// Channel for submitting block requests from a volume. /// Number of readers accessing this dictionary. Used during shutdown / cleanup. - public SleepableDictionary(LocalRestoreDatabase db, IWriteChannel volume_request, Options options, int readers) + private SleepableDictionary(IWriteChannel volume_request, Options options, int readers) { m_options = options; m_volume_request = volume_request; @@ -171,14 +170,31 @@ namespace Duplicati.Library.Main.Operation.Restore sw_set_set = options.InternalProfiling ? new() : null; sw_set_wake_get = options.InternalProfiling ? new() : null; sw_set_wake_set = options.InternalProfiling ? new() : null; + } - foreach (var (block_id, volume_id) in db.GetBlocksAndVolumeIDs(options.SkipMetadata)) + /// + /// Asynchronously creates a new instance of the class. + /// This method initializes the block and volume counts based on the data in the database. + /// + /// The database holding information about how many of each block this restore requires. + /// CoCoL channel for submitting block requests from a volume. + /// The restore options. + /// The number of readers accessing this dictionary. Used during shutdown / cleanup. + /// A cancellation token to cancel the operation. + /// A task that when awaited returns a new instance of the class. + public static async Task CreateAsync(LocalRestoreDatabase db, IWriteChannel volume_request, Options options, int readers, CancellationToken cancellationToken) + { + var sd = new SleepableDictionary(volume_request, options, readers); + + await foreach (var (block_id, volume_id) in db.GetBlocksAndVolumeIDs(options.SkipMetadata, cancellationToken).ConfigureAwait(false)) { - var bc = m_blockcount.TryGetValue(block_id, out var c); - m_blockcount[block_id] = bc ? c + 1 : 1; - var vc = m_volumecount.TryGetValue(volume_id, out var v); - m_volumecount[volume_id] = vc ? v + 1 : 1; + var bc = sd.m_blockcount.TryGetValue(block_id, out var c); + sd.m_blockcount[block_id] = bc ? c + 1 : 1; + var vc = sd.m_volumecount.TryGetValue(volume_id, out var v); + sd.m_volumecount[volume_id] = vc ? v + 1 : 1; } + + return sd; } /// @@ -303,7 +319,7 @@ namespace Duplicati.Library.Main.Operation.Restore sw_get_wait?.Stop(); } - return await new_tcs.Task; + return await new_tcs.Task.ConfigureAwait(false); } /// @@ -421,10 +437,11 @@ namespace Duplicati.Library.Main.Operation.Restore /// /// The named channels for the restore operation. /// The database holding information about how many of each block this restore requires. - /// The restore options. /// The channels for reading block requests from the `FileProcessor`. /// The channels for writing block responses back to the `FileProcessor`. - public static Task Run(Channels channels, LocalRestoreDatabase db, Options options, IChannel[] fp_requests, IChannel>[] fp_responses) + /// The restore options. + /// The results of the restore operation. + public static Task Run(Channels channels, LocalRestoreDatabase db, IChannel[] fp_requests, IChannel>[] fp_responses, Options options, RestoreResults results) { return AutomationExtensions.RunTask( new @@ -435,7 +452,9 @@ namespace Duplicati.Library.Main.Operation.Restore async self => { // Create a cache for the blocks, - using SleepableDictionary cache = new(db, self.Output, options, fp_requests.Length); + using SleepableDictionary cache = + await SleepableDictionary.CreateAsync(db, self.Output, options, fp_requests.Length, results.TaskControl.ProgressToken) + .ConfigureAwait(false); // The volume consumer will read blocks from the input channel (data blocks from the volumes) and store them in the cache. var volume_consumer = Task.Run(async () => diff --git a/Duplicati/Library/Main/Operation/Restore/FileLister.cs b/Duplicati/Library/Main/Operation/Restore/FileLister.cs index 4f3438f48..7abeffd78 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileLister.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileLister.cs @@ -66,7 +66,12 @@ namespace Duplicati.Library.Main.Operation.Restore try { sw_get_files?.Start(); - var files = db.GetFilesAndSymlinksToRestore().ToArray(); + // The enumerables are cast to arrays to force the query to be executed and release the database lock. + var files = await db + .GetFilesAndSymlinksToRestore(result.TaskControl.ProgressToken) + .ToArrayAsync() + .ConfigureAwait(false); + result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_DownloadingRemoteFiles); sw_get_files?.Stop(); @@ -78,7 +83,11 @@ namespace Duplicati.Library.Main.Operation.Restore if (!options.SkipMetadata) { sw_get_folders?.Start(); - var folders = db.GetFolderMetadataToRestore().ToArray(); + // The enumerables are cast to arrays to force the query to be executed and release the database lock. + var folders = await db + .GetFolderMetadataToRestore(result.TaskControl.ProgressToken) + .ToArrayAsync() + .ConfigureAwait(false); sw_get_folders?.Stop(); sw_write_folder?.Start(); diff --git a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs index 1ebd2504c..adecbb0ea 100644 --- a/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs +++ b/Duplicati/Library/Main/Operation/Restore/FileProcessor.cs @@ -53,6 +53,8 @@ namespace Duplicati.Library.Main.Operation.Restore /// The number of file processors that are still restoring files. It is set by the . /// public static int file_processors_restoring_files; + public static object file_processor_continue_lock = new object(); + public static TaskCompletionSource file_processor_continue = new(); /// /// Runs the file processor process that restores the files that need to be restored. @@ -110,11 +112,16 @@ namespace Duplicati.Library.Main.Operation.Restore // Check if there are other FileProcessor's still restoring files if (!decremented) { - await RendezvousBeforeProcessingFolderMetadata(); + if (file_processors_restoring_files <= 0 && !file_processor_continue.Task.IsCompleted) + file_processor_continue.SetResult(); + else + await RendezvousBeforeProcessingFolderMetadata() + .ConfigureAwait(false); decremented = true; } - await RestoreMetadata(db, file, block_request, block_response, options, sw_meta, sw_work_meta, sw_req, sw_resp); + await RestoreMetadata(db, file, block_request, block_response, options, sw_meta, sw_work_meta, sw_req, sw_resp, results.TaskControl.ProgressToken) + .ConfigureAwait(false); continue; } @@ -122,7 +129,10 @@ namespace Duplicati.Library.Main.Operation.Restore // Get information about the blocks for the file // TODO rather than keeping all of the blocks in memory, we could do a single pass over the blocks using a cursor, only keeping the relevant block requests in memory. Maybe even only a single block request at a time. sw_block?.Start(); - var blocks = db.GetBlocksFromFile(file.BlocksetID).ToArray(); + var blocks = await db + .GetBlocksFromFile(file.BlocksetID, results.TaskControl.ProgressToken) + .ToArrayAsync() + .ConfigureAwait(false); sw_block?.Stop(); sw_work_verify_target?.Start(); @@ -362,12 +372,17 @@ namespace Duplicati.Library.Main.Operation.Restore else { // Not a missing block, so read from the file - sw_work_read?.Start(); - var read = await fs?.ReadAsync(buffer, 0, buffer.Length); - sw_work_read?.Stop(); - sw_work_hash?.Start(); - filehasher.TransformBlock(buffer, 0, read, buffer, 0); - sw_work_hash?.Stop(); + if (fs != null) + { + sw_work_read?.Start(); + var read = await fs + .ReadAsync(buffer, 0, buffer.Length) + .ConfigureAwait(false); + sw_work_read?.Stop(); + sw_work_hash?.Start(); + filehasher.TransformBlock(buffer, 0, read, buffer, 0); + sw_work_hash?.Stop(); + } } } @@ -420,7 +435,7 @@ namespace Duplicati.Library.Main.Operation.Restore if (!options.SkipMetadata) { - empty_file_or_symlink |= await RestoreMetadata(db, file, block_request, block_response, options, sw_meta, sw_work_meta, sw_req, sw_resp).ConfigureAwait(false); + empty_file_or_symlink |= await RestoreMetadata(db, file, block_request, block_response, options, sw_meta, sw_work_meta, sw_req, sw_resp, results.TaskControl.ProgressToken).ConfigureAwait(false); sw_work_meta?.Stop(); } @@ -459,7 +474,12 @@ namespace Duplicati.Library.Main.Operation.Restore finally { if (!decremented) - Interlocked.Decrement(ref file_processors_restoring_files); + lock (file_processor_continue_lock) + { + file_processors_restoring_files--; + if (file_processors_restoring_files <= 0 && !file_processor_continue.Task.IsCompleted) + file_processor_continue.SetResult(); + } block_request.Retire(); block_response.Retire(); @@ -554,11 +574,18 @@ namespace Duplicati.Library.Main.Operation.Restore /// An awaitable task that completes once all of the FileProcessor's have rendezvoused. private static async Task RendezvousBeforeProcessingFolderMetadata() { - Interlocked.Decrement(ref file_processors_restoring_files); - - while (file_processors_restoring_files > 0) + var should_wait = false; + lock (file_processor_continue_lock) { - await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); + file_processors_restoring_files--; + if (file_processors_restoring_files <= 0 && !file_processor_continue.Task.IsCompleted) + file_processor_continue.SetResult(); + should_wait = file_processors_restoring_files > 0; + } + + if (should_wait) + { + await file_processor_continue.Task.ConfigureAwait(false); } } @@ -574,15 +601,22 @@ namespace Duplicati.Library.Main.Operation.Restore /// The stopwatch for internal profiling of the general processing. /// The stopwatch for internal profiling of the block requests. /// The stopwatch for internal profiling of the block responses. - private static async Task RestoreMetadata(LocalRestoreDatabase db, FileRequest file, IChannel block_request, IChannel> block_response, Options options, Stopwatch sw_meta, Stopwatch sw_work, Stopwatch sw_req, Stopwatch sw_resp) + /// The cancellation token to cancel the operation. + /// An awaitable `Task`, which returns `true` if the metadata was restored successfully, `false` otherwise. + private static async Task RestoreMetadata(LocalRestoreDatabase db, FileRequest file, IChannel block_request, IChannel> block_response, Options options, Stopwatch sw_meta, Stopwatch sw_work, Stopwatch sw_req, Stopwatch sw_resp, CancellationToken cancellationToken) { sw_meta?.Start(); - var blocks = db.GetMetadataBlocksFromFile(file.ID); + // Since each FileProcessor should have its own connection, it's ok + // to keep the lock associated with the read transaction for a long + // time, as the other processes should still be able to read. + // Therefore, we don't need to read the entire result set into + // memory. + var blocks = db.GetMetadataBlocksFromFile(file.ID, cancellationToken); sw_meta?.Stop(); using var ms = new MemoryStream(); - foreach (var block in blocks) + await foreach (var block in blocks.ConfigureAwait(false)) { sw_work?.Stop(); sw_req?.Start(); @@ -768,7 +802,7 @@ namespace Duplicati.Library.Main.Operation.Restore int j = 0; for (long i = 0; i < total_blocks; i++) { - int read; + int read = 0; if (j < blocks.Count && blocks[j].BlockOffset == i) { // The current block is a missing block @@ -797,18 +831,23 @@ namespace Duplicati.Library.Main.Operation.Restore { if (!options.Dryrun) { - try + if (f_target != null) { - f_target?.Seek(blocks[j].BlockOffset * options.Blocksize, SeekOrigin.Begin); - await f_target?.WriteAsync(buffer, 0, read); - } - catch (Exception) - { - lock (results) + try { - results.BrokenLocalFiles.Add(file.TargetPath); + f_target.Seek(blocks[j].BlockOffset * options.Blocksize, SeekOrigin.Begin); + await f_target + .WriteAsync(buffer, 0, read) + .ConfigureAwait(false); + } + catch (Exception) + { + lock (results) + { + results.BrokenLocalFiles.Add(file.TargetPath); + } + throw; } - throw; } } bytes_read += read; @@ -841,18 +880,23 @@ namespace Duplicati.Library.Main.Operation.Restore else { // The current block is not a missing block - read from the target file. - try + if (f_target != null) { - f_target?.Seek(i * options.Blocksize, SeekOrigin.Begin); - read = await f_target?.ReadAsync(buffer, 0, options.Blocksize); - } - catch (Exception) - { - lock (results) + try { - results.BrokenLocalFiles.Add(file.TargetPath); + f_target.Seek(i * options.Blocksize, SeekOrigin.Begin); + read = await f_target + .ReadAsync(buffer, 0, options.Blocksize) + .ConfigureAwait(false); + } + catch (Exception) + { + lock (results) + { + results.BrokenLocalFiles.Add(file.TargetPath); + } + throw; } - throw; } } filehasher.TransformBlock(buffer, 0, read, buffer, 0); diff --git a/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs b/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs index b615c4d8e..a2c4aed9c 100644 --- a/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs +++ b/Duplicati/Library/Main/Operation/Restore/VolumeDownloader.cs @@ -75,10 +75,13 @@ namespace Duplicati.Library.Main.Operation.Restore // Trigger the download. sw_wait?.Start(); TempFile f; - var (volume_name, size, hash) = db.GetVolumeInfo(volume_id).First(); + var (volume_name, size, hash) = await db + .GetVolumeInfo(volume_id, results.TaskControl.ProgressToken) + .FirstAsync() + .ConfigureAwait(false); try { - f = await backend.GetDirectAsync(volume_name, hash, size, CancellationToken.None).ConfigureAwait(false); + f = await backend.GetDirectAsync(volume_name, hash, size, results.TaskControl.TransferToken).ConfigureAwait(false); } catch (Exception) { diff --git a/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs b/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs index e481c0f9c..b64382de9 100644 --- a/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreControlFilesHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; @@ -45,62 +45,68 @@ namespace Duplicati.Library.Main.Operation if (!Directory.Exists(m_options.Restorepath)) Directory.CreateDirectory(m_options.Restorepath); - using (var tmpdb = new TempFile()) - using (var db = new Database.LocalDatabase(File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles", true, m_options.SqlitePageCache)) + using var tmpdb = new TempFile(); + await using var db = await Database.LocalDatabase.CreateLocalDatabaseAsync( + File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles", + true, + null, + m_result.TaskControl.ProgressToken + ).ConfigureAwait(false); + + var filter = JoinedFilterExpression.Join(new FilterExpression(filterstrings), compositefilter); + + try { - var filter = JoinedFilterExpression.Join(new FilterExpression(filterstrings), compositefilter); + var filteredList = ListFilesHandler.ParseAndFilterFilesets(await backendManager.ListAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false), m_options); + if (filteredList.Count == 0) + throw new Exception("No filesets found on remote target"); - try - { - var filteredList = ListFilesHandler.ParseAndFilterFilesets(await backendManager.ListAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false), m_options); - if (filteredList.Count == 0) - throw new Exception("No filesets found on remote target"); + var lastEx = new Exception("No suitable files found on remote target"); - var lastEx = new Exception("No suitable files found on remote target"); - - foreach (var fileversion in filteredList) - try + foreach (var fileversion in filteredList) + try + { + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - { - await backendManager.WaitForEmptyAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - return; - } - - var file = fileversion.Value.File; - var entry = db.GetRemoteVolume(file.Name); - - var res = new List(); - using (var tmpfile = await backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) - using (var tmp = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(file.Name), tmpfile, m_options)) - foreach (var cf in tmp.ControlFiles) - if (FilterExpression.Matches(filter, cf.Key)) - { - var targetpath = Path.Combine(m_options.Restorepath, cf.Key); - using (var ts = File.Create(targetpath)) - await Library.Utility.Utility.CopyStreamAsync(cf.Value, ts, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - res.Add(targetpath); - } - - m_result.SetResult(res); - - lastEx = null; - break; - } - catch (Exception ex) - { - lastEx = ex; - if (ex.IsAbortException()) - throw; + await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + return; } - if (lastEx != null) - throw lastEx; - } - finally - { - await backendManager.WaitForEmptyAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); - } + var file = fileversion.Value.File; + var entry = await db + .GetRemoteVolume(file.Name, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + var res = new List(); + using (var tmpfile = await backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) + using (var tmp = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(file.Name), tmpfile, m_options)) + foreach (var cf in tmp.ControlFiles) + if (FilterExpression.Matches(filter, cf.Key)) + { + var targetpath = Path.Combine(m_options.Restorepath, cf.Key); + using (var ts = File.Create(targetpath)) + await Library.Utility.Utility.CopyStreamAsync(cf.Value, ts, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + res.Add(targetpath); + } + + m_result.SetResult(res); + + lastEx = null; + break; + } + catch (Exception ex) + { + lastEx = ex; + if (ex.IsAbortException()) + throw; + } + + if (lastEx != null) + throw lastEx; + } + finally + { + await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index 9913aa753..2334dc0b6 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -115,14 +115,16 @@ namespace Duplicati.Library.Main.Operation { if (!m_options.NoLocalDb && SystemIO.IO_OS.FileExists(m_options.Dbpath)) { - db = new LocalRestoreDatabase(m_options.Dbpath, m_options.SqlitePageCache); + db = await LocalRestoreDatabase.CreateAsync(m_options.Dbpath, null, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); } else { Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, building a temporary database"); tmpdb = new TempFile(); RecreateDatabaseHandler.NumberedFilterFilelistDelegate filelistfilter = FilterNumberedFilelist(m_options.Time, m_options.Version); - db = new LocalRestoreDatabase(tmpdb, m_options.SqlitePageCache); + db = await LocalRestoreDatabase.CreateAsync(tmpdb, null, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); m_result.RecreateDatabaseResults = new RecreateDatabaseResults(m_result); using (new Logging.Timer(LOGTAG, "RecreateTempDbForRestore", "Recreate temporary database for restore")) await new RecreateDatabaseHandler(m_options, (RecreateDatabaseResults)m_result.RecreateDatabaseResults) @@ -145,124 +147,131 @@ namespace Duplicati.Library.Main.Operation } finally { - db?.Dispose(); + if (db != null) + await db.DisposeAsync().ConfigureAwait(false); tmpdb?.Dispose(); } } - private static void PatchWithBlocklist(LocalRestoreDatabase database, BlockVolumeReader blocks, Options options, RestoreResults result, byte[] blockbuffer, RestoreHandlerMetadataStorage metadatastorage) + private static async Task PatchWithBlocklist(LocalRestoreDatabase database, BlockVolumeReader blocks, Options options, RestoreResults result, byte[] blockbuffer, RestoreHandlerMetadataStorage metadatastorage, CancellationToken cancellationToken) { var blocksize = options.Blocksize; var updateCounter = 0L; var fullblockverification = options.FullBlockVerification; - using (var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm)) - using (var blockmarker = database.CreateBlockMarker()) - using (var volumekeeper = database.GetMissingBlockData(blocks, options.Blocksize)) + using var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm); + await using var blockmarker = await database.CreateBlockMarkerAsync(cancellationToken).ConfigureAwait(false); + await using var volumekeeper = await database.GetMissingBlockData(blocks, options.Blocksize, cancellationToken).ConfigureAwait(false); + await foreach (var restorelist in volumekeeper.FilesWithMissingBlocks(cancellationToken).ConfigureAwait(false)) { - foreach (var restorelist in volumekeeper.FilesWithMissingBlocks) + var targetpath = restorelist.Path; + + if (options.Dryrun) { - var targetpath = restorelist.Path; + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchFile", "Would patch file with remote data: {0}", targetpath); + } + else + { + Logging.Log.WriteVerboseMessage(LOGTAG, "PatchingFile", "Patching file with remote data: {0}", targetpath); - if (options.Dryrun) + try { - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchFile", "Would patch file with remote data: {0}", targetpath); - } - else - { - Logging.Log.WriteVerboseMessage(LOGTAG, "PatchingFile", "Patching file with remote data: {0}", targetpath); - - try + var folderpath = SystemIO.IO_OS.PathGetDirectoryName(targetpath); + if (!options.Dryrun && !SystemIO.IO_OS.DirectoryExists(folderpath)) { - var folderpath = SystemIO.IO_OS.PathGetDirectoryName(targetpath); - if (!options.Dryrun && !SystemIO.IO_OS.DirectoryExists(folderpath)) + Logging.Log.WriteWarningMessage(LOGTAG, "CreateMissingFolder", null, "Creating missing folder {0} for file {1}", folderpath, targetpath); + SystemIO.IO_OS.DirectoryCreate(folderpath); + } + + // TODO: Much faster if we iterate the volume and checks what blocks are used, + // because the compressors usually like sequential reading + using (var file = SystemIO.IO_OS.FileOpenWrite(targetpath)) + await foreach (var targetblock in restorelist.Blocks(cancellationToken).ConfigureAwait(false)) { - Logging.Log.WriteWarningMessage(LOGTAG, "CreateMissingFolder", null, "Creating missing folder {0} for file {1}", folderpath, targetpath); - SystemIO.IO_OS.DirectoryCreate(folderpath); - } - - // TODO: Much faster if we iterate the volume and checks what blocks are used, - // because the compressors usually like sequential reading - using (var file = SystemIO.IO_OS.FileOpenWrite(targetpath)) - foreach (var targetblock in restorelist.Blocks) + file.Position = targetblock.Offset; + var size = blocks.ReadBlock(targetblock.Key, blockbuffer); + if (targetblock.Size == size) { - file.Position = targetblock.Offset; - var size = blocks.ReadBlock(targetblock.Key, blockbuffer); - if (targetblock.Size == size) + var valid = !fullblockverification; + if (!valid) { - var valid = !fullblockverification; - if (!valid) - { - var key = Convert.ToBase64String(blockhasher.ComputeHash(blockbuffer, 0, size)); - if (targetblock.Key == key) - valid = true; - else - Logging.Log.WriteWarningMessage(LOGTAG, "InvalidBlock", null, "Invalid block detected for {0}, expected hash: {1}, actual hash: {2}", targetpath, targetblock.Key, key); - } - - if (valid) - { - file.Write(blockbuffer, 0, size); - blockmarker.SetBlockRestored(restorelist.FileID, targetblock.Offset / blocksize, targetblock.Key, size, false); - } + var key = Convert.ToBase64String(blockhasher.ComputeHash(blockbuffer, 0, size)); + if (targetblock.Key == key) + valid = true; + else + Logging.Log.WriteWarningMessage(LOGTAG, "InvalidBlock", null, "Invalid block detected for {0}, expected hash: {1}, actual hash: {2}", targetpath, targetblock.Key, key); } - else + + if (valid) { - Logging.Log.WriteWarningMessage(LOGTAG, "WrongBlockSize", null, "Block with hash {0} should have size {1} but has size {2}", targetblock.Key, targetblock.Size, size); + file.Write(blockbuffer, 0, size); + await blockmarker + .SetBlockRestored(restorelist.FileID, targetblock.Offset / blocksize, targetblock.Key, size, false, cancellationToken) + .ConfigureAwait(false); } } - - if ((++updateCounter) % 20 == 0) - blockmarker.UpdateProcessed(result.OperationProgressUpdater); - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "PatchFailed", ex, "Failed to patch file: \"{0}\", message: {1}, message: {1}", targetpath, ex.Message); - if (options.UnittestMode) - throw; - } - } - } - - if (!options.SkipMetadata) - { - foreach (var restoremetadata in volumekeeper.MetadataWithMissingBlocks) - { - var targetpath = restoremetadata.Path; - Logging.Log.WriteVerboseMessage(LOGTAG, "RecordingMetadata", "Recording metadata from remote data: {0}", targetpath); - - try - { - // TODO: When we support multi-block metadata this needs to deal with it - using (var ms = new System.IO.MemoryStream()) - { - foreach (var targetblock in restoremetadata.Blocks) + else { - ms.Position = targetblock.Offset; - var size = blocks.ReadBlock(targetblock.Key, blockbuffer); - if (targetblock.Size == size) - { - ms.Write(blockbuffer, 0, size); - blockmarker.SetBlockRestored(restoremetadata.FileID, targetblock.Offset / blocksize, targetblock.Key, size, true); - } + Logging.Log.WriteWarningMessage(LOGTAG, "WrongBlockSize", null, "Block with hash {0} should have size {1} but has size {2}", targetblock.Key, targetblock.Size, size); } - - ms.Position = 0; - metadatastorage.Add(targetpath, ms); - //blockmarker.RecordMetadata(restoremetadata.FileID, ms); } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "MetatdataRecordFailed", ex, "Failed to record metadata for file: \"{0}\", message: {1}", targetpath, ex.Message); - if (options.UnittestMode) - throw; - } + + if ((++updateCounter) % 20 == 0) + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater, cancellationToken) + .ConfigureAwait(false); + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "PatchFailed", ex, "Failed to patch file: \"{0}\", message: {1}, message: {1}", targetpath, ex.Message); + if (options.UnittestMode) + throw; } } - blockmarker.UpdateProcessed(result.OperationProgressUpdater); - blockmarker.Commit(); } + + if (!options.SkipMetadata) + { + await foreach (var restoremetadata in volumekeeper.MetadataWithMissingBlocks(cancellationToken).ConfigureAwait(false)) + { + var targetpath = restoremetadata.Path; + Logging.Log.WriteVerboseMessage(LOGTAG, "RecordingMetadata", "Recording metadata from remote data: {0}", targetpath); + + try + { + // TODO: When we support multi-block metadata this needs to deal with it + using (var ms = new System.IO.MemoryStream()) + { + await foreach (var targetblock in restoremetadata.Blocks(cancellationToken).ConfigureAwait(false)) + { + ms.Position = targetblock.Offset; + var size = blocks.ReadBlock(targetblock.Key, blockbuffer); + if (targetblock.Size == size) + { + ms.Write(blockbuffer, 0, size); + await blockmarker + .SetBlockRestored(restoremetadata.FileID, targetblock.Offset / blocksize, targetblock.Key, size, true, cancellationToken) + .ConfigureAwait(false); + } + } + + ms.Position = 0; + metadatastorage.Add(targetpath, ms); + //blockmarker.RecordMetadata(restoremetadata.FileID, ms); + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "MetatdataRecordFailed", ex, "Failed to record metadata for file: \"{0}\", message: {1}", targetpath, ex.Message); + if (options.UnittestMode) + throw; + } + } + } + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater, cancellationToken) + .ConfigureAwait(false); + await blockmarker.CommitAsync(cancellationToken).ConfigureAwait(false); } private static void ApplyStoredMetadata(Options options, RestoreHandlerMetadataStorage metadatastorage) @@ -308,24 +317,32 @@ namespace Duplicati.Library.Main.Operation private async Task DoRunNewAsync(IBackendManager backendManager, LocalRestoreDatabase database, Library.Utility.IFilter filter, CancellationToken cancellationToken) { // Perform initial setup - Utility.UpdateOptionsFromDb(database, m_options); - Utility.VerifyOptionsAndUpdateDatabase(database, m_options); + await Utility.UpdateOptionsFromDb(database, m_options, cancellationToken) + .ConfigureAwait(false); + await Utility.VerifyOptionsAndUpdateDatabase(database, m_options, cancellationToken) + .ConfigureAwait(false); // Verify the backend if necessary if (!m_options.NoBackendverification) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify); - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, null).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, cancellationToken).ConfigureAwait(false); } // Prepare the block and file list and create the directory structure m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateFileList); using (new Logging.Timer(LOGTAG, "PrepareBlockList", "PrepareBlockList")) - PrepareBlockAndFileList(database, m_options, filter, m_result); + await PrepareBlockAndFileList(database, m_options, filter, m_result) + .ConfigureAwait(false); m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateTargetFolders); using (new Logging.Timer(LOGTAG, "CreateDirectory", "CreateDirectory")) await CreateDirectoryStructure(database, m_options, m_result).ConfigureAwait(false); + // At this point, there should be no more writes to the database, so we have to unlock the database: + await database.Transaction + .CommitAsync("CommitBeforeRestore", token: cancellationToken) + .ConfigureAwait(false); + using var setup_log_timer = new Logging.Timer(LOGTAG, "RestoreNetworkSetup", "RestoreNetworkSetup"); // Create the channels between BlockManager and FileProcessor Restore.Channels.BufferSize = m_options.RestoreChannelBufferSize; @@ -337,7 +354,7 @@ namespace Duplicati.Library.Main.Operation var filelister = Restore.FileLister.Run(channels, database, m_options, m_result); Restore.FileProcessor.file_processors_restoring_files = m_options.RestoreFileProcessors; var fileprocessors = Enumerable.Range(0, m_options.RestoreFileProcessors).Select(i => Restore.FileProcessor.Run(channels, database, fileprocessor_requests[i], fileprocessor_responses[i], m_options, m_result)).ToArray(); - var blockmanager = Restore.BlockManager.Run(channels, database, m_options, fileprocessor_requests, fileprocessor_responses); + var blockmanager = Restore.BlockManager.Run(channels, database, fileprocessor_requests, fileprocessor_responses, m_options, m_result); var volumecache = Restore.VolumeManager.Run(channels, m_options); var volumedownloaders = Enumerable.Range(0, m_options.RestoreVolumeDownloaders).Select(i => Restore.VolumeDownloader.Run(channels, database, backendManager, m_options, m_result)).ToArray(); var volumedecryptors = Enumerable.Range(0, m_options.RestoreVolumeDecryptors).Select(i => Restore.VolumeDecryptor.Run(channels, backendManager, m_options)).ToArray(); @@ -374,6 +391,12 @@ namespace Duplicati.Library.Main.Operation kill_updater.Cancel(); } + await database.Transaction + .CommitAsync("CommitAfterRestore", token: cancellationToken) + .ConfigureAwait(false); + + await database.DisposePoolAsync().ConfigureAwait(false); + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) return; @@ -400,8 +423,8 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesRestored", null, "Restore completed without errors but no files were restored"); // Drop the temp tables - database.DropRestoreTable(); - await backendManager.WaitForEmptyAsync(database, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await database.DropRestoreTable(cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(database, m_result.TaskControl.ProgressToken).ConfigureAwait(false); // Report that the restore is complete m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Complete); @@ -421,25 +444,29 @@ namespace Duplicati.Library.Main.Operation //using (var database = new LocalRestoreDatabase(dbparent)) using (var metadatastorage = new RestoreHandlerMetadataStorage()) { - Utility.UpdateOptionsFromDb(database, m_options); - Utility.VerifyOptionsAndUpdateDatabase(database, m_options); + await Utility.UpdateOptionsFromDb(database, m_options, cancellationToken) + .ConfigureAwait(false); + await Utility.VerifyOptionsAndUpdateDatabase(database, m_options, cancellationToken) + .ConfigureAwait(false); m_blockbuffer = new byte[m_options.Blocksize]; if (!m_options.NoBackendverification) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_PreRestoreVerify); - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, null).ConfigureAwait(false); + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, database, m_result.BackendWriter, latestVolumesOnly: false, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, cancellationToken).ConfigureAwait(false); } //Figure out what files are to be patched, and what blocks are needed m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateFileList); using (new Logging.Timer(LOGTAG, "PrepareBlockList", "PrepareBlockList")) - PrepareBlockAndFileList(database, m_options, filter, m_result); + await PrepareBlockAndFileList(database, m_options, filter, m_result) + .ConfigureAwait(false); //Make the entire output setup m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_CreateTargetFolders); using (new Logging.Timer(LOGTAG, "CreateDirectory", "CreateDirectory")) - await CreateDirectoryStructure(database, m_options, m_result); + await CreateDirectoryStructure(database, m_options, m_result) + .ConfigureAwait(false); //If we are patching an existing target folder, do not touch stuff that is already updated m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_ScanForExistingFiles); @@ -461,7 +488,7 @@ namespace Duplicati.Library.Main.Operation if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false); return; } @@ -476,14 +503,17 @@ namespace Duplicati.Library.Main.Operation if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false); return; } // Fill BLOCKS with remote sources List volumes; using (new Logging.Timer(LOGTAG, "GetMissingVolumes", "GetMissingVolumes")) - volumes = database.GetMissingVolumes().ToList(); + volumes = await database + .GetMissingVolumes(cancellationToken) + .ToListAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); if (volumes.Count > 0) { @@ -500,13 +530,14 @@ namespace Duplicati.Library.Main.Operation { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false); return; } using (tmpfile) using (var blocks = new BlockVolumeReader(GetCompressionModule(name), tmpfile, m_options)) - PatchWithBlocklist(database, blocks, m_options, m_result, m_blockbuffer, metadatastorage); + await PatchWithBlocklist(database, blocks, m_options, m_result, m_blockbuffer, metadatastorage, cancellationToken) + .ConfigureAwait(false); } catch (Exception ex) { @@ -520,7 +551,7 @@ namespace Duplicati.Library.Main.Operation var fileErrors = 0L; // Restore empty files. They might not have any blocks so don't appear in any volume. - foreach (var file in database.GetFilesToRestore(true).Where(item => item.Length == 0)) + await foreach (var file in database.GetFilesToRestore(true, cancellationToken).Where(item => item.Length == 0).ConfigureAwait(false)) { Logging.Log.WriteVerboseMessage(LOGTAG, "RestoreEmptyFile", "Restoring empty file \"{0}\"", file.Path); @@ -559,13 +590,13 @@ namespace Duplicati.Library.Main.Operation // After all blocks in the files are restored, verify the file hash using (var filehasher = HashFactory.CreateHasher(m_options.FileHashAlgorithm)) using (new Logging.Timer(LOGTAG, "RestoreVerification", "RestoreVerification")) - foreach (var file in database.GetFilesToRestore(true)) + await foreach (var file in database.GetFilesToRestore(true, cancellationToken).ConfigureAwait(false)) { try { if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false); return; } @@ -602,8 +633,8 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteWarningMessage(LOGTAG, "NoFilesRestored", null, "Restore completed without errors but no files were restored"); // Drop the temp tables - database.DropRestoreTable(); - await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false); + await database.DropRestoreTable(cancellationToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false); } m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Complete); @@ -682,124 +713,19 @@ namespace Duplicati.Library.Main.Operation private static async Task ScanForExistingSourceBlocksFast(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result) { // Fill BLOCKS with data from known local source files - using (var blockmarker = database.CreateBlockMarker()) + await using var blockmarker = await database.CreateBlockMarkerAsync(result.TaskControl.ProgressToken).ConfigureAwait(false); + var updateCount = 0L; + await foreach (var entry in database.GetFilesAndSourceBlocksFast(options.Blocksize, result.TaskControl.ProgressToken).ConfigureAwait(false)) { - var updateCount = 0L; - foreach (var entry in database.GetFilesAndSourceBlocksFast(options.Blocksize)) + var targetpath = entry.TargetPath; + var targetfileid = entry.TargetFileID; + var sourcepath = entry.SourcePath; + var patched = false; + + try { - var targetpath = entry.TargetPath; - var targetfileid = entry.TargetFileID; - var sourcepath = entry.SourcePath; - var patched = false; - - try + if (SystemIO.IO_OS.FileExists(sourcepath)) { - if (SystemIO.IO_OS.FileExists(sourcepath)) - { - var folderpath = SystemIO.IO_OS.PathGetDirectoryName(targetpath); - if (!options.Dryrun && !SystemIO.IO_OS.DirectoryExists(folderpath)) - { - Logging.Log.WriteWarningMessage(LOGTAG, "CreateMissingFolder", null, "Creating missing folder {0} for file {1}", folderpath, targetpath); - SystemIO.IO_OS.DirectoryCreate(folderpath); - } - - using (var targetstream = options.Dryrun ? null : SystemIO.IO_OS.FileOpenWrite(targetpath)) - { - try - { - using (var sourcestream = SystemIO.IO_OS.FileOpenRead(sourcepath)) - { - foreach (var block in entry.Blocks) - { - if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; - - //TODO: Handle metadata - - if (sourcestream.Length > block.Offset) - { - sourcestream.Position = block.Offset; - - int size = Library.Utility.Utility.ForceStreamRead(sourcestream, blockbuffer, blockbuffer.Length); - if (size == block.Size) - { - var key = Convert.ToBase64String(hasher.ComputeHash(blockbuffer, 0, size)); - if (key == block.Hash) - { - patched = true; - if (!options.Dryrun) - { - targetstream.Position = block.Offset; - targetstream.Write(blockbuffer, 0, size); - } - - blockmarker.SetBlockRestored(targetfileid, block.Index, key, block.Size, false); - } - } - } - } - } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with data from local file \"{1}\", message: {2}", targetpath, sourcepath, ex.Message); - if (ex.IsAbortException()) - throw; - } - } - - if ((++updateCount) % 20 == 0) - { - blockmarker.UpdateProcessed(result.OperationProgressUpdater); - if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; - } - - } - else - { - Logging.Log.WriteVerboseMessage(LOGTAG, "LocalSourceMissing", "Local source file not found: {0}", sourcepath); - } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with local data, message: {1}", targetpath, ex.Message); - if (ex.IsAbortException()) - throw; - if (options.UnittestMode) - throw; - } - - if (patched) - Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is patched with some local data: {0}", targetpath); - else - Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is not patched any local data: {0}", targetpath); - - if (patched && options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchWithLocal", "Would patch file with local data: {0}", targetpath); - } - - blockmarker.UpdateProcessed(result.OperationProgressUpdater); - blockmarker.Commit(); - } - } - - private static async Task ScanForExistingSourceBlocks(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result, RestoreHandlerMetadataStorage metadatastorage) - { - // Fill BLOCKS with data from known local source files - using (var blockmarker = database.CreateBlockMarker()) - { - var updateCount = 0L; - foreach (var restorelist in database.GetFilesAndSourceBlocks(options.SkipMetadata, options.Blocksize)) - { - var targetpath = restorelist.TargetPath; - var targetfileid = restorelist.TargetFileID; - var patched = false; - try - { - if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; - var folderpath = SystemIO.IO_OS.PathGetDirectoryName(targetpath); if (!options.Dryrun && !SystemIO.IO_OS.DirectoryExists(folderpath)) { @@ -807,96 +733,206 @@ namespace Duplicati.Library.Main.Operation SystemIO.IO_OS.DirectoryCreate(folderpath); } - using (var file = options.Dryrun ? null : SystemIO.IO_OS.FileOpenWrite(targetpath)) - foreach (var targetblock in restorelist.Blocks) + using (var targetstream = options.Dryrun ? null : SystemIO.IO_OS.FileOpenWrite(targetpath)) + { + try { - foreach (var source in targetblock.Blocksources) + using var sourcestream = SystemIO.IO_OS.FileOpenRead(sourcepath); + await foreach (var block in entry.Blocks(result.TaskControl.ProgressToken).ConfigureAwait(false)) { - try + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + + //TODO: Handle metadata + + if (sourcestream.Length > block.Offset) { - if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; + sourcestream.Position = block.Offset; - if (SystemIO.IO_OS.FileExists(source.Path)) + int size = Library.Utility.Utility.ForceStreamRead(sourcestream, blockbuffer, blockbuffer.Length); + if (size == block.Size) { - if (source.IsMetadata) + var key = Convert.ToBase64String(hasher.ComputeHash(blockbuffer, 0, size)); + if (key == block.Hash) { - // TODO: Handle this by reconstructing - // metadata from file and checking the hash - - continue; - } - else - { - using (var sourcefile = SystemIO.IO_OS.FileOpenRead(source.Path)) + patched = true; + if (!options.Dryrun) { - sourcefile.Position = source.Offset; - int size = Library.Utility.Utility.ForceStreamRead(sourcefile, blockbuffer, blockbuffer.Length); - if (size == targetblock.Size) - { - var key = Convert.ToBase64String(hasher.ComputeHash(blockbuffer, 0, size)); - if (key == targetblock.Hash) - { - if (!options.Dryrun) - { - if (targetblock.IsMetadata) - metadatastorage.Add(targetpath, new System.IO.MemoryStream(blockbuffer, 0, size)); - else - { - file.Position = targetblock.Offset; - file.Write(blockbuffer, 0, size); - } - } + targetstream.Position = block.Offset; + targetstream.Write(blockbuffer, 0, size); + } - blockmarker.SetBlockRestored(targetfileid, targetblock.Index, key, targetblock.Size, false); - patched = true; - break; + await blockmarker.SetBlockRestored(targetfileid, block.Index, key, block.Size, false, result.TaskControl.ProgressToken) + .ConfigureAwait(false); + } + } + } + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with data from local file \"{1}\", message: {2}", targetpath, sourcepath, ex.Message); + if (ex.IsAbortException()) + throw; + } + } + + if ((++updateCount) % 20 == 0) + { + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken) + .ConfigureAwait(false); + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + } + + } + else + { + Logging.Log.WriteVerboseMessage(LOGTAG, "LocalSourceMissing", "Local source file not found: {0}", sourcepath); + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with local data, message: {1}", targetpath, ex.Message); + if (ex.IsAbortException()) + throw; + if (options.UnittestMode) + throw; + } + + if (patched) + Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is patched with some local data: {0}", targetpath); + else + Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is not patched any local data: {0}", targetpath); + + if (patched && options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchWithLocal", "Would patch file with local data: {0}", targetpath); + } + + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken) + .ConfigureAwait(false); + await blockmarker.CommitAsync(result.TaskControl.ProgressToken).ConfigureAwait(false); + } + + private static async Task ScanForExistingSourceBlocks(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, RestoreResults result, RestoreHandlerMetadataStorage metadatastorage) + { + // Fill BLOCKS with data from known local source files + await using var blockmarker = await database.CreateBlockMarkerAsync(result.TaskControl.ProgressToken).ConfigureAwait(false); + var updateCount = 0L; + await foreach (var restorelist in database.GetFilesAndSourceBlocks(options.SkipMetadata, options.Blocksize, result.TaskControl.ProgressToken).ConfigureAwait(false)) + { + var targetpath = restorelist.TargetPath; + var targetfileid = restorelist.TargetFileID; + var patched = false; + try + { + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + + var folderpath = SystemIO.IO_OS.PathGetDirectoryName(targetpath); + if (!options.Dryrun && !SystemIO.IO_OS.DirectoryExists(folderpath)) + { + Logging.Log.WriteWarningMessage(LOGTAG, "CreateMissingFolder", null, "Creating missing folder {0} for file {1}", folderpath, targetpath); + SystemIO.IO_OS.DirectoryCreate(folderpath); + } + + using (var file = options.Dryrun ? null : SystemIO.IO_OS.FileOpenWrite(targetpath)) + await foreach (var targetblock in restorelist.Blocks(result.TaskControl.ProgressToken).ConfigureAwait(false)) + { + await foreach (var source in targetblock.BlockSources(result.TaskControl.ProgressToken).ConfigureAwait(false)) + { + try + { + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + + if (SystemIO.IO_OS.FileExists(source.Path)) + { + if (source.IsMetadata) + { + // TODO: Handle this by reconstructing + // metadata from file and checking the hash + + continue; + } + else + { + using var sourcefile = SystemIO.IO_OS.FileOpenRead(source.Path); + sourcefile.Position = source.Offset; + int size = Library.Utility.Utility.ForceStreamRead(sourcefile, blockbuffer, blockbuffer.Length); + if (size == targetblock.Size) + { + var key = Convert.ToBase64String(hasher.ComputeHash(blockbuffer, 0, size)); + if (key == targetblock.Hash) + { + if (!options.Dryrun) + { + if (targetblock.IsMetadata) + metadatastorage.Add(targetpath, new System.IO.MemoryStream(blockbuffer, 0, size)); + else + { + file.Position = targetblock.Offset; + file.Write(blockbuffer, 0, size); } } + + await blockmarker + .SetBlockRestored(targetfileid, targetblock.Index, key, targetblock.Size, false, result.TaskControl.ProgressToken) + .ConfigureAwait(false); + patched = true; + break; } } } } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with data from local file \"{1}\", message: {2}", targetpath, source.Path, ex.Message); - if (ex.IsAbortException()) - throw; - } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with data from local file \"{1}\", message: {2}", targetpath, source.Path, ex.Message); + if (ex.IsAbortException()) + throw; } } + } - if ((++updateCount) % 20 == 0) - blockmarker.UpdateProcessed(result.OperationProgressUpdater); - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with local data, message: {1}", targetpath, ex.Message); - if (options.UnittestMode) - throw; - } - - if (patched) - Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is patched with some local data: {0}", targetpath); - else - Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is not patched any local data: {0}", targetpath); - - if (patched && options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchWithLocal", string.Format("Would patch file with local data: {0}", targetpath)); + if ((++updateCount) % 20 == 0) + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken) + .ConfigureAwait(false); + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "PatchingFileLocalFailed", ex, "Failed to patch file: \"{0}\" with local data, message: {1}", targetpath, ex.Message); + if (options.UnittestMode) + throw; } - blockmarker.UpdateProcessed(result.OperationProgressUpdater); - blockmarker.Commit(); + if (patched) + Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is patched with some local data: {0}", targetpath); + else + Logging.Log.WriteVerboseMessage(LOGTAG, "FilePatchedWithLocal", "Target file is not patched any local data: {0}", targetpath); + + if (patched && options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPatchWithLocal", string.Format("Would patch file with local data: {0}", targetpath)); } + + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken) + .ConfigureAwait(false); + await blockmarker.CommitAsync(result.TaskControl.ProgressToken).ConfigureAwait(false); } - private static void PrepareBlockAndFileList(LocalRestoreDatabase database, Options options, Library.Utility.IFilter filter, RestoreResults result) + private static async Task PrepareBlockAndFileList(LocalRestoreDatabase database, Options options, Library.Utility.IFilter filter, RestoreResults result) { // Create a temporary table FILES by selecting the files from fileset that matches a specific operation id // Delete all entries from the temp table that are excluded by the filter(s) using (new Logging.Timer(LOGTAG, "PrepareRestoreFileList", "PrepareRestoreFileList")) { - var c = database.PrepareRestoreFilelist(options.Time, options.Version, filter); + var c = await database + .PrepareRestoreFilelist(options.Time, options.Version, filter, result.TaskControl.ProgressToken) + .ConfigureAwait(false); result.OperationProgressUpdater.UpdatefileCount(c.Item1, c.Item2, true); } @@ -904,25 +940,33 @@ namespace Duplicati.Library.Main.Operation if (!string.IsNullOrEmpty(options.Restorepath)) { // Find the largest common prefix - var largest_prefix = options.DontCompressRestorePaths ? "" : database.GetLargestPrefix(); + var largest_prefix = options.DontCompressRestorePaths + ? "" : + await database.GetLargestPrefix(result.TaskControl.ProgressToken).ConfigureAwait(false); Logging.Log.WriteVerboseMessage(LOGTAG, "MappingRestorePath", "Mapping restore path prefix to \"{0}\" to \"{1}\"", largest_prefix, Util.AppendDirSeparator(options.Restorepath)); // Set the target paths, special care with C:\ and / - database.SetTargetPaths(largest_prefix, Util.AppendDirSeparator(options.Restorepath)); + await database + .SetTargetPaths(largest_prefix, Util.AppendDirSeparator(options.Restorepath), result.TaskControl.ProgressToken) + .ConfigureAwait(false); } else { - database.SetTargetPaths("", ""); + await database.SetTargetPaths("", "", result.TaskControl.ProgressToken).ConfigureAwait(false); } // Create a temporary table BLOCKS that lists all blocks that needs to be recovered using (new Logging.Timer(LOGTAG, "FindMissingBlocks", "FindMissingBlocks")) - database.FindMissingBlocks(options.SkipMetadata); + await database + .FindMissingBlocks(options.SkipMetadata, result.TaskControl.ProgressToken) + .ConfigureAwait(false); // Create temporary tables and triggers that automatically track progress using (new Logging.Timer(LOGTAG, "CreateProgressTracker", "CreateProgressTracker")) - database.CreateProgressTracker(false); + await database + .CreateProgressTracker(false, result.TaskControl.ProgressToken) + .ConfigureAwait(false); } @@ -940,7 +984,7 @@ namespace Duplicati.Library.Main.Operation SystemIO.IO_OS.DirectoryCreate(options.Restorepath); } - foreach (var folder in database.GetTargetFolders()) + await foreach (var folder in database.GetTargetFolders(result.TaskControl.ProgressToken).ConfigureAwait(false)) { try { @@ -971,208 +1015,224 @@ namespace Duplicati.Library.Main.Operation private static async Task ScanForExistingTargetBlocks(LocalRestoreDatabase database, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm blockhasher, System.Security.Cryptography.HashAlgorithm filehasher, Options options, RestoreResults result) { // Scan existing files for existing BLOCKS - using (var blockmarker = database.CreateBlockMarker()) + await using var blockmarker = await database.CreateBlockMarkerAsync(result.TaskControl.ProgressToken).ConfigureAwait(false); + var updateCount = 0L; + await foreach (var restorelist in database.GetExistingFilesWithBlocks(result.TaskControl.ProgressToken).ConfigureAwait(false)) { - var updateCount = 0L; - foreach (var restorelist in database.GetExistingFilesWithBlocks()) + var rename = !options.Overwrite; + var targetpath = restorelist.TargetPath; + var targetfileid = restorelist.TargetFileID; + var targetfilehash = restorelist.TargetHash; + var targetfilelength = restorelist.Length; + if (SystemIO.IO_OS.FileExists(targetpath)) { - var rename = !options.Overwrite; - var targetpath = restorelist.TargetPath; - var targetfileid = restorelist.TargetFileID; - var targetfilehash = restorelist.TargetHash; - var targetfilelength = restorelist.Length; - if (SystemIO.IO_OS.FileExists(targetpath)) + try + { + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + + var currentfilelength = SystemIO.IO_OS.FileLength(targetpath); + var wasTruncated = false; + + // Adjust file length in overwrite mode if necessary (smaller is ok, will be extended during restore) + // We do it before scanning for blocks. This allows full verification on files that only needs to + // be truncated (i.e. forthwritten log files). + if (!rename && currentfilelength > targetfilelength) + { + var currentAttr = SystemIO.IO_OS.GetFileAttributes(targetpath); + if ((currentAttr & System.IO.FileAttributes.ReadOnly) != 0) // clear readonly attribute + { + if (options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldResetReadOnlyAttribute", "Would reset read-only attribute on file: {0}", targetpath); + else SystemIO.IO_OS.SetFileAttributes(targetpath, currentAttr & ~System.IO.FileAttributes.ReadOnly); + } + if (options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldTruncateFile", "Would truncate file '{0}' to length of {1:N0} bytes", targetpath, targetfilelength); + else + { + using (var file = SystemIO.IO_OS.FileOpenWrite(targetpath)) + file.SetLength(targetfilelength); + currentfilelength = targetfilelength; + } + wasTruncated = true; + } + + // If file size does not match and we have to rename on conflict, + // the whole scan can be skipped here because all blocks have to be restored anyway. + // For the other cases, we will check block and and file hashes and look for blocks + // to be restored and files that can already be verified. + if (!rename || currentfilelength == targetfilelength) + { + // a file hash for verification will only be necessary if the file has exactly + // the wanted size so we have a chance to already mark the file as data-verified. + bool calcFileHash = (currentfilelength == targetfilelength); + if (calcFileHash) filehasher.Initialize(); + + using (var file = SystemIO.IO_OS.FileOpenRead(targetpath)) + using (var block = new Blockprocessor(file, blockbuffer)) + await foreach (var targetblock in restorelist.Blocks(result.TaskControl.ProgressToken).ConfigureAwait(false)) + { + var size = block.Readblock(); + if (size <= 0) + break; + + //TODO: Handle Metadata + + bool blockhashmatch = false; + if (size == targetblock.Size) + { + // Parallelize file hash calculation on rename. Running read-only on same array should not cause conflicts or races. + // Actually, in future always calculate the file hash and mark the file data as already verified. + + System.Threading.Tasks.Task calcFileHashTask = null; + if (calcFileHash) + calcFileHashTask = System.Threading.Tasks.Task.Run( + () => filehasher.TransformBlock(blockbuffer, 0, size, blockbuffer, 0)); + + var key = Convert.ToBase64String(blockhasher.ComputeHash(blockbuffer, 0, size)); + + if (calcFileHashTask != null) await calcFileHashTask.ConfigureAwait(false); // wait because blockbuffer will be overwritten. + + if (key == targetblock.Hash) + { + await blockmarker + .SetBlockRestored(targetfileid, targetblock.Index, key, size, false, result.TaskControl.ProgressToken) + .ConfigureAwait(false); + blockhashmatch = true; + } + } + if (calcFileHash && !blockhashmatch) // will not be necessary anymore + { + filehasher.TransformFinalBlock(blockbuffer, 0, 0); // So a new initialize will not throw + calcFileHash = false; + if (rename) // file does not match. So break. + break; + } + } + + bool fullfilehashmatch = false; + if (calcFileHash) // now check if files are identical + { + filehasher.TransformFinalBlock(blockbuffer, 0, 0); + var filekey = Convert.ToBase64String(filehasher.Hash); + fullfilehashmatch = (filekey == targetfilehash); + } + + if (!rename && !fullfilehashmatch && !wasTruncated) // Reset read-only attribute (if set) to overwrite + { + var currentAttr = SystemIO.IO_OS.GetFileAttributes(targetpath); + if ((currentAttr & System.IO.FileAttributes.ReadOnly) != 0) + { + if (options.Dryrun) + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldResetReadOnlyAttribyte", "Would reset read-only attribute on file: {0}", targetpath); + else SystemIO.IO_OS.SetFileAttributes(targetpath, currentAttr & ~System.IO.FileAttributes.ReadOnly); + } + } + + if (fullfilehashmatch) + { + //TODO: Check metadata to trigger rename? If metadata changed, it will still be restored for the file in-place. + await blockmarker + .SetFileDataVerified(targetfileid, result.TaskControl.ProgressToken) + .ConfigureAwait(false); + Logging.Log.WriteVerboseMessage(LOGTAG, "TargetExistsInCorrectVersion", "Target file exists{1} and is correct version: {0}", targetpath, wasTruncated ? " (but was truncated)" : ""); + rename = false; + } + else if (rename) + { + // The new file will have none of the correct blocks, + // even if the scanned file had some + await blockmarker + .SetAllBlocksMissing(targetfileid, result.TaskControl.ProgressToken) + .ConfigureAwait(false); + } + } + + if ((++updateCount) % 20 == 0) + { + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken) + .ConfigureAwait(false); + if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + return; + } + } + catch (Exception ex) + { + Logging.Log.WriteWarningMessage(LOGTAG, "TargetFileReadError", ex, "Failed to read target file: \"{0}\", message: {1}", targetpath, ex.Message); + if (ex.IsAbortException()) + throw; + if (options.UnittestMode) + throw; + } + } + else + { + Logging.Log.WriteVerboseMessage(LOGTAG, "MissingTargetFile", "Target file does not exist: {0}", targetpath); + rename = false; + } + + if (rename) + { + //Select a new filename + var ext = SystemIO.IO_OS.PathGetExtension(targetpath) ?? ""; + if (!string.IsNullOrEmpty(ext) && !ext.StartsWith(".", StringComparison.Ordinal)) + ext = "." + ext; + + // First we try with a simple date append, assuming that there are not many conflicts there + var newname = SystemIO.IO_OS.PathChangeExtension(targetpath, null) + "." + database.RestoreTime.ToLocalTime().ToString("yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture); + var tr = newname + ext; + var c = 0; + while (SystemIO.IO_OS.FileExists(tr) && c < 1000) { try { - if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; + // If we have a file with the correct name, + // it is most likely the file we want + filehasher.Initialize(); - var currentfilelength = SystemIO.IO_OS.FileLength(targetpath); - var wasTruncated = false; + string key; + using (var file = SystemIO.IO_OS.FileOpenRead(tr)) + key = Convert.ToBase64String(filehasher.ComputeHash(file)); - // Adjust file length in overwrite mode if necessary (smaller is ok, will be extended during restore) - // We do it before scanning for blocks. This allows full verification on files that only needs to - // be truncated (i.e. forthwritten log files). - if (!rename && currentfilelength > targetfilelength) + if (key == targetfilehash) { - var currentAttr = SystemIO.IO_OS.GetFileAttributes(targetpath); - if ((currentAttr & System.IO.FileAttributes.ReadOnly) != 0) // clear readonly attribute - { - if (options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldResetReadOnlyAttribute", "Would reset read-only attribute on file: {0}", targetpath); - else SystemIO.IO_OS.SetFileAttributes(targetpath, currentAttr & ~System.IO.FileAttributes.ReadOnly); - } - if (options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldTruncateFile", "Would truncate file '{0}' to length of {1:N0} bytes", targetpath, targetfilelength); - else - { - using (var file = SystemIO.IO_OS.FileOpenWrite(targetpath)) - file.SetLength(targetfilelength); - currentfilelength = targetfilelength; - } - wasTruncated = true; - } - - // If file size does not match and we have to rename on conflict, - // the whole scan can be skipped here because all blocks have to be restored anyway. - // For the other cases, we will check block and and file hashes and look for blocks - // to be restored and files that can already be verified. - if (!rename || currentfilelength == targetfilelength) - { - // a file hash for verification will only be necessary if the file has exactly - // the wanted size so we have a chance to already mark the file as data-verified. - bool calcFileHash = (currentfilelength == targetfilelength); - if (calcFileHash) filehasher.Initialize(); - - using (var file = SystemIO.IO_OS.FileOpenRead(targetpath)) - using (var block = new Blockprocessor(file, blockbuffer)) - foreach (var targetblock in restorelist.Blocks) - { - var size = block.Readblock(); - if (size <= 0) - break; - - //TODO: Handle Metadata - - bool blockhashmatch = false; - if (size == targetblock.Size) - { - // Parallelize file hash calculation on rename. Running read-only on same array should not cause conflicts or races. - // Actually, in future always calculate the file hash and mark the file data as already verified. - - System.Threading.Tasks.Task calcFileHashTask = null; - if (calcFileHash) - calcFileHashTask = System.Threading.Tasks.Task.Run( - () => filehasher.TransformBlock(blockbuffer, 0, size, blockbuffer, 0)); - - var key = Convert.ToBase64String(blockhasher.ComputeHash(blockbuffer, 0, size)); - - if (calcFileHashTask != null) await calcFileHashTask.ConfigureAwait(false); // wait because blockbuffer will be overwritten. - - if (key == targetblock.Hash) - { - blockmarker.SetBlockRestored(targetfileid, targetblock.Index, key, size, false); - blockhashmatch = true; - } - } - if (calcFileHash && !blockhashmatch) // will not be necessary anymore - { - filehasher.TransformFinalBlock(blockbuffer, 0, 0); // So a new initialize will not throw - calcFileHash = false; - if (rename) // file does not match. So break. - break; - } - } - - bool fullfilehashmatch = false; - if (calcFileHash) // now check if files are identical - { - filehasher.TransformFinalBlock(blockbuffer, 0, 0); - var filekey = Convert.ToBase64String(filehasher.Hash); - fullfilehashmatch = (filekey == targetfilehash); - } - - if (!rename && !fullfilehashmatch && !wasTruncated) // Reset read-only attribute (if set) to overwrite - { - var currentAttr = SystemIO.IO_OS.GetFileAttributes(targetpath); - if ((currentAttr & System.IO.FileAttributes.ReadOnly) != 0) - { - if (options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldResetReadOnlyAttribyte", "Would reset read-only attribute on file: {0}", targetpath); - else SystemIO.IO_OS.SetFileAttributes(targetpath, currentAttr & ~System.IO.FileAttributes.ReadOnly); - } - } - - if (fullfilehashmatch) - { - //TODO: Check metadata to trigger rename? If metadata changed, it will still be restored for the file in-place. - blockmarker.SetFileDataVerified(targetfileid); - Logging.Log.WriteVerboseMessage(LOGTAG, "TargetExistsInCorrectVersion", "Target file exists{1} and is correct version: {0}", targetpath, wasTruncated ? " (but was truncated)" : ""); - rename = false; - } - else if (rename) - { - // The new file will have none of the correct blocks, - // even if the scanned file had some - blockmarker.SetAllBlocksMissing(targetfileid); - } - } - - if ((++updateCount) % 20 == 0) - { - blockmarker.UpdateProcessed(result.OperationProgressUpdater); - if (!await result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) - return; + //TODO: Also needs metadata check to make correct decision. + // We stick to the policy to restore metadata in place, if data ok. So, metadata block may be restored. + await blockmarker + .SetAllBlocksRestored(targetfileid, false, result.TaskControl.ProgressToken) + .ConfigureAwait(false); + await blockmarker + .SetFileDataVerified(targetfileid, result.TaskControl.ProgressToken) + .ConfigureAwait(false); + break; } } catch (Exception ex) { - Logging.Log.WriteWarningMessage(LOGTAG, "TargetFileReadError", ex, "Failed to read target file: \"{0}\", message: {1}", targetpath, ex.Message); - if (ex.IsAbortException()) - throw; + Logging.Log.WriteWarningMessage(LOGTAG, "FailedToReadRestoreTarget", ex, "Failed to read candidate restore target {0}", tr); if (options.UnittestMode) throw; } - } - else - { - Logging.Log.WriteVerboseMessage(LOGTAG, "MissingTargetFile", "Target file does not exist: {0}", targetpath); - rename = false; + tr = newname + " (" + (c++).ToString() + ")" + ext; } - if (rename) - { - //Select a new filename - var ext = SystemIO.IO_OS.PathGetExtension(targetpath) ?? ""; - if (!string.IsNullOrEmpty(ext) && !ext.StartsWith(".", StringComparison.Ordinal)) - ext = "." + ext; - - // First we try with a simple date append, assuming that there are not many conflicts there - var newname = SystemIO.IO_OS.PathChangeExtension(targetpath, null) + "." + database.RestoreTime.ToLocalTime().ToString("yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture); - var tr = newname + ext; - var c = 0; - while (SystemIO.IO_OS.FileExists(tr) && c < 1000) - { - try - { - // If we have a file with the correct name, - // it is most likely the file we want - filehasher.Initialize(); - - string key; - using (var file = SystemIO.IO_OS.FileOpenRead(tr)) - key = Convert.ToBase64String(filehasher.ComputeHash(file)); - - if (key == targetfilehash) - { - //TODO: Also needs metadata check to make correct decision. - // We stick to the policy to restore metadata in place, if data ok. So, metadata block may be restored. - blockmarker.SetAllBlocksRestored(targetfileid, false); - blockmarker.SetFileDataVerified(targetfileid); - break; - } - } - catch (Exception ex) - { - Logging.Log.WriteWarningMessage(LOGTAG, "FailedToReadRestoreTarget", ex, "Failed to read candidate restore target {0}", tr); - if (options.UnittestMode) - throw; - } - tr = newname + " (" + (c++).ToString() + ")" + ext; - } - - newname = tr; - - Logging.Log.WriteVerboseMessage(LOGTAG, "TargetFileRetargeted", "Target file exists and will be restored to: {0}", newname); - database.UpdateTargetPath(targetfileid, newname); - } + newname = tr; + Logging.Log.WriteVerboseMessage(LOGTAG, "TargetFileRetargeted", "Target file exists and will be restored to: {0}", newname); + await database + .UpdateTargetPath(targetfileid, newname, result.TaskControl.ProgressToken) + .ConfigureAwait(false); } - blockmarker.UpdateProcessed(result.OperationProgressUpdater); - blockmarker.Commit(); } + + await blockmarker + .UpdateProcessed(result.OperationProgressUpdater, result.TaskControl.ProgressToken) + .ConfigureAwait(false); + await blockmarker + .CommitAsync(result.TaskControl.ProgressToken) + .ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs b/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs index bae0f7408..b8c4610d5 100644 --- a/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs +++ b/Duplicati/Library/Main/Operation/SearchEntriesHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable @@ -44,16 +44,22 @@ internal static class SearchEntriesHandler /// The offset to start searching from /// The maximum number of results to return /// A task representing the asynchronous operation - public static Task RunAsync(Options options, SearchFilesResults result, string[] paths, IFilter filter, long offset, long limit) + public static async Task RunAsync(Options options, SearchFilesResults result, string[] paths, IFilter filter, long offset, long limit) { if (!System.IO.File.Exists(options.Dbpath)) throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase"); - using var db = new Database.LocalListDatabase(options.Dbpath, options.SqlitePageCache); + await using var db = + await Database.LocalListDatabase.CreateAsync(options.Dbpath, null, result.TaskControl.ProgressToken) + .ConfigureAwait(false); long[]? filesetIds = null; if (!options.AllVersions) { - filesetIds = db.GetFilesetIDs(options.Time, options.Version).ToArray(); + filesetIds = await db + .GetFilesetIDs(options.Time, options.Version, false, result.TaskControl.ProgressToken) + .ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken) + .ConfigureAwait(false); + if (filesetIds.Length == 0) throw new UserInformationException("No filesets found", "NoFilesetsFound"); } @@ -61,7 +67,8 @@ internal static class SearchEntriesHandler if (paths != null) paths = paths.Select(path => Util.AppendDirSeparator(path)).ToArray(); - result.FileVersions = db.SearchEntries(paths, filter, filesetIds, offset, limit); - return Task.CompletedTask; + result.FileVersions = await db + .SearchEntries(paths, filter, filesetIds, offset, limit, result.TaskControl.ProgressToken) + .ConfigureAwait(false); } } diff --git a/Duplicati/Library/Main/Operation/TestHandler.cs b/Duplicati/Library/Main/Operation/TestHandler.cs index 0e123a923..b8b48581d 100644 --- a/Duplicati/Library/Main/Operation/TestHandler.cs +++ b/Duplicati/Library/Main/Operation/TestHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -40,12 +40,12 @@ namespace Duplicati.Library.Main.Operation private static readonly string LOGTAG = Logging.Log.LogTagFromType(); private readonly Options m_options; - private readonly TestResults m_results; + private readonly TestResults m_result; public TestHandler(Options options, TestResults results) { m_options = options; - m_results = results; + m_result = results; } public async Task RunAsync(long samples, IBackendManager backendManager) @@ -53,47 +53,56 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(LC.L("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist"); - using (var db = new LocalTestDatabase(m_options.Dbpath, m_options.SqlitePageCache)) - using (var rtr = new ReusableTransaction(db)) - { - Utility.UpdateOptionsFromDb(db, m_options, rtr.Transaction); - Utility.VerifyOptionsAndUpdateDatabase(db, m_options, rtr.Transaction); - db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks, rtr.Transaction); - await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_results.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, rtr.Transaction).ConfigureAwait(false); - await DoRunAsync(samples, db, rtr, backendManager).ConfigureAwait(false); - rtr.Commit("TestHandlerComplete"); - } + await using var db = await LocalTestDatabase.CreateAsync(m_options.Dbpath, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + await db + .VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + + await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await DoRunAsync(samples, db, backendManager).ConfigureAwait(false); + await db.Transaction + .CommitAsync("TestHandlerComplete") + .ConfigureAwait(false); } - public async Task DoRunAsync(long samples, LocalTestDatabase db, ReusableTransaction rtr, IBackendManager backend) + public async Task DoRunAsync(long samples, LocalTestDatabase db, IBackendManager backend) { - var files = db.SelectTestTargets(samples, m_options, rtr.Transaction).ToList(); + var files = await db + .SelectTestTargets(samples, m_options, m_result.TaskControl.ProgressToken) + .ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); - m_results.OperationProgressUpdater.UpdatePhase(OperationPhase.Verify_Running); - m_results.OperationProgressUpdater.UpdateProgress(0); + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Verify_Running); + m_result.OperationProgressUpdater.UpdateProgress(0); var progress = 0L; if (m_options.FullRemoteVerification != Options.RemoteTestStrategy.False) { var faultyIndexFiles = new List(); - await foreach (var (tf, hash, size, name) in backend.GetFilesOverlappedAsync(files, m_results.TaskControl.ProgressToken).ConfigureAwait(false)) + await foreach (var (tf, hash, size, name) in backend.GetFilesOverlappedAsync(files, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { var vol = new RemoteVolume(name, hash, size); try { - if (!await m_results.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - await backend.WaitForEmptyAsync(db, rtr.Transaction, m_results.TaskControl.ProgressToken).ConfigureAwait(false); - m_results.EndTime = DateTime.UtcNow; + await backend.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + m_result.EndTime = DateTime.UtcNow; return; } progress++; - m_results.OperationProgressUpdater.UpdateProgress((float)progress / files.Count); + m_result.OperationProgressUpdater.UpdateProgress((float)progress / files.Count); KeyValuePair>> res; using (tf) - res = TestVolumeInternals(db, rtr, vol, tf, m_options, m_options.FullBlockVerification ? 1.0 : 0.2); + res = await TestVolumeInternals(db, vol, tf, m_options, m_options.FullBlockVerification ? 1.0 : 0.2, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); var parsedInfo = VolumeBase.ParseFilename(vol.Name); if (parsedInfo.FileType == RemoteVolumeType.Index) @@ -111,13 +120,15 @@ namespace Duplicati.Library.Main.Operation faultyIndexFiles.Add(vol); } - m_results.AddResult(res.Key, res.Value); + m_result.AddResult(res.Key, res.Value); if (!string.IsNullOrWhiteSpace(vol.Hash) && vol.Size > 0) { if (res.Value == null || !res.Value.Any()) { - var rv = db.GetRemoteVolume(vol.Name, rtr.Transaction); + var rv = await db + .GetRemoteVolume(vol.Name, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); if (rv.ID < 0) { @@ -130,22 +141,26 @@ namespace Duplicati.Library.Main.Operation else { Logging.Log.WriteInformationMessage(LOGTAG, "CaptureHashAndSize", LC.L("Successfully captured hash and size for {0}, updating database", vol.Name)); - db.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Verified, vol.Size, vol.Hash, rtr.Transaction); + await db + .UpdateRemoteVolume(vol.Name, RemoteVolumeState.Verified, vol.Size, vol.Hash, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); } } } } } - db.UpdateVerificationCount(vol.Name, rtr.Transaction); + await db + .UpdateVerificationCount(vol.Name, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); } catch (Exception ex) { - m_results.AddResult(vol.Name, [new KeyValuePair(TestEntryStatus.Error, ex.Message)]); + m_result.AddResult(vol.Name, [new KeyValuePair(TestEntryStatus.Error, ex.Message)]); Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileProcessingFailed", ex, LC.L("Failed to process file {0}", vol.Name)); if (ex.IsAbortException()) { - m_results.EndTime = DateTime.UtcNow; + m_result.EndTime = DateTime.UtcNow; throw; } } @@ -156,7 +171,7 @@ namespace Duplicati.Library.Main.Operation if (m_options.ReplaceFaultyIndexFiles) { Logging.Log.WriteWarningMessage(LOGTAG, "FaultyIndexFiles", null, LC.L("Found {0} faulty index files, repairing now", faultyIndexFiles.Count)); - await ReplaceFaultyIndexFilesAsync(faultyIndexFiles, backend, db, rtr, m_results.TaskControl.ProgressToken).ConfigureAwait(false); + await ReplaceFaultyIndexFilesAsync(faultyIndexFiles, backend, db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } else Logging.Log.WriteWarningMessage(LOGTAG, "FaultyIndexFiles", null, LC.L("Found {0} faulty index files, remove the option {1} to repair them", faultyIndexFiles.Count, "--dont-replace-faulty-index-files")); @@ -168,25 +183,26 @@ namespace Duplicati.Library.Main.Operation { try { - if (!await m_results.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - m_results.EndTime = DateTime.UtcNow; + m_result.EndTime = DateTime.UtcNow; return; } progress++; - m_results.OperationProgressUpdater.UpdateProgress((float)progress / files.Count); + m_result.OperationProgressUpdater.UpdateProgress((float)progress / files.Count); if (f.Size <= 0 || string.IsNullOrWhiteSpace(f.Hash)) { Logging.Log.WriteInformationMessage(LOGTAG, "MissingRemoteHash", LC.L("No hash or size recorded for {0}, performing full verification", f.Name)); KeyValuePair>> res; - (var tf, var hash, var size) = await backend.GetWithInfoAsync(f.Name, f.Hash, f.Size, m_results.TaskControl.ProgressToken).ConfigureAwait(false); + (var tf, var hash, var size) = await backend.GetWithInfoAsync(f.Name, f.Hash, f.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false); using (tf) - res = TestVolumeInternals(db, rtr, f, tf, m_options, 1); - m_results.AddResult(res.Key, res.Value); + res = await TestVolumeInternals(db, f, tf, m_options, 1, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + m_result.AddResult(res.Key, res.Value); if (!string.IsNullOrWhiteSpace(hash) && size > 0) { @@ -199,54 +215,62 @@ namespace Duplicati.Library.Main.Operation else { Logging.Log.WriteInformationMessage(LOGTAG, "CapturedHashAndSize", LC.L("Successfully captured hash and size for {0}, updating database", f.Name)); - db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Verified, size, hash, rtr.Transaction); + await db + .UpdateRemoteVolume(f.Name, RemoteVolumeState.Verified, size, hash, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); } } } } else { - using (var tf = await backend.GetAsync(f.Name, f.Hash, f.Size, m_results.TaskControl.ProgressToken).ConfigureAwait(false)) + using (var tf = await backend.GetAsync(f.Name, f.Hash, f.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false)) { } } - db.UpdateVerificationCount(f.Name, rtr.Transaction); - m_results.AddResult(f.Name, []); + await db + .UpdateVerificationCount(f.Name, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + m_result.AddResult(f.Name, []); } catch (Exception ex) { - m_results.AddResult(f.Name, [new KeyValuePair(TestEntryStatus.Error, ex.Message)]); + m_result.AddResult(f.Name, [new KeyValuePair(TestEntryStatus.Error, ex.Message)]); Logging.Log.WriteErrorMessage(LOGTAG, "FailedToProcessFile", ex, LC.L("Failed to process file {0}", f.Name)); if (ex.IsAbortOrCancelException()) { - m_results.EndTime = DateTime.UtcNow; + m_result.EndTime = DateTime.UtcNow; throw; } } } } - m_results.EndTime = DateTime.UtcNow; + m_result.EndTime = DateTime.UtcNow; // generate a backup error status when any test is failing - except for 'extra' status // because these problems don't block database rebuilding. - var filtered = from n in m_results.Verifications where n.Value.Any(x => x.Key != TestEntryStatus.Extra) select n; + var filtered = from n in m_result.Verifications where n.Value.Any(x => x.Key != TestEntryStatus.Extra) select n; if (!filtered.Any()) { - Logging.Log.WriteInformationMessage(LOGTAG, "Test results", LC.L("Successfully verified {0} remote files", m_results.VerificationsActualLength)); + Logging.Log.WriteInformationMessage(LOGTAG, "Test results", LC.L("Successfully verified {0} remote files", m_result.VerificationsActualLength)); } else { - Logging.Log.WriteErrorMessage(LOGTAG, "Test results", null, LC.L("Verified {0} remote files with {1} problem(s)", m_results.VerificationsActualLength, filtered.Count())); + Logging.Log.WriteErrorMessage(LOGTAG, "Test results", null, LC.L("Verified {0} remote files with {1} problem(s)", m_result.VerificationsActualLength, filtered.Count())); } } /// - /// Tests the volume by examining the internal contents + /// Tests the volume by examining the internal contents. /// - /// The remote volume being examined - /// The path to the downloaded copy of the file - /// A value between 0 and 1 that indicates how many blocks are tested in a dblock file - public static KeyValuePair>> TestVolumeInternals(LocalTestDatabase db, ReusableTransaction rtr, IRemoteVolume vol, string tf, Options options, double sample_percent) + /// The local test database to use for verification. + /// The remote volume being examined. + /// The path to the downloaded copy of the file. + /// The options to use for the test. + /// A value between 0 and 1 that indicates how many blocks are tested in a dblock file. + /// A cancellation token to cancel the operation. + /// A task that when awaited returns a key-value pair where the key is the volume name and the value is a list of test results. + public static async Task>>> TestVolumeInternals(LocalTestDatabase db, IRemoteVolume vol, string tf, Options options, double sample_percent, CancellationToken cancellationToken) { var hashsize = HashFactory.HashSizeBytes(options.BlockHashAlgorithm); var parsedInfo = Volumes.VolumeBase.ParseFilename(vol.Name); @@ -255,15 +279,17 @@ namespace Duplicati.Library.Main.Operation switch (parsedInfo.FileType) { case RemoteVolumeType.Files: - //Compare with db and see if all files are accounted for + //Compare with db and see if all files are accounted for // with correct file hashes and blocklist hashes - using (var fl = db.CreateFilelist(vol.Name, rtr)) + await using (var fl = await db.CreateFilelist(vol.Name, cancellationToken).ConfigureAwait(false)) { using (var rd = new Volumes.FilesetVolumeReader(parsedInfo.CompressionModule, tf, options)) foreach (var f in rd.Files) - fl.Add(f.Path, f.Size, f.Hash, f.Metasize, f.Metahash, f.BlocklistHashes, f.Type, f.Time); + await fl + .Add(f.Path, f.Size, f.Hash, f.Metasize, f.Metahash, f.BlocklistHashes, f.Type, f.Time, cancellationToken) + .ConfigureAwait(false); - return new KeyValuePair>>(vol.Name, fl.Compare().ToList()); + return new KeyValuePair>>(vol.Name, await fl.Compare(cancellationToken).ToListAsync(cancellationToken: cancellationToken).ConfigureAwait(false)); } case RemoteVolumeType.Index: @@ -276,46 +302,69 @@ namespace Duplicati.Library.Main.Operation foreach (var v in rd.Volumes) { blocklinks.Add(new Tuple(v.Filename, v.Hash, v.Length)); - using (var bl = db.CreateBlocklist(v.Filename, rtr)) + await using (var bl = await db.CreateBlocklist(v.Filename, cancellationToken).ConfigureAwait(false)) { foreach (var h in v.Blocks) - bl.AddBlock(h.Key, h.Value); + await bl + .AddBlock(h.Key, h.Value, cancellationToken) + .ConfigureAwait(false); - combined.AddRange(bl.Compare()); + combined.AddRange( + await bl + .Compare(cancellationToken) + .ToListAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false) + ); } } if (options.IndexfilePolicy == Options.IndexFileStrategy.Full) { var hashesPerBlock = options.Blocksize / options.BlockhashSize; - using (var bl = db.CreateBlocklistHashList(vol.Name, rtr)) + await using (var bl = await db.CreateBlocklistHashList(vol.Name, cancellationToken).ConfigureAwait(false)) { foreach (var b in rd.BlockLists) - bl.AddBlockHash(b.Hash, b.Length); + await bl + .AddBlockHash(b.Hash, b.Length, cancellationToken) + .ConfigureAwait(false); - combined.AddRange(bl.Compare(hashesPerBlock, options.BlockhashSize, options.Blocksize)); + combined.AddRange( + await bl + .Compare(hashesPerBlock, options.BlockhashSize, options.Blocksize, cancellationToken) + .ToListAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false) + ); } } } // Compare with db and see that all blocklists are listed - using (var il = db.CreateIndexlist(vol.Name, rtr)) + await using (var il = await db.CreateIndexlist(vol.Name, cancellationToken).ConfigureAwait(false)) { foreach (var t in blocklinks) - il.AddBlockLink(t.Item1, t.Item2, t.Item3); + await il + .AddBlockLink(t.Item1, t.Item2, t.Item3, cancellationToken) + .ConfigureAwait(false); - combined.AddRange(il.Compare()); + combined.AddRange( + await il + .Compare(cancellationToken) + .ToArrayAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false) + ); } return new KeyValuePair>>(vol.Name, combined); case RemoteVolumeType.Blocks: using (var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm)) - using (var bl = db.CreateBlocklist(vol.Name, rtr)) + await using (var bl = await db.CreateBlocklist(vol.Name, cancellationToken).ConfigureAwait(false)) using (var rd = new Volumes.BlockVolumeReader(parsedInfo.CompressionModule, tf, options)) { //Verify that all blocks are in the file foreach (var b in rd.Blocks) - bl.AddBlock(b.Key, b.Value); + await bl + .AddBlock(b.Key, b.Value, cancellationToken) + .ConfigureAwait(false); //Select random blocks and verify their hashes match the filename and size var hashsamples = new List>(rd.Blocks); @@ -340,7 +389,15 @@ namespace Duplicati.Library.Main.Operation } } - return new KeyValuePair>>(vol.Name, changes.Union(bl.Compare().ToList())); + return new KeyValuePair>>( + vol.Name, + changes.Union( + await bl + .Compare(cancellationToken) + .ToListAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false) + ) + ); } } @@ -348,14 +405,17 @@ namespace Duplicati.Library.Main.Operation return new KeyValuePair>>(vol.Name, null); } - private async Task ReplaceFaultyIndexFilesAsync(List faultyIndexFiles, IBackendManager backendManager, LocalTestDatabase db, ReusableTransaction rtr, CancellationToken cancellationToken) + private async Task ReplaceFaultyIndexFilesAsync(List faultyIndexFiles, IBackendManager backendManager, LocalTestDatabase db, CancellationToken cancellationToken) { - using var repairdb = new LocalRepairDatabase(db); + await using var repairdb = + await LocalRepairDatabase.CreateAsync(db, null, cancellationToken) + .ConfigureAwait(false); + foreach (var vol in faultyIndexFiles) { - if (!await m_results.TaskControl.ProgressRendevouz().ConfigureAwait(false)) + if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false)) { - m_results.EndTime = DateTime.UtcNow; + m_result.EndTime = DateTime.UtcNow; return; } @@ -363,17 +423,19 @@ namespace Duplicati.Library.Main.Operation try { var w = newEntry = new IndexVolumeWriter(m_options); - await RepairHandler.RunRepairDindex(backendManager, repairdb, rtr, w, vol, m_options, cancellationToken).ConfigureAwait(false); + await RepairHandler.RunRepairDindex(backendManager, repairdb, w, vol, m_options, cancellationToken).ConfigureAwait(false); if (m_options.Dryrun) { Logging.Log.WriteDryrunMessage(LOGTAG, "ReplaceFaultyIndexFile", LC.L("Would replace faulty index file {0} with {1}", vol.Name, w.RemoteFilename)); } else { - await backendManager.DeleteAsync(vol.Name, vol.Size, true, m_results.TaskControl.ProgressToken).ConfigureAwait(false); - await backendManager.WaitForEmptyAsync(repairdb, rtr.Transaction, m_results.TaskControl.ProgressToken).ConfigureAwait(false); - rtr.Commit("ReplaceFaultyIndexFileCommit"); - m_results.RemoveResult(vol.Name); + await backendManager.DeleteAsync(vol.Name, vol.Size, true, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await backendManager.WaitForEmptyAsync(repairdb, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await repairdb.Transaction + .CommitAsync("ReplaceFaultyIndexFileCommit", token: cancellationToken) + .ConfigureAwait(false); + m_result.RemoveResult(vol.Name); } } catch (Exception ex) diff --git a/Duplicati/Library/Main/Operation/VacuumHandler.cs b/Duplicati/Library/Main/Operation/VacuumHandler.cs index 2c376e716..ef90425a7 100644 --- a/Duplicati/Library/Main/Operation/VacuumHandler.cs +++ b/Duplicati/Library/Main/Operation/VacuumHandler.cs @@ -1,27 +1,25 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. + using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Threading.Tasks; namespace Duplicati.Library.Main.Operation @@ -37,15 +35,14 @@ namespace Duplicati.Library.Main.Operation m_result = result; } - public virtual Task RunAsync() - => Task.Run(() => - { - using (var db = new Database.LocalDatabase(m_options.Dbpath, "Vacuum", false, m_options.SqlitePageCache)) - { - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Vacuum_Running); - db.Vacuum(); - m_result.EndTime = DateTime.UtcNow; - } - }); + public virtual async Task RunAsync() + { + await using var db = + await Database.LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Vacuum", false, null, m_result.TaskControl.ProgressToken) + .ConfigureAwait(false); + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Vacuum_Running); + await db.Vacuum(m_result.TaskControl.ProgressToken).ConfigureAwait(false); + m_result.EndTime = DateTime.UtcNow; + } } } diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index 191cf88c7..5fcad330b 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -176,11 +176,6 @@ namespace Duplicati.Library.Main /// private static readonly int DEFAULT_RESTORE_CHANNEL_BUFFER_SIZE = Environment.ProcessorCount; - /// - /// The default value for the size of the SQLite page cache - /// - private static readonly string DEFAULT_SQLITE_PAGE_CACHE_SIZE = MemoryInfo.GetTotalMemoryString(0.01, SQLiteLoader.MINIMUM_SQLITE_PAGE_CACHE_SIZE); // 1% of the total memory - /// /// An enumeration that describes the supported strategies for an optimization /// @@ -535,7 +530,6 @@ namespace Duplicati.Library.Main new CommandLineArgument("restore-volume-downloaders", CommandLineArgument.ArgumentType.Integer, Strings.Options.RestoreVolumeDownloadersShort, Strings.Options.RestoreVolumeDownloadersLong, DEFAULT_RESTORE_VOLUME_DOWNLOADERS.ToString()), new CommandLineArgument("restore-channel-buffer-size", CommandLineArgument.ArgumentType.Integer, Strings.Options.RestoreChannelBufferSizeShort, Strings.Options.RestoreChannelBufferSizeLong, DEFAULT_RESTORE_CHANNEL_BUFFER_SIZE.ToString()), new CommandLineArgument("internal-profiling", CommandLineArgument.ArgumentType.Boolean, Strings.Options.InternalProfilingShort, Strings.Options.InternalProfilingLong, "false"), - new CommandLineArgument("sqlite-page-cache", CommandLineArgument.ArgumentType.Size, Strings.Options.SqlitePageCacheShort, Strings.Options.SqlitePageCacheLong(SQLiteLoader.MINIMUM_SQLITE_PAGE_CACHE_SIZE), DEFAULT_SQLITE_PAGE_CACHE_SIZE), new CommandLineArgument("ignore-update-if-version-exists", CommandLineArgument.ArgumentType.Boolean, Strings.Options.IgnoreUpdateIfVersionExistsShort, Strings.Options.IgnoreUpdateIfVersionExistsLong, "false"), ]; @@ -1699,19 +1693,6 @@ namespace Duplicati.Library.Main /// public bool InternalProfiling => GetBool("internal-profiling"); - /// - /// Gets the size of file-blocks - /// - public long SqlitePageCache - { - get - { - var pagesize = GetSize("sqlite-page-cache", "kb", DEFAULT_SQLITE_PAGE_CACHE_SIZE); - return pagesize <= SQLiteLoader.MINIMUM_SQLITE_PAGE_CACHE_SIZE - ? 0 - : pagesize; - } - } /// /// Ignores the update if the version already exists in the database. /// diff --git a/Duplicati/Library/Main/ResultClasses.cs b/Duplicati/Library/Main/ResultClasses.cs index 578998c83..ed149145c 100644 --- a/Duplicati/Library/Main/ResultClasses.cs +++ b/Duplicati/Library/Main/ResultClasses.cs @@ -1,33 +1,35 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; -using Duplicati.Library.Interface; using System.Collections.Generic; -using Duplicati.Library.Main.Database; -using Duplicati.Library.Logging; using System.Linq; -using Newtonsoft.Json; using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.Interface; +using Duplicati.Library.Logging; +using Duplicati.Library.Main.Database; using Duplicati.Library.Main.Operation.Common; +using Newtonsoft.Json; namespace Duplicati.Library.Main { @@ -179,7 +181,7 @@ namespace Duplicati.Library.Main protected readonly BasicResults m_parent; protected System.Threading.Thread m_callerThread; - protected readonly object m_lock = new object(); + protected readonly SemaphoreSlim m_lock = new(1, 1); protected readonly Queue m_dbqueue; public virtual ParsedResultType ParsedResult @@ -258,20 +260,33 @@ namespace Duplicati.Library.Main } } - public void FlushLog(LocalDatabase db) + /// + /// Flushes the log messages to the database. + /// + /// The database to flush the log messages to. + /// The cancellation token to use. + /// A task that completes when the log messages have been flushed. + public async Task FlushLog(LocalDatabase db, CancellationToken token) { if (m_parent != null) - m_parent.FlushLog(db); + await m_parent.FlushLog(db, token).ConfigureAwait(false); else { - lock (m_lock) + await m_lock.WaitAsync(token).ConfigureAwait(false); + try { while (m_dbqueue.Count > 0) { var el = m_dbqueue.Dequeue(); - db.LogMessage(el.Type, el.Message, el.Exception, null); + await db + .LogMessage(el.Type, el.Message, el.Exception, token) + .ConfigureAwait(false); } } + finally + { + m_lock.Release(); + } } } diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index cb3f223cf..88db62acf 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -364,8 +364,6 @@ namespace Duplicati.Library.Main.Strings public static string RestoreChannelBufferSizeLong { get { return LC.L("Use this option to set the size of the buffers of the channels used during restore. The buffers are used to allow for better asynchronous communication between the processes in the restore flow. Increasing the buffer size may improve restore performance."); } } public static string InternalProfilingShort { get { return LC.L("Enable internal profiling"); } } public static string InternalProfilingLong { get { return LC.L("Use this option to enable internal profiling. Profiling is used to measure the performance of the internal code. The profiling data is written to the log file and can be used to identify performance bottlenecks."); } } - public static string SqlitePageCacheShort { get { return LC.L("Size of the SQLite page cache"); } } - public static string SqlitePageCacheLong(long minimumvalue) { return LC.L("Use this option to set the size of the SQLite page cache. The page cache is used to store the pages of the database in memory. Increasing the page cache size may improve performance, but will also increase memory usage. If the supplied value is the same or less than {0} bytes, the default SQLite cache value is used.", minimumvalue); } public static string IgnoreUpdateIfVersionExistsShort { get { return LC.L("Ignore update if version exists"); } } public static string IgnoreUpdateIfVersionExistsLong { get { return LC.L("Use this option to ignore the update if the version already exists. This can be used to avoid errors if asking to update the database with a version that already exists."); } } } diff --git a/Duplicati/Library/Main/Utility.cs b/Duplicati/Library/Main/Utility.cs index d77e621cc..0d01d04b7 100644 --- a/Duplicati/Library/Main/Utility.cs +++ b/Duplicati/Library/Main/Utility.cs @@ -1,32 +1,34 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.IO; -using Newtonsoft.Json; +using System.Linq; using System.Text; +using System.Threading.Tasks; +using Newtonsoft.Json; using Duplicati.Library.Main.Database; using Duplicati.Library.Utility; -using System.Linq; +using System.Threading; namespace Duplicati.Library.Main { @@ -98,15 +100,16 @@ namespace Duplicati.Library.Main } /// - /// Updates the options with settings from the data, if any + /// Updates the options with settings from the data, if any. /// - /// The database to read from - /// The options to update - /// The transaction to use, if any - internal static void UpdateOptionsFromDb(LocalDatabase db, Options options, System.Data.IDbTransaction transaction = null) + /// The database to read from. + /// The options to update. + /// A cancellation token to cancel the operation. + /// A task that completes when the options have been updated. + internal static async Task UpdateOptionsFromDb(LocalDatabase db, Options options, CancellationToken cancellationToken) { string n = null; - var opts = db.GetDbOptions(transaction); + var opts = await db.GetDbOptions(cancellationToken).ConfigureAwait(false); if (opts.ContainsKey("blocksize") && (!options.RawOptions.TryGetValue("blocksize", out n) || string.IsNullOrEmpty(n))) options.RawOptions["blocksize"] = opts["blocksize"] + "b"; @@ -117,23 +120,32 @@ namespace Duplicati.Library.Main } /// - /// Checks if the database contains options that need to be verified, such as the blocksize + /// Checks if the database contains options that need to be verified, such as the blocksize. /// - /// The database to check - /// true if the database contains options that need to be verified; false otherwise - internal static bool ContainsOptionsForVerification(LocalDatabase db) + /// The database to check. + /// A cancellation token to cancel the operation. + /// A task that when awaited returns true if the database contains options that need to be verified; false otherwise. + internal static async Task ContainsOptionsForVerification(LocalDatabase db, CancellationToken cancellationToken) { - var opts = db.GetDbOptions(); - return new[] { "blocksize", "blockhash", "filehash", "passphrase" }.Any(opts.ContainsKey); + var opts = await db.GetDbOptions(cancellationToken).ConfigureAwait(false); + await db.Transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + return new[] { + "blocksize", + "blockhash", + "filehash", + "passphrase" + } + .Any(opts.ContainsKey); } /// - /// Verifies the parameters in the database, and updates the database if needed + /// Verifies the parameters in the database, and updates the database if needed. /// - /// The database to check - /// The options to verify - /// The transaction to use, if any - internal static void VerifyOptionsAndUpdateDatabase(LocalDatabase db, Options options, System.Data.IDbTransaction transaction = null) + /// The database to check. + /// The options to verify. + /// A cancellation token to cancel the operation. + /// A task that completes when the options have been verified and the database has been updated if needed. + internal static async Task VerifyOptionsAndUpdateDatabase(LocalDatabase db, Options options, CancellationToken cancellationToken) { var newDict = new Dictionary { @@ -141,7 +153,7 @@ namespace Duplicati.Library.Main { "blockhash", options.BlockHashAlgorithm }, { "filehash", options.FileHashAlgorithm } }; - var opts = db.GetDbOptions(transaction); + var opts = await db.GetDbOptions(cancellationToken).ConfigureAwait(false); if (options.NoEncryption) { @@ -162,7 +174,7 @@ namespace Duplicati.Library.Main newDict["passphrase-salt"] = salt; - // We avoid storing the passphrase directly, + // We avoid storing the passphrase directly, // instead we salt and rehash repeatedly newDict.Add("passphrase", Library.Utility.Utility.ByteArrayAsHexString(Library.Utility.Utility.RepeatedHashWithSalt(options.Passphrase, salt, 1200))); } @@ -191,7 +203,7 @@ namespace Duplicati.Library.Main } //Extra sanity check - if (db.GetBlocksLargerThan(options.Blocksize) > 0) + if (await db.GetBlocksLargerThan(options.Blocksize, cancellationToken).ConfigureAwait(false) > 0) throw new Duplicati.Library.Interface.UserInformationException("You have attempted to change the block-size on an existing backup, which is not supported. Please configure a new clean backup if you want to change the block-size.", "BlockSizeChangeNotSupported"); if (needsUpdate) @@ -201,7 +213,7 @@ namespace Duplicati.Library.Main if (!newDict.ContainsKey(k.Key)) newDict[k.Key] = k.Value; - db.SetDbOptions(newDict, transaction); + await db.SetDbOptions(newDict, cancellationToken).ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Volumes/BlockVolumeWriter.cs b/Duplicati/Library/Main/Volumes/BlockVolumeWriter.cs index 865c91807..4e0841891 100644 --- a/Duplicati/Library/Main/Volumes/BlockVolumeWriter.cs +++ b/Duplicati/Library/Main/Volumes/BlockVolumeWriter.cs @@ -1,27 +1,28 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.Linq; using System.Text; +using System.Threading.Tasks; using Duplicati.Library.Interface; namespace Duplicati.Library.Main.Volumes @@ -33,7 +34,7 @@ namespace Duplicati.Library.Main.Volumes public long Blocks { get { return m_blocks; } } public long SourceSize { get { return m_sourcesize; } } - + public override RemoteVolumeType FileType { get { return RemoteVolumeType.Blocks; } } public BlockVolumeWriter(Options options) @@ -41,14 +42,14 @@ namespace Duplicati.Library.Main.Volumes { } - public void AddBlock(string hash, byte[] data, int offset, int size, CompressionHint hint) + public async Task AddBlock(string hash, byte[] data, int offset, int size, CompressionHint hint) { m_blocks++; m_sourcesize += size; - //Filenames are encoded with "modified Base64 for URL" https://en.wikipedia.org/wiki/Base64#URL_applications, + //Filenames are encoded with "modified Base64 for URL" https://en.wikipedia.org/wiki/Base64#URL_applications, using (var s = m_compression.CreateFile(Library.Utility.Utility.Base64PlainToBase64Url(hash), hint, DateTime.UtcNow)) - s.Write(data, offset, size); + await s.WriteAsync(data.AsMemory(offset, size)).ConfigureAwait(false); } } } diff --git a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs index 79f384494..ec7e4ade7 100644 --- a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs +++ b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; @@ -28,6 +28,7 @@ using Duplicati.Library.Main.Operation.Common; using System.Threading.Tasks; using Duplicati.Library.Utility; using System.Data; +using System.Threading; namespace Duplicati.Library.Main.Volumes { @@ -82,17 +83,19 @@ namespace Duplicati.Library.Main.Volumes } } - public void AddFile(string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IEnumerable blocklisthashes, IEnumerable metablocklisthashes) + public async Task AddFile(string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IAsyncEnumerable blocklisthashes, IEnumerable metablocklisthashes) { - AddFileEntry(FilelistEntryType.File, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes, metablocklisthashes); + await AddFileEntry(FilelistEntryType.File, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes, metablocklisthashes) + .ConfigureAwait(false); } - public void AddAlternateStream(string name, string filehash, long size, DateTime lastmodified, string metahash, string metablockhash, long metasize, string blockhash, long blocksize, IEnumerable blocklisthashes, IEnumerable metablocklisthashes) + public async Task AddAlternateStream(string name, string filehash, long size, DateTime lastmodified, string metahash, string metablockhash, long metasize, string blockhash, long blocksize, IAsyncEnumerable blocklisthashes, IEnumerable metablocklisthashes) { - AddFileEntry(FilelistEntryType.AlternateStream, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes, metablocklisthashes); + await AddFileEntry(FilelistEntryType.AlternateStream, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes, metablocklisthashes) + .ConfigureAwait(false); } - private void AddFileEntry(FilelistEntryType type, string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IEnumerable blocklisthashes, IEnumerable metablocklisthashes) + private async Task AddFileEntry(FilelistEntryType type, string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IAsyncEnumerable blocklisthashes, IEnumerable metablocklisthashes) { m_filecount++; m_writer.WriteStartObject(); @@ -111,15 +114,15 @@ namespace Duplicati.Library.Main.Volumes if (blocklisthashes != null) { - //Slightly awkward, but we avoid writing if there are no entries - using (var en = blocklisthashes.GetEnumerator()) + //Slightly awkward, but we avoid writing if there are no entries + await using (var en = blocklisthashes.GetAsyncEnumerator()) { - if (en.MoveNext() && !string.IsNullOrEmpty(en.Current)) + if (await en.MoveNextAsync().ConfigureAwait(false) && !string.IsNullOrEmpty(en.Current)) { m_writer.WritePropertyName("blocklists"); m_writer.WriteStartArray(); m_writer.WriteValue(en.Current); - while (en.MoveNext()) + while (await en.MoveNextAsync().ConfigureAwait(false)) m_writer.WriteValue(en.Current); m_writer.WriteEndArray(); } @@ -234,24 +237,25 @@ namespace Duplicati.Library.Main.Volumes /// The database to check for clashes /// The options to use for the filename /// The starting time to probe from + /// The cancellation token to use for the operation /// The time to increment by each probe /// The maximum number of tries to probe /// The first unused filename - internal static Task ProbeUnusedFilenameName(DatabaseCommon database, Options options, DateTime start, TimeSpan increment = default, int maxTries = 60) - => ProbeUnusedFilenameName(name => database.GetRemoteVolumeIDAsync(name), options, start, increment, maxTries); + internal static Task ProbeUnusedFilenameName(DatabaseCommon database, Options options, DateTime start, CancellationToken cancellationToken, TimeSpan increment = default, int maxTries = 60) + => ProbeUnusedFilenameName((name, CancellationToken) => database.GetRemoteVolumeIDAsync(name, cancellationToken), options, start, cancellationToken, increment, maxTries); /// - /// Probes for an unused filename, using the current time as a starting point + /// Probes for an unused filename, using the current time as a starting point. /// - /// The database to check for clashes - /// The options to use for the filename - /// The transaction to use - /// The starting time to probe from - /// The time to increment by each probe - /// The maximum number of tries to probe - /// The first unused filename - internal static DateTime ProbeUnusedFilenameName(LocalDatabase database, IDbTransaction transaction, Options options, DateTime start, TimeSpan increment = default, int maxTries = 60) - => ProbeUnusedFilenameName((name) => Task.FromResult(database.GetRemoteVolumeID(name, transaction)), options, start, increment, maxTries).Await(); + /// The database to check for clashes. + /// The options to use for the filename. + /// The starting time to probe from. + /// The time to increment by each probe. + /// The maximum number of tries to probe. + /// A task that when awaited returns the first unused filename. + internal static async Task ProbeUnusedFilenameName(LocalDatabase database, Options options, DateTime start, CancellationToken cancellationToken, TimeSpan increment = default, int maxTries = 60) + => await ProbeUnusedFilenameName(database.GetRemoteVolumeID, options, start, cancellationToken, increment, maxTries) + .ConfigureAwait(false); /// /// Probes for an unused filename, using the current time as a starting point @@ -262,7 +266,7 @@ namespace Duplicati.Library.Main.Volumes /// The time to increment by each probe /// The maximum number of tries to probe /// The first unused filename - private static async Task ProbeUnusedFilenameName(Func> ProbeForId, Options options, DateTime start, TimeSpan increment = default, int maxTries = 60) + private static async Task ProbeUnusedFilenameName(Func> ProbeForId, Options options, DateTime start, CancellationToken cancellationToken, TimeSpan increment = default, int maxTries = 60) { if (increment == default) increment = TimeSpan.FromSeconds(1); @@ -272,7 +276,7 @@ namespace Duplicati.Library.Main.Volumes while (s < maxTries) { - var id = await ProbeForId(GenerateFilename(RemoteVolumeType.Files, options, null, time)).ConfigureAwait(false); + var id = await ProbeForId(GenerateFilename(RemoteVolumeType.Files, options, null, time), cancellationToken).ConfigureAwait(false); if (id < 0) return time; diff --git a/Duplicati/Library/RestAPI/Database/Connection.cs b/Duplicati/Library/RestAPI/Database/Connection.cs index 761a189f4..161c9e84c 100644 --- a/Duplicati/Library/RestAPI/Database/Connection.cs +++ b/Duplicati/Library/RestAPI/Database/Connection.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -108,7 +108,7 @@ namespace Duplicati.Server.Database public void ReWriteAllFieldsIfEncryptionChanged() { - // The token is automatically decrypted when the settings are loaded + // The token is automatically decrypted when the settings are loaded // In case the password has changed, this will fail and return the encrypted // hex-string, but will crash before reaching this point if (this.ApplicationSettings.EncryptedFields != m_encryptSensitiveFields) @@ -246,22 +246,25 @@ namespace Duplicati.Server.Database internal void SetMetadata(IDictionary values, long id, IDbTransaction? transaction) { lock (m_lock) - using (var tr = transaction == null ? m_connection.BeginTransactionSafe() : null) - { - OverwriteAndUpdateDb( - tr, - cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Metadata"" WHERE ""BackupID"" = @Id") - .SetParameterValue("@Id", id), - values ?? new Dictionary(), - cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Metadata"" (""BackupID"", ""Name"", ""Value"") VALUES (@BackupId, @Name, @Value)"), - (cmd, f) => cmd.SetParameterValue("@BackupId", id) - .SetParameterValue("@Name", f.Key) - .SetParameterValue("@Value", f.Value) - ); + { + var tr = transaction ?? m_connection.BeginTransaction(); + OverwriteAndUpdateDb( + tr, + cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Metadata"" WHERE ""BackupID"" = @Id") + .SetParameterValue("@Id", id), + values ?? new Dictionary(), + cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Metadata"" (""BackupID"", ""Name"", ""Value"") VALUES (@BackupId, @Name, @Value)"), + (cmd, f) => cmd.SetParameterValue("@BackupId", id) + .SetParameterValue("@Name", f.Key) + .SetParameterValue("@Value", f.Value) + ); - if (tr != null) - tr.Commit(); + if (transaction == null) + { + tr.Commit(); + tr.Dispose(); } + } } internal IFilter[] GetFilters(long id) @@ -282,23 +285,26 @@ namespace Duplicati.Server.Database internal void SetFilters(IEnumerable values, long id, IDbTransaction? transaction = null) { lock (m_lock) - using (var tr = transaction == null ? m_connection.BeginTransactionSafe() : null) - { - OverwriteAndUpdateDb( - tr, - cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Filter"" WHERE ""BackupID"" = @Id") - .SetParameterValue("@Id", id), - values, - cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Filter"" (""BackupID"", ""Order"", ""Include"", ""Expression"") VALUES (@Id, @Order, @Include, @Expression)"), - (cmd, f) => cmd.SetParameterValue("@Id", id) - .SetParameterValue("@Order", f.Order) - .SetParameterValue("@Include", f.Include) - .SetParameterValue("@Expression", f.Expression) - ); + { + var tr = transaction ?? m_connection.BeginTransaction(); + OverwriteAndUpdateDb( + tr, + cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Filter"" WHERE ""BackupID"" = @Id") + .SetParameterValue("@Id", id), + values, + cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Filter"" (""BackupID"", ""Order"", ""Include"", ""Expression"") VALUES (@Id, @Order, @Include, @Expression)"), + (cmd, f) => cmd.SetParameterValue("@Id", id) + .SetParameterValue("@Order", f.Order) + .SetParameterValue("@Include", f.Include) + .SetParameterValue("@Expression", f.Expression) + ); - if (tr != null) - tr.Commit(); + if (transaction == null) + { + tr.Commit(); + tr.Dispose(); } + } } public ISetting[] GetSettings(long id) @@ -320,37 +326,40 @@ namespace Duplicati.Server.Database internal void SetSettings(IEnumerable values, long id, IDbTransaction? transaction = null) { lock (m_lock) - using (var tr = transaction == null ? m_connection.BeginTransactionSafe() : null) + { + var tr = transaction ?? m_connection.BeginTransaction(); + if (m_encryptSensitiveFields) + values = values.Select(x => new Setting + { + Filter = x.Filter, + Name = x.Name, + Value = EncryptSensitiveFields(x.Name, x.Value, m_key) + }).ToList(); + + OverwriteAndUpdateDb( + tr, + cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Option"" WHERE ""BackupID"" = @Id") + .SetParameterValue("@Id", id), + values, + cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Option"" (""BackupID"", ""Filter"", ""Name"", ""Value"") VALUES (@BackupId, @Filter, @Name, @Value)"), + (cmd, f) => + { + if (PASSWORD_PLACEHOLDER.Equals(f.Value)) + throw new Exception("Attempted to save a property with the placeholder password"); + + cmd.SetParameterValue("@BackupId", id) + .SetParameterValue("@Filter", f.Filter ?? "") + .SetParameterValue("@Name", f.Name ?? "") + .SetParameterValue("@Value", f.Value ?? ""); + } + ); + + if (transaction == null) { - if (m_encryptSensitiveFields) - values = values.Select(x => new Setting - { - Filter = x.Filter, - Name = x.Name, - Value = EncryptSensitiveFields(x.Name, x.Value, m_key) - }).ToList(); - - OverwriteAndUpdateDb( - tr, - cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Option"" WHERE ""BackupID"" = @Id") - .SetParameterValue("@Id", id), - values, - cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Option"" (""BackupID"", ""Filter"", ""Name"", ""Value"") VALUES (@BackupId, @Filter, @Name, @Value)"), - (cmd, f) => - { - if (PASSWORD_PLACEHOLDER.Equals(f.Value)) - throw new Exception("Attempted to save a property with the placeholder password"); - - cmd.SetParameterValue("@BackupId", id) - .SetParameterValue("@Filter", f.Filter ?? "") - .SetParameterValue("@Name", f.Name ?? "") - .SetParameterValue("@Value", f.Value ?? ""); - } - ); - - if (tr != null) - tr.Commit(); + tr.Commit(); + tr.Dispose(); } + } } internal string?[] GetSources(long id) @@ -366,21 +375,24 @@ namespace Duplicati.Server.Database internal void SetSources(IEnumerable values, long id, IDbTransaction transaction) { lock (m_lock) - using (var tr = transaction == null ? m_connection.BeginTransactionSafe() : null) - { - OverwriteAndUpdateDb( - tr, - cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Source"" WHERE ""BackupID"" = @Id") - .SetParameterValue("@Id", id), - values, - cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Source"" (""BackupID"", ""Path"") VALUES (@BackupId, @Path)"), - (cmd, f) => cmd.SetParameterValue("@BackupId", id) - .SetParameterValue("@Path", f) - ); + { + var tr = transaction ?? m_connection.BeginTransaction(); + OverwriteAndUpdateDb( + tr, + cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Source"" WHERE ""BackupID"" = @Id") + .SetParameterValue("@Id", id), + values, + cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Source"" (""BackupID"", ""Path"") VALUES (@BackupId, @Path)"), + (cmd, f) => cmd.SetParameterValue("@BackupId", id) + .SetParameterValue("@Path", f) + ); - if (tr != null) - tr.Commit(); + if (transaction == null) + { + tr.Commit(); + tr.Dispose(); } + } } internal long[] GetBackupIDsForTags(string[] tags) @@ -624,7 +636,7 @@ namespace Duplicati.Server.Database { lock (m_lock) { - using (var tr = m_connection.BeginTransactionSafe()) + using (var tr = m_connection.BeginTransaction()) { using (var cmd = m_connection.CreateCommand(tr, @"UPDATE ""Backup"" SET ""DBPath""= @Dbpath WHERE ""ID""= @Id")) { @@ -665,7 +677,7 @@ namespace Duplicati.Server.Database throw new Exception("Unable to generate a unique database file name"); } - using (var tr = m_connection.BeginTransactionSafe()) + using (var tr = m_connection.BeginTransaction()) { OverwriteAndUpdateDb( tr, @@ -760,7 +772,7 @@ namespace Duplicati.Server.Database internal void AddOrUpdateSchedule(ISchedule item) { lock (m_lock) - using (var tr = m_connection.BeginTransactionSafe()) + using (var tr = m_connection.BeginTransaction()) { AddOrUpdateSchedule(item, tr); tr.Commit(); @@ -811,7 +823,7 @@ namespace Duplicati.Server.Database lock (m_lock) { - using (var tr = m_connection.BeginTransactionSafe()) + using (var tr = m_connection.BeginTransaction()) { var existing = GetScheduleIDsFromTags(new string[] { "ID=" + ID.ToString() }); if (existing.Any()) @@ -1000,7 +1012,7 @@ namespace Duplicati.Server.Database //Workaround to clean up the database after invalid settings update public void FixInvalidBackupId() { - using (var tr = m_connection.BeginTransactionSafe()) + using (var tr = m_connection.BeginTransaction()) using (var cmd = m_connection.CreateCommand(tr)) { cmd.SetCommandAndParameters(@"DELETE FROM ""Option"" WHERE ""BackupID"" = @BackupId") @@ -1051,7 +1063,7 @@ namespace Duplicati.Server.Database public void SetUISettings(string scheme, IDictionary values, IDbTransaction? transaction = null) { lock (m_lock) - using (var tr = transaction == null ? m_connection.BeginTransactionSafe() : null) + using (var tr = transaction == null ? m_connection.BeginTransaction() : null) { OverwriteAndUpdateDb( tr, @@ -1075,7 +1087,7 @@ namespace Duplicati.Server.Database public void UpdateUISettings(string scheme, IDictionary values, IDbTransaction? transaction = null) { lock (m_lock) - using (var tr = transaction == null ? m_connection.BeginTransactionSafe() : null) + using (var tr = transaction == null ? m_connection.BeginTransaction() : null) { OverwriteAndUpdateDb( tr, @@ -1128,7 +1140,7 @@ namespace Duplicati.Server.Database { var t = Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(purgeDate); - using (var tr = m_connection.BeginTransactionSafe()) + using (var tr = m_connection.BeginTransaction()) using (var cmd = m_connection.CreateCommand(tr)) { cmd.SetCommandAndParameters(@"DELETE FROM ""ErrorLog"" WHERE ""Timestamp"" < @Time") @@ -1207,7 +1219,7 @@ namespace Duplicati.Server.Database { if (transaction == null) { - using (var tr = m_connection.BeginTransactionSafe()) + using (var tr = m_connection.BeginTransaction()) { var r = DeleteFromDb(tablename, id, tr); tr.Commit(); diff --git a/Duplicati/Library/SQLiteHelper/DatabaseUpgrader.cs b/Duplicati/Library/SQLiteHelper/DatabaseUpgrader.cs index 259eccdcd..52cba922a 100644 --- a/Duplicati/Library/SQLiteHelper/DatabaseUpgrader.cs +++ b/Duplicati/Library/SQLiteHelper/DatabaseUpgrader.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -36,24 +36,24 @@ namespace Duplicati.Library.SQLiteHelper /// Updates should have the form "1.Sample upgrade.sql". /// When the database schema changes, simply put a new file into the folder /// and set it to be embedded in the binary. - /// - /// Additionally, it's possible to execute custom code before and after + /// + /// Additionally, it's possible to execute custom code before and after /// the SQL is executed. To set up a custom upgrade stage, add your /// code to DbUpgradesRegistry along with the DB version to apply it with. - /// + /// /// Even if all the DB upgrade code is handled in C#, you still have to add /// a dummy SQL file to indicate the version ID is already taken. - /// + /// /// Each upgrade file should ONLY upgrade from the previous version. /// If done correctly, a user may be upgrade from the very first version /// to the very latest. - /// - /// The Schema.sql file should ALWAYS have the latest schema, as that will + /// + /// The Schema.sql file should ALWAYS have the latest schema, as that will /// ensure that new installs do not run upgrades after installation. - /// Also remember to update the last line in Schema.sql to insert the + /// Also remember to update the last line in Schema.sql to insert the /// current version number in the version table. - /// - /// Currently no upgrades may contain semicolons, except as the SQL statement + /// + /// Currently no upgrades may contain semicolons, except as the SQL statement /// delimiter. /// public static class DatabaseUpgrader @@ -203,14 +203,21 @@ namespace Duplicati.Library.SQLiteHelper try { //See if the version table is present, - cmd.CommandText = "SELECT COUNT(*) FROM SQLITE_MASTER WHERE Name LIKE 'Version'"; + cmd.CommandText = @" + SELECT COUNT(*) + FROM SQLITE_MASTER + WHERE Name LIKE 'Version' + "; int count = Convert.ToInt32(cmd.ExecuteScalar()); if (count == 0) dbversion = -1; //Empty else if (count == 1) { - cmd.CommandText = "SELECT max(Version) FROM Version"; + cmd.CommandText = @" + SELECT max(Version) + FROM Version + "; dbversion = Convert.ToInt32(cmd.ExecuteScalar()); } else diff --git a/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj b/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj index 64ceb5dab..7019eecdf 100644 --- a/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj +++ b/Duplicati/Library/SQLiteHelper/Duplicati.Library.SQLiteHelper.csproj @@ -10,21 +10,7 @@ - - - - - - - - + diff --git a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs index 99f15e42b..73c50db51 100644 --- a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs +++ b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs @@ -1,52 +1,56 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. #nullable enable using System; +using System.Collections.Generic; using System.Globalization; +using System.Threading; +using System.Threading.Tasks; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; +using Duplicati.Library.Utility; namespace Duplicati.Library.SQLiteHelper { + /// + /// Provides methods to load and manage SQLite connections, including handling encrypted databases. + /// public static class SQLiteLoader { /// - /// The minimum value for the SQLite page cache size - /// - public const long MINIMUM_SQLITE_PAGE_CACHE_SIZE = 2048000L; - - /// - /// The tag used for logging + /// The tag used for logging. /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(SQLiteLoader)); /// - /// Helper method with logic to handle opening a database in possibly encrypted format + /// Helper method with logic to handle opening a database in possibly encrypted format. /// - /// The SQLite connection object + /// The SQLite connection object. /// The location of Duplicati's database. /// The password to use for decryption. - public static void OpenDatabase(System.Data.IDbConnection con, string databasePath, string? decryptionPassword) + /// A task that completes when the database is opened. + /// Thrown if the database cannot be opened or decrypted. + public static async Task OpenDatabaseAsync(Microsoft.Data.Sqlite.SqliteConnection con, string databasePath, string? decryptionPassword) { if (!string.IsNullOrWhiteSpace(decryptionPassword) && SQLiteRC4Decrypter.IsDatabaseEncrypted(databasePath)) { @@ -66,12 +70,13 @@ namespace Duplicati.Library.SQLiteHelper try { //Attempt to open in preferred state - OpenSQLiteFile(con, databasePath); - TestSQLiteFile(con); + await OpenSQLiteFileAsync(con, databasePath) + .ConfigureAwait(false); + await TestSQLiteFileAsync(con).ConfigureAwait(false); } catch { - try { con.Dispose(); } + try { await con.DisposeAsync().ConfigureAwait(false); } catch { } throw; @@ -82,22 +87,34 @@ namespace Duplicati.Library.SQLiteHelper } /// - /// Loads an SQLite connection instance and opening the database + /// Loads an SQLite connection instance and opening the database. /// /// The SQLite connection instance. - public static System.Data.IDbConnection LoadConnection() + /// + /// This method is synchronous and should be used when you need to load the connection immediately. It calls the asynchronous version and waits for it to complete. + /// + public static Microsoft.Data.Sqlite.SqliteConnection LoadConnection() { - System.Data.IDbConnection? con = null; + return LoadConnectionAsync().Await(); + } + + /// + /// Loads an SQLite connection instance and opening the database. + /// + /// A task that when awaited returns the SQLite connection instance. + public static async Task LoadConnectionAsync() + { + Microsoft.Data.Sqlite.SqliteConnection? con = null; SetEnvironmentVariablesForSQLiteTempDir(); try { - con = (System.Data.IDbConnection?)Activator.CreateInstance(Duplicati.Library.SQLiteHelper.SQLiteLoader.SQLiteConnectionType); + con = new Microsoft.Data.Sqlite.SqliteConnection(); } catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "FailedToLoadConnectionSQLite", ex, "Failed to load connection."); - DisposeConnection(con); + await DisposeConnectionAsync(con).ConfigureAwait(false); throw; } @@ -106,79 +123,109 @@ namespace Duplicati.Library.SQLiteHelper } /// - /// Applies user-supplied custom pragmas to the SQLite connection + /// Applies user-supplied custom pragmas to the SQLite connection. /// /// The connection to apply the pragmas to. - /// The page cache size to set. - /// The connection with the pragmas applied. - public static System.Data.IDbConnection ApplyCustomPragmas(System.Data.IDbConnection con, long pagecachesize) + /// A task that when awaited returns the connection with the pragmas applied. + public static async Task ApplyCustomPragmasAsync(Microsoft.Data.Sqlite.SqliteConnection con) { - var opts = Environment.GetEnvironmentVariable("CUSTOMSQLITEOPTIONS_DUPLICATI") ?? ""; - if (pagecachesize > MINIMUM_SQLITE_PAGE_CACHE_SIZE) - opts = string.Format(CultureInfo.InvariantCulture, "cache_size=-{0};{1}", pagecachesize / 1024L, opts); - - if (string.IsNullOrWhiteSpace(opts)) - return con; + Dictionary customOptions = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + { "synchronous", "NORMAL" }, // NORMAL is more performant than FULL (the default), but less safe, as it no longer blocks until the disk sync syscall to the OS has completed. + { "temp_store", "MEMORY" }, // Use memory for temporary storage to improve performance. + { "journal_mode", "WAL" }, // Use Write-Ahead Logging for better concurrency and performance. + { "cache_size", "-65536" }, // Set cache size to 64 MB (negative value means in KB, so -64000 = 64 MB). Default is 2000 pages, which is 2 MB (2000 * 1024 bytes). + { "mmap_size", "67108864" }, // 64 MB. + { "threads", "8" }, // Use 8 threads for parallel processing where applicable. + { "shared_cache", "true" } // + }; + // Override the default options with any custom options set in the environment variable. + var customOptionsEnv = Environment.GetEnvironmentVariable("CUSTOMSQLITEOPTIONS"); + if (!string.IsNullOrWhiteSpace(customOptionsEnv)) + { + foreach (var opt in customOptionsEnv.Split([';'], StringSplitOptions.RemoveEmptyEntries)) + { + var parts = opt.Split(new[] { '=' }, 2); + if (parts.Length == 2) + customOptions[parts[0].Trim()] = parts[1].Trim(); + } + } using (var cmd = con.CreateCommand()) - foreach (var opt in opts.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries)) + { + foreach (var (key, value) in customOptions) { + var opt = $"{key}={value}"; Logging.Log.WriteVerboseMessage(LOGTAG, "CustomSQLiteOption", @"Setting custom SQLite option '{0}'.", opt); try { cmd.CommandText = string.Format(CultureInfo.InvariantCulture, "PRAGMA {0}", opt); - cmd.ExecuteNonQuery(); + await cmd.ExecuteNonQueryAsync().ConfigureAwait(false); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "CustomSQLiteOption", ex, @"Error setting custom SQLite option '{0}'.", opt); } } + } return con; } /// - /// Loads an SQLite connection instance and opening the database + /// Loads an SQLite connection instance and opening the database with a specified page cache size. /// - /// The SQLite connection instance. /// The optional path to the database. - /// The page cache size to set. - public static System.Data.IDbConnection LoadConnection(string targetpath, long pagecachesize) + /// The SQLite connection instance. + /// + /// This method is synchronous and should be used when you need to load the connection immediately. It calls the asynchronous version and waits for it to complete. + /// + public static Microsoft.Data.Sqlite.SqliteConnection LoadConnection(string targetpath) + { + return LoadConnectionAsync(targetpath).Await(); + } + + /// + /// Loads an SQLite connection instance and opening the database. + /// + /// The optional path to the database. + /// A task that when waited returns the SQLite connection instance. + public static async Task LoadConnectionAsync(string targetpath) { if (string.IsNullOrWhiteSpace(targetpath)) throw new ArgumentNullException(nameof(targetpath)); - var con = LoadConnection(); + var con = await LoadConnectionAsync().ConfigureAwait(false); try { - OpenSQLiteFile(con, targetpath); + await OpenSQLiteFileAsync(con, targetpath).ConfigureAwait(false); } catch (Exception ex) { Logging.Log.WriteErrorMessage(LOGTAG, "FailedToLoadConnectionSQLite", ex, @"Failed to load connection with path '{0}'.", targetpath); - DisposeConnection(con); + await DisposeConnectionAsync(con).ConfigureAwait(false); throw; } // set custom Sqlite options - return ApplyCustomPragmas(con, pagecachesize); + return await ApplyCustomPragmasAsync(con) + .ConfigureAwait(false); } /// - /// Returns the SQLiteCommand type for the current architecture + /// Returns the SQLiteCommand type for the current architecture. /// public static Type SQLiteConnectionType { get { - return typeof(System.Data.SQLite.SQLiteConnection); + return typeof(Microsoft.Data.Sqlite.SqliteConnection); } } /// - /// Returns the version string from the SQLite type + /// Returns the version string from the SQLite type. /// public static string? SQLiteVersion { @@ -212,31 +259,27 @@ namespace Duplicati.Library.SQLiteHelper } /// - /// Wrapper to dispose the SQLite connection + /// Wrapper to dispose the SQLite connection. /// /// The connection to close. - private static void DisposeConnection(System.Data.IDbConnection? con) + /// A task that completes when the connection is disposed. + private static async Task DisposeConnectionAsync(Microsoft.Data.Sqlite.SqliteConnection? con) { if (con != null) - try { con.Dispose(); } + try { await con.DisposeAsync().ConfigureAwait(false); } catch (Exception ex) { Logging.Log.WriteExplicitMessage(LOGTAG, "ConnectionDisposeError", ex, "Failed to dispose connection"); } } /// - /// Opens the SQLite file in the given connection, creating the file if required + /// Opens the SQLite file in the given connection, creating the file if required. /// /// The connection to use. /// Path to the file to open, which may not exist. - private static void OpenSQLiteFile(System.Data.IDbConnection con, string path) + /// A task that completes when the file is opened. + private static async Task OpenSQLiteFileAsync(Microsoft.Data.Sqlite.SqliteConnection con, string path) { - con.ConnectionString = "Data Source=" + path; - con.Open(); - if (con is System.Data.SQLite.SQLiteConnection sqlitecon && !OperatingSystem.IsMacOS()) - { - // These configuration options crash on MacOS (arm64), but the other platforms should be enough to detect incorrect SQL - sqlitecon.SetConfigurationOption(System.Data.SQLite.SQLiteConfigDbOpsEnum.SQLITE_DBCONFIG_DQS_DDL, false); - sqlitecon.SetConfigurationOption(System.Data.SQLite.SQLiteConfigDbOpsEnum.SQLITE_DBCONFIG_DQS_DML, false); - } + con.ConnectionString = $"Data Source={path};Pooling=false"; + await con.OpenAsync().ConfigureAwait(false); // Make the file only accessible by the current user, unless opting out if (!SystemIO.IO_OS.FileExists(SystemIO.IO_OS.PathCombine(SystemIO.IO_OS.PathGetDirectoryName(path), Util.InsecurePermissionsMarkerFile))) @@ -245,17 +288,16 @@ namespace Duplicati.Library.SQLiteHelper } /// - /// Tests the SQLite connection, throwing an exception if the connection does not work + /// Tests the SQLite connection, throwing an exception if the connection does not work. /// /// The connection to test. - private static void TestSQLiteFile(System.Data.IDbConnection con) + /// A task that completes when the test query is executed. + private static async Task TestSQLiteFileAsync(Microsoft.Data.Sqlite.SqliteConnection con) { // Do a dummy query to make sure we have a working db - using (var cmd = con.CreateCommand()) - { - cmd.CommandText = "SELECT COUNT(*) FROM SQLITE_MASTER"; - cmd.ExecuteScalar(); - } + using var cmd = con.CreateCommand(); + cmd.CommandText = "SELECT COUNT(*) FROM SQLITE_MASTER"; + await cmd.ExecuteScalarAsync().ConfigureAwait(false); } } } diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index 970d45061..aa86179da 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -762,7 +762,7 @@ namespace Duplicati.Server System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(databasePath)); // Attempt to open the database, removing any encryption present - Library.SQLiteHelper.SQLiteLoader.OpenDatabase(con, databasePath, Library.SQLiteHelper.SQLiteRC4Decrypter.GetEncryptionPassword(commandlineOptions)); + Library.SQLiteHelper.SQLiteLoader.OpenDatabaseAsync(con, databasePath, Library.SQLiteHelper.SQLiteRC4Decrypter.GetEncryptionPassword(commandlineOptions)).Await(); Library.SQLiteHelper.DatabaseUpgrader.UpgradeDatabase(con, databasePath, typeof(Library.RestAPI.Database.DatabaseSchemaMarker)); } diff --git a/Duplicati/UnitTest/DatabaseToolTests.cs b/Duplicati/UnitTest/DatabaseToolTests.cs index 778f33146..7ac1e09ea 100644 --- a/Duplicati/UnitTest/DatabaseToolTests.cs +++ b/Duplicati/UnitTest/DatabaseToolTests.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.Threading.Tasks; using Duplicati.CommandLine.DatabaseTool; @@ -35,7 +35,7 @@ namespace Duplicati.UnitTest public async Task TestLocalDbMethods() { using var dbfile = new TempFile(); - using var db = SQLiteLoader.LoadConnection(dbfile, 0); + using var db = SQLiteLoader.LoadConnection(dbfile); using var cmd = db.CreateCommand(); cmd.CommandText = LocalSchemaV12; cmd.ExecuteNonQuery(); @@ -55,7 +55,7 @@ namespace Duplicati.UnitTest public async Task TestServerDbMethods() { using var dbfile = new TempFile(); - using var db = SQLiteLoader.LoadConnection(dbfile, 0); + using var db = SQLiteLoader.LoadConnection(dbfile); using var cmd = db.CreateCommand(); cmd.CommandText = ServerSchemaV6; cmd.ExecuteNonQuery(); @@ -142,8 +142,8 @@ CREATE TABLE ""Notification"" ( ""ID"" INTEGER PRIMARY KEY, ""Type"" TEXT NOT NULL, ""Title"" TEXT NOT NULL, - ""Message"" TEXT NOT NULL, - ""Exception"" TEXT NOT NULL, + ""Message"" TEXT NOT NULL, + ""Exception"" TEXT NOT NULL, ""BackupID"" TEXT NULL, ""Action"" TEXT NOT NULL, ""Timestamp"" INTEGER NOT NULL, @@ -153,15 +153,15 @@ CREATE TABLE ""Notification"" ( ); CREATE TABLE ""UIStorage"" ( - ""Scheme"" TEXT NOT NULL, - ""Key"" TEXT NOT NULL, + ""Scheme"" TEXT NOT NULL, + ""Key"" TEXT NOT NULL, ""Value"" TEXT NOT NULL ); CREATE TABLE ""TempFile"" ( ""ID"" INTEGER PRIMARY KEY, - ""Origin"" TEXT NOT NULL, - ""Path"" TEXT NOT NULL, + ""Origin"" TEXT NOT NULL, + ""Path"" TEXT NOT NULL, ""Timestamp"" INTEGER NOT NULL, ""Expires"" INTEGER NOT NULL ); @@ -251,7 +251,7 @@ CREATE TABLE ""Blockset"" ( ); CREATE UNIQUE INDEX ""BlocksetFullHash"" ON ""Blockset"" (""FullHash"", ""Length""); - + CREATE TABLE ""BlocksetEntry"" ( ""BlocksetID"" INTEGER NOT NULL, ""Index"" INTEGER NOT NULL, @@ -326,11 +326,11 @@ CREATE TABLE ""Configuration"" ( CREATE TABLE ""ChangeJournalData"" ( ""ID"" INTEGER PRIMARY KEY, - ""FilesetID"" INTEGER NOT NULL, - ""VolumeName"" TEXT NOT NULL, - ""JournalID"" INTEGER NOT NULL, - ""NextUsn"" INTEGER NOT NULL, - ""ConfigHash"" TEXT NOT NULL + ""FilesetID"" INTEGER NOT NULL, + ""VolumeName"" TEXT NOT NULL, + ""JournalID"" INTEGER NOT NULL, + ""NextUsn"" INTEGER NOT NULL, + ""ConfigHash"" TEXT NOT NULL ); INSERT INTO ""Version"" (""Version"") VALUES (12); diff --git a/Duplicati/UnitTest/DirectListHandlerTests.cs b/Duplicati/UnitTest/DirectListHandlerTests.cs index 624a82115..4da12acab 100644 --- a/Duplicati/UnitTest/DirectListHandlerTests.cs +++ b/Duplicati/UnitTest/DirectListHandlerTests.cs @@ -2,6 +2,8 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Duplicati.Library.Main; using Duplicati.Library.Main.Database; using Duplicati.Library.Utility; @@ -321,10 +323,11 @@ namespace Duplicati.UnitTest } [Test] - public void GetMinimalUniquePrefixEntries_ShouldReturnCorrectLinuxPrefixes() + public async Task GetMinimalUniquePrefixEntries_ShouldReturnCorrectLinuxPrefixes() { using var tempFile = new TempFile(); - using var db = new LocalListDatabase(tempFile, 1); + using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None) + .ConfigureAwait(false); SeedTestData(db, [ "/folder1/", "/folder1/sub1/", @@ -333,7 +336,11 @@ namespace Duplicati.UnitTest "/folder3/" ]); - var result = db.GetMinimalUniquePrefixEntries(1).Select(e => e.Path).ToList(); + var result = await db + .GetMinimalUniquePrefixEntries(1, CancellationToken.None) + .Select(e => e.Path) + .ToListAsync() + .ConfigureAwait(false); Assert.That(result, Does.Contain("/folder1/")); Assert.That(result, Does.Contain("/folder2/")); @@ -344,10 +351,11 @@ namespace Duplicati.UnitTest } [Test] - public void GetMinimalUniquePrefixEntries_ShouldReturnCorrectWindowsDrivePrefixes() + public async Task GetMinimalUniquePrefixEntries_ShouldReturnCorrectWindowsDrivePrefixes() { using var tempFile = new TempFile(); - using var db = new LocalListDatabase(tempFile, 1); + using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None) + .ConfigureAwait(false); SeedTestData(db, [ "C:\\folder1\\", "C:\\folder1\\sub1\\", @@ -355,7 +363,11 @@ namespace Duplicati.UnitTest "D:\\otherfolder\\" ]); - var result = db.GetMinimalUniquePrefixEntries(1).Select(e => e.Path).ToList(); + var result = await db + .GetMinimalUniquePrefixEntries(1, CancellationToken.None) + .Select(e => e.Path) + .ToListAsync() + .ConfigureAwait(false); Assert.That(result, Does.Contain("C:\\folder1\\")); Assert.That(result, Does.Contain("C:\\folder2\\")); @@ -365,17 +377,22 @@ namespace Duplicati.UnitTest } [Test] - public void GetMinimalUniquePrefixEntries_ShouldReturnCorrectWindowsUncPrefixes() + public async Task GetMinimalUniquePrefixEntries_ShouldReturnCorrectWindowsUncPrefixes() { using var tempFile = new TempFile(); - using var db = new LocalListDatabase(tempFile, 1); + using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None) + .ConfigureAwait(false); SeedTestData(db, [ "\\\\server\\share\\folder\\", "\\\\server\\share\\folder\\subfolder\\", "\\\\server\\share\\otherfolder\\" ]); - var result = db.GetMinimalUniquePrefixEntries(1).Select(e => e.Path).ToList(); + var result = await db + .GetMinimalUniquePrefixEntries(1, CancellationToken.None) + .Select(e => e.Path) + .ToListAsync() + .ConfigureAwait(false); Assert.That(result, Does.Contain("\\\\server\\share\\folder\\")); Assert.That(result, Does.Contain("\\\\server\\share\\otherfolder\\")); @@ -384,10 +401,11 @@ namespace Duplicati.UnitTest } [Test] - public void GetMinimalUniquePrefixEntries_ShouldHandleMixedWindowsDriveAndUncPaths() + public async Task GetMinimalUniquePrefixEntries_ShouldHandleMixedWindowsDriveAndUncPaths() { using var tempFile = new TempFile(); - using var db = new LocalListDatabase(tempFile, 1); + using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None) + .ConfigureAwait(false); SeedTestData(db, [ "C:\\data\\", "C:\\data\\sub1\\", @@ -398,7 +416,11 @@ namespace Duplicati.UnitTest "\\\\server\\share\\pictures\\" ]); - var result = db.GetMinimalUniquePrefixEntries(1).Select(e => e.Path).ToList(); + var result = await db + .GetMinimalUniquePrefixEntries(1, CancellationToken.None) + .Select(e => e.Path) + .ToListAsync() + .ConfigureAwait(false); Assert.That(result, Does.Contain("C:\\data\\")); Assert.That(result, Does.Contain("C:\\music\\")); @@ -411,7 +433,7 @@ namespace Duplicati.UnitTest } [Test] - public void GetMinimalUniquePrefixEntries_ShouldReturnExpectedMinimalRoots() + public async Task GetMinimalUniquePrefixEntries_ShouldReturnExpectedMinimalRoots() { // Arrange: Prepare prefixes (minimal unique) and contents var testPrefixes = new[] @@ -430,10 +452,15 @@ namespace Duplicati.UnitTest // Act using var tempFile = new TempFile(); - using var db = new LocalListDatabase(tempFile, 1); + await using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None) + .ConfigureAwait(false); SeedTestData(db, testPrefixes); - var resultItems = db.GetMinimalUniquePrefixEntries(1).ToList(); + var resultItems = await db + .GetMinimalUniquePrefixEntries(1, CancellationToken.None) + .ToListAsync() + .ConfigureAwait(false); + var result = resultItems.Select(e => e.Path).ToList(); Assert.That(result, Does.Contain(@"C:\Downloads\testsource\AA\")); diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs index 6123b2ece..f72e30858 100644 --- a/Duplicati/UnitTest/DisruptionTests.cs +++ b/Duplicati/UnitTest/DisruptionTests.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; @@ -732,7 +732,7 @@ namespace Duplicati.UnitTest [Test] [Category("Disruption")] - public void TestUploadExceptionWithNoRetries() + public async Task TestUploadExceptionWithNoRetries() { var testopts = TestOptions; testopts["number-of-retries"] = "0"; @@ -806,7 +806,7 @@ namespace Duplicati.UnitTest } catch (TestUtils.TestVerificationException e) { - using var db = new LocalDatabase(testopts["dbpath"], "test", true, 0); + using var db = await LocalDatabase.CreateLocalDatabaseAsync(testopts["dbpath"], "test", true, null, CancellationToken.None).ConfigureAwait(false); using var cmd = db.Connection.CreateCommand(); var sb = new StringBuilder(); @@ -1097,7 +1097,7 @@ namespace Duplicati.UnitTest // If we prevent the file from being uploaded, // it will be a temporary file, and a new synthetic file // will be attempted, but that fails as well - // If it is after, then the upload completes, + // If it is after, then the upload completes, // and it will be promoted to a regular uploaded backup expectedFilesets += before ? 1 : runs; diff --git a/Duplicati/UnitTest/EmptyMetadataTests.cs b/Duplicati/UnitTest/EmptyMetadataTests.cs index f4e9c2a0e..aca37e554 100644 --- a/Duplicati/UnitTest/EmptyMetadataTests.cs +++ b/Duplicati/UnitTest/EmptyMetadataTests.cs @@ -28,6 +28,9 @@ using Duplicati.Library.Main.Database; using Duplicati.Library.Interface; using Duplicati.Library.SQLiteHelper; using NUnit.Framework; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.Utility; namespace Duplicati.UnitTest; @@ -35,7 +38,7 @@ public class EmptyMetadataTests : BasicSetupHelper { [Test] [Category("Targeted")] - public void ReplaceMissingMetadataRestoresConsistency() + public async Task ReplaceMissingMetadataRestoresConsistency() { var testopts = TestOptions.Expand(new { no_encryption = true }); @@ -49,7 +52,7 @@ public class EmptyMetadataTests : BasicSetupHelper long metaBlocksetId; long filesetId; - using (var db = SQLiteLoader.LoadConnection(DBFILE, 0)) + using (var db = SQLiteLoader.LoadConnection(DBFILE)) using (var cmd = db.CreateCommand()) { // Find metadata blockset ID and fileset ID for the backed up folder @@ -79,26 +82,28 @@ public class EmptyMetadataTests : BasicSetupHelper } var opts = new Options(testopts); - var emptyMeta = Utility.WrapMetadata(new Dictionary(), opts); + var emptyMeta = Duplicati.Library.Main.Utility.WrapMetadata(new Dictionary(), opts); Assert.Throws(() => { - using var db = new LocalDatabase(DBFILE, "verify", true, 0); - db.VerifyConsistency(opts.Blocksize, opts.BlockhashSize, true, null); + using var db = LocalDatabase.CreateLocalDatabaseAsync(DBFILE, "verify", true, null, CancellationToken.None).Await(); + db.VerifyConsistency(opts.Blocksize, opts.BlockhashSize, true, CancellationToken.None).Await(); }); - using (var db = new LocalListBrokenFilesDatabase(DBFILE, 0)) + await using (var db = await LocalListBrokenFilesDatabase.CreateAsync(DBFILE, null, CancellationToken.None).ConfigureAwait(false)) { var blockVolumeIds = Array.Empty(); - var emptyId = db.GetEmptyMetadataBlocksetId(blockVolumeIds, emptyMeta.FileHash, emptyMeta.Blob.Length, null); + var emptyId = await db.GetEmptyMetadataBlocksetId(blockVolumeIds, emptyMeta.FileHash, emptyMeta.Blob.Length, CancellationToken.None); Assert.That(emptyId, Is.GreaterThanOrEqualTo(0), "Empty metadata blockset not found"); - var replaced = db.ReplaceMetadata(filesetId, emptyId, null); + var replaced = await db.ReplaceMetadata(filesetId, emptyId, CancellationToken.None); Assert.That(replaced, Is.GreaterThan(0), "No metadata rows replaced"); + + await db.Transaction.CommitAsync(CancellationToken.None).ConfigureAwait(false); } - using (var db = new LocalDatabase(DBFILE, "verify", true, 0)) - db.VerifyConsistency(opts.Blocksize, opts.BlockhashSize, true, null); + await using (var db = await LocalDatabase.CreateLocalDatabaseAsync(DBFILE, "verify", true, null, CancellationToken.None)) + await db.VerifyConsistency(opts.Blocksize, opts.BlockhashSize, true, CancellationToken.None); } } diff --git a/Duplicati/UnitTest/ImportExportTests.cs b/Duplicati/UnitTest/ImportExportTests.cs index 3d8fe743f..527e3a4fb 100644 --- a/Duplicati/UnitTest/ImportExportTests.cs +++ b/Duplicati/UnitTest/ImportExportTests.cs @@ -1,27 +1,28 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.IO; +using System.Threading.Tasks; using Duplicati.Library.RestAPI; using Duplicati.Library.SQLiteHelper; using Duplicati.Server; @@ -124,7 +125,7 @@ namespace Duplicati.UnitTest [Test] [Category("ImportExport")] - public void RoundTrip() + public async Task RoundTrip() { var metadata = new Dictionary { { "SourceFilesCount", "1" } }; var advancedOptions = new Dictionary { { "server-datafolder", this.serverDatafolder } }; @@ -133,8 +134,8 @@ namespace Duplicati.UnitTest if (File.Exists(dbpath)) File.Delete(dbpath); - var con = SQLiteLoader.LoadConnection(); - SQLiteLoader.OpenDatabase(con, dbpath, null); + var con = await SQLiteLoader.LoadConnectionAsync().ConfigureAwait(false); + await SQLiteLoader.OpenDatabaseAsync(con, dbpath, null).ConfigureAwait(false); DatabaseUpgrader.UpgradeDatabase(con, dbpath, typeof(Library.RestAPI.Database.DatabaseSchemaMarker)); using (var connection = new Connection(con, true, null, this.serverDatafolder, null)) diff --git a/Duplicati/UnitTest/Issue4693.cs b/Duplicati/UnitTest/Issue4693.cs index ff881cc06..a3555bfec 100644 --- a/Duplicati/UnitTest/Issue4693.cs +++ b/Duplicati/UnitTest/Issue4693.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -156,7 +156,7 @@ namespace Duplicati.UnitTest using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) TestUtils.AssertResults(c.Backup([DATAFOLDER])); - using (var db = SQLiteLoader.LoadConnection(DBFILE, 0)) + using (var db = SQLiteLoader.LoadConnection(DBFILE)) using (var cmd = db.CreateCommand()) { var deletedBlocks = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""DeletedBlock"""); @@ -172,14 +172,14 @@ namespace Duplicati.UnitTest using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) TestUtils.AssertResults(c.Backup([DATAFOLDER])); - using (var db = SQLiteLoader.LoadConnection(DBFILE, 0)) + using (var db = SQLiteLoader.LoadConnection(DBFILE)) using (var cmd = db.CreateCommand()) { var deletedBlocks = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""DeletedBlock"""); Assert.That(deletedBlocks, Is.GreaterThan(0), "DeletedBlock table is empty"); } - using (var db = SQLiteLoader.LoadConnection(DBFILE, 0)) + using (var db = SQLiteLoader.LoadConnection(DBFILE)) using (var cmd = db.CreateCommand()) { var deletedBlocks = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""DeletedBlock"""); @@ -193,7 +193,7 @@ namespace Duplicati.UnitTest using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) TestUtils.AssertResults(c.Backup([DATAFOLDER])); - using (var db = SQLiteLoader.LoadConnection(DBFILE, 0)) + using (var db = SQLiteLoader.LoadConnection(DBFILE)) using (var cmd = db.CreateCommand()) { var alldeletedBlocks = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""DeletedBlock"""); diff --git a/Duplicati/UnitTest/Issue5066.cs b/Duplicati/UnitTest/Issue5066.cs index 60197e4fb..73638764d 100644 --- a/Duplicati/UnitTest/Issue5066.cs +++ b/Duplicati/UnitTest/Issue5066.cs @@ -1,150 +1,150 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using Duplicati.Library.Utility; -using NUnit.Framework; - -namespace Duplicati.UnitTest -{ - public class Issue5066 : BasicSetupHelper - { - - private static string CalculateFileHash(string filename) - { - using (var fs = File.OpenRead(filename)) - using (var hasher = HashFactory.CreateHasher("SHA256")) - return Convert.ToBase64String(hasher.ComputeHash(fs)); - } - - [Test] - [Category("Targeted")] - [TestCase(true)] - [TestCase(false)] - public void TestDuplicatedBlocklists1(bool deleteAllIndexFiles) - { - var testopts = TestOptions.Expand(new { blocksize = "1kb" }); - var hashes = new List(); - - // Full blocklist with zeroes - var data = new byte[32769]; - data.AsSpan().Fill((byte)'a'); - File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); - hashes.Add(CalculateFileHash(Path.Combine(DATAFOLDER, "a"))); - - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - TestUtils.AssertResults(c.Backup([DATAFOLDER])); - - // Make the first blocklist different - data[0] = (byte)'b'; - File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); - hashes.Insert(0, CalculateFileHash(Path.Combine(DATAFOLDER, "a"))); - - // Record existing dindex files - var existingDIndexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly).ToList(); - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - TestUtils.AssertResults(c.Backup([DATAFOLDER])); - - // Delete new index files - foreach (var file in Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly)) - { - if (existingDIndexFiles.Contains(file) || deleteAllIndexFiles) - File.Delete(file); - } - - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - { - TestUtils.AssertResults(c.Repair()); - TestUtils.AssertResults(c.Test()); - } - - for (var version = 0; version < 2; version++) - { - File.Delete(Path.Combine(DATAFOLDER, "a")); - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = version }), null)) - { - TestUtils.AssertResults(c.Restore(null)); - var hash = CalculateFileHash(Path.Combine(DATAFOLDER, "a")); - Assert.That(hashes[version], Is.EqualTo(hash), "Hash mismatch for version " + version); - } - } - } - - [Test] - [Category("Targeted")] - [TestCase(true)] - [TestCase(false)] - public void TestDuplicatedBlocklists2(bool deleteAllIndexFiles) - { - var testopts = TestOptions.Expand(new { blocksize = "1kb" }); - var hashes = new List(); - - // Full blocklist with zeroes - var data = new byte[32769]; - data.AsSpan().Fill((byte)'a'); - File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); - hashes.Add(CalculateFileHash(Path.Combine(DATAFOLDER, "a"))); - - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - TestUtils.AssertResults(c.Backup([DATAFOLDER])); - - // Expand to an new set - data = new byte[32769 * 2 + 1]; - data.AsSpan().Fill((byte)'b'); - data[data.Length - 1] = (byte)'a'; - File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); - hashes.Insert(0, CalculateFileHash(Path.Combine(DATAFOLDER, "a"))); - - // Record existing dindex files - var existingDIndexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly).ToList(); - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - TestUtils.AssertResults(c.Backup([DATAFOLDER])); - - // Delete new index files - foreach (var file in Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly)) - { - if (existingDIndexFiles.Contains(file) || deleteAllIndexFiles) - File.Delete(file); - } - - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) - { - TestUtils.AssertResults(c.Repair()); - TestUtils.AssertResults(c.Test()); - } - - for (var version = 0; version < 2; version++) - { - File.Delete(Path.Combine(DATAFOLDER, "a")); - using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = version }), null)) - { - TestUtils.AssertResults(c.Restore(null)); - var hash = CalculateFileHash(Path.Combine(DATAFOLDER, "a")); - Assert.That(hashes[version], Is.EqualTo(hash), "Hash mismatch for version " + version); - } - } - } - } -} - +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Duplicati.Library.Utility; +using NUnit.Framework; + +namespace Duplicati.UnitTest +{ + public class Issue5066 : BasicSetupHelper + { + + private static string CalculateFileHash(string filename) + { + using (var fs = File.OpenRead(filename)) + using (var hasher = HashFactory.CreateHasher("SHA256")) + return Convert.ToBase64String(hasher.ComputeHash(fs)); + } + + [Test] + [Category("Targeted")] + [TestCase(true)] + [TestCase(false)] + public void TestDuplicatedBlocklists1(bool deleteAllIndexFiles) + { + var testopts = TestOptions.Expand(new { blocksize = "1kb" }); + var hashes = new List(); + + // Full blocklist with zeroes + var data = new byte[32769]; + data.AsSpan().Fill((byte)'a'); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + hashes.Add(CalculateFileHash(Path.Combine(DATAFOLDER, "a"))); + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // Make the first blocklist different + data[0] = (byte)'b'; + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + hashes.Insert(0, CalculateFileHash(Path.Combine(DATAFOLDER, "a"))); + + // Record existing dindex files + var existingDIndexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly).ToList(); + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // Delete new index files + foreach (var file in Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly)) + { + if (existingDIndexFiles.Contains(file) || deleteAllIndexFiles) + File.Delete(file); + } + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + TestUtils.AssertResults(c.Repair()); + TestUtils.AssertResults(c.Test()); + } + + for (var version = 0; version < 2; version++) + { + File.Delete(Path.Combine(DATAFOLDER, "a")); + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = version }), null)) + { + TestUtils.AssertResults(c.Restore(null)); + var hash = CalculateFileHash(Path.Combine(DATAFOLDER, "a")); + Assert.That(hashes[version], Is.EqualTo(hash), "Hash mismatch for version " + version); + } + } + } + + [Test] + [Category("Targeted")] + [TestCase(true)] + [TestCase(false)] + public void TestDuplicatedBlocklists2(bool deleteAllIndexFiles) + { + var testopts = TestOptions.Expand(new { blocksize = "1kb" }); + var hashes = new List(); + + // Full blocklist with zeroes + var data = new byte[32769]; + data.AsSpan().Fill((byte)'a'); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + hashes.Add(CalculateFileHash(Path.Combine(DATAFOLDER, "a"))); + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // Expand to an new set + data = new byte[32769 * 2 + 1]; + data.AsSpan().Fill((byte)'b'); + data[data.Length - 1] = (byte)'a'; + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + hashes.Insert(0, CalculateFileHash(Path.Combine(DATAFOLDER, "a"))); + + // Record existing dindex files + var existingDIndexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly).ToList(); + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + TestUtils.AssertResults(c.Backup([DATAFOLDER])); + + // Delete new index files + foreach (var file in Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly)) + { + if (existingDIndexFiles.Contains(file) || deleteAllIndexFiles) + File.Delete(file); + } + + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null)) + { + TestUtils.AssertResults(c.Repair()); + TestUtils.AssertResults(c.Test()); + } + + for (var version = 0; version < 2; version++) + { + File.Delete(Path.Combine(DATAFOLDER, "a")); + using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = version }), null)) + { + TestUtils.AssertResults(c.Restore(null)); + var hash = CalculateFileHash(Path.Combine(DATAFOLDER, "a")); + Assert.That(hashes[version], Is.EqualTo(hash), "Hash mismatch for version " + version); + } + } + } + } +} + diff --git a/Duplicati/UnitTest/RepairHandlerTests.cs b/Duplicati/UnitTest/RepairHandlerTests.cs index f99b8b910..d6fe983bf 100644 --- a/Duplicati/UnitTest/RepairHandlerTests.cs +++ b/Duplicati/UnitTest/RepairHandlerTests.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; @@ -25,6 +25,7 @@ using System.IO; using System.IO.Compression; using System.Linq; using System.Threading; +using System.Threading.Tasks; using Duplicati.Library.Interface; using Duplicati.Library.Main; using Duplicati.Library.Main.Database; @@ -86,7 +87,7 @@ namespace Duplicati.UnitTest List expectedBlocksetIDs = new List(); List expectedIndexes = new List(); List expectedHashes = new List(); - using (IDbConnection connection = SQLiteLoader.LoadConnection(options["dbpath"], 0)) + using (IDbConnection connection = SQLiteLoader.LoadConnection(options["dbpath"])) { // Read the contents of the BlocklistHash table so that we can // compare them to the contents after the repair operation. @@ -123,7 +124,7 @@ namespace Duplicati.UnitTest List repairedBlocksetIDs = new List(); List repairedIndexes = new List(); List repairedHashes = new List(); - using (IDbConnection connection = SQLiteLoader.LoadConnection(options["dbpath"], 0)) + using (IDbConnection connection = SQLiteLoader.LoadConnection(options["dbpath"])) { using (IDbCommand command = connection.CreateCommand()) { @@ -403,7 +404,7 @@ namespace Duplicati.UnitTest [Category("RepairHandler")] [TestCase("true")] [TestCase("false")] - public void RepairExtraIndexFiles(string noEncryption) + public async Task RepairExtraIndexFiles(string noEncryption) { // Extra index files will be added to the database and should have a correct link established Dictionary options = new Dictionary(this.TestOptions) { ["no-encryption"] = noEncryption }; @@ -427,18 +428,30 @@ namespace Duplicati.UnitTest } // Check database block link - using (LocalDatabase db = new LocalDatabase(DBFILE, "Test", true, 0)) + using (LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(DBFILE, "Test", true, null, CancellationToken.None).ConfigureAwait(false)) { - var indexVolumeId = db.GetRemoteVolumeID(Path.GetFileName(origFile)); - var duplicateVolumeId = db.GetRemoteVolumeID(Path.GetFileName(dupFile)); + var indexVolumeId = await db + .GetRemoteVolumeID(Path.GetFileName(origFile), CancellationToken.None) + .ConfigureAwait(false); + var duplicateVolumeId = await db + .GetRemoteVolumeID(Path.GetFileName(dupFile), CancellationToken.None) + .ConfigureAwait(false); Assert.AreNotEqual(-1, indexVolumeId); Assert.AreNotEqual(-1, duplicateVolumeId); using (var cmd = db.Connection.CreateCommand()) { var sql = @"SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"" WHERE ""IndexVolumeID"" = @VolumeId"; - var linkedIndexId = cmd.SetCommandAndParameters(sql).SetParameterValue("@VolumeId", indexVolumeId).ExecuteScalarInt64(-1); - var linkedDuplicateId = cmd.SetCommandAndParameters(sql).SetParameterValue("@VolumeId", duplicateVolumeId).ExecuteScalarInt64(-1); + var linkedIndexId = await cmd + .SetCommandAndParameters(sql) + .SetParameterValue("@VolumeId", indexVolumeId) + .ExecuteScalarInt64Async(-1, CancellationToken.None) + .ConfigureAwait(false); + var linkedDuplicateId = await cmd + .SetCommandAndParameters(sql) + .SetParameterValue("@VolumeId", duplicateVolumeId) + .ExecuteScalarInt64Async(-1, CancellationToken.None) + .ConfigureAwait(false); Assert.AreEqual(linkedIndexId, linkedDuplicateId); } } @@ -494,7 +507,7 @@ namespace Duplicati.UnitTest dlistFiles = dlistFiles.Reverse(); var dlistFile = dlistFiles.First(); - using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0)) + using (var con = SQLiteLoader.LoadConnection(options["dbpath"])) using (var cmd = con.CreateCommand("DELETE FROM RemoteVolume WHERE Name = @Name")) Assert.AreEqual(1, cmd.SetParameterValue("@Name", Path.GetFileName(dlistFile)).ExecuteNonQuery()); @@ -514,7 +527,7 @@ namespace Duplicati.UnitTest } // Check that the entry was recreated - using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0)) + using (var con = SQLiteLoader.LoadConnection(options["dbpath"])) using (var cmd = con.CreateCommand("SELECT COUNT(*) FROM RemoteVolume WHERE Name LIKE '%.dlist.%' AND State != @State")) Assert.AreEqual(2, cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()).ExecuteScalarInt64(-1)); @@ -551,7 +564,7 @@ namespace Duplicati.UnitTest TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); // Remove a fileset - using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0)) + using (var con = SQLiteLoader.LoadConnection(options["dbpath"])) using (var cmd = con.CreateCommand()) { var filesetId = cmd.ExecuteScalarInt64("SELECT Id FROM Fileset ORDER BY Id DESC LIMIT 1"); @@ -581,7 +594,7 @@ namespace Duplicati.UnitTest } // Check that entry was recreated - using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0)) + using (var con = SQLiteLoader.LoadConnection(options["dbpath"])) using (var cmd = con.CreateCommand()) Assert.AreEqual(2, cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM Fileset")); } @@ -605,7 +618,7 @@ namespace Duplicati.UnitTest // Remove a fileset long filesetEntries; long fileLookupEntries; - using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0)) + using (var con = SQLiteLoader.LoadConnection(options["dbpath"])) using (var cmd = con.CreateCommand()) { filesetEntries = cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM FilesetEntry"); @@ -628,7 +641,7 @@ namespace Duplicati.UnitTest } // Check that entry was recreated - using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0)) + using (var con = SQLiteLoader.LoadConnection(options["dbpath"])) using (var cmd = con.CreateCommand()) { Assert.AreEqual(filesetEntries, cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM FilesetEntry")); @@ -646,7 +659,7 @@ namespace Duplicati.UnitTest using (var c = new Controller("file://" + this.TARGETFOLDER, options, null)) TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER })); - using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0)) + using (var con = SQLiteLoader.LoadConnection(options["dbpath"])) using (var cmd = con.CreateCommand()) { var lookupId = cmd.ExecuteScalarInt64("SELECT ID FROM FileLookup LIMIT 1"); @@ -664,7 +677,7 @@ namespace Duplicati.UnitTest Assert.AreEqual(0, res.Errors.Count()); Assert.AreEqual(0, res.Warnings.Count()); } - using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0)) + using (var con = SQLiteLoader.LoadConnection(options["dbpath"])) using (var cmd = con.CreateCommand()) { var remaining = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM Metadataset JOIN Blockset ON Metadataset.BlocksetID = Blockset.ID WHERE Blockset.Length = 0"); diff --git a/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs b/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs index bc2425d9b..0236143af 100644 --- a/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs +++ b/Duplicati/UnitTest/RepairWithMissingRemoteFiles.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -325,7 +325,7 @@ namespace Duplicati.UnitTest (long remoteVolumes, long duplicatedBlocks, List blockVolumeIds, List<(long, long)> indexVolumeLinks) GetDbSignature() { - using var db = SQLiteLoader.LoadConnection(DBFILE, 0); + using var db = SQLiteLoader.LoadConnection(DBFILE); using var cmd = db.CreateCommand(); var remoteVolumes = cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM RemoteVolume"); diff --git a/Duplicati/UnitTest/SQLiteTests.cs b/Duplicati/UnitTest/SQLiteTests.cs index 50ab02888..da0f9f04a 100755 --- a/Duplicati/UnitTest/SQLiteTests.cs +++ b/Duplicati/UnitTest/SQLiteTests.cs @@ -1,28 +1,31 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. + using System.Collections.Generic; -using System.Data; +using System.Threading; +using System.Threading.Tasks; using Duplicati.Library.Main.Database; using Duplicati.Library.SQLiteHelper; using Duplicati.Library.Utility; +using Microsoft.Data.Sqlite; using NUnit.Framework; using Assert = NUnit.Framework.Legacy.ClassicAssert; @@ -31,17 +34,17 @@ namespace Duplicati.UnitTest [TestFixture] public class SQLiteTests : BasicSetupHelper { - private static IDbConnection CreateDummyDatabase(string path) + private static async Task CreateDummyDatabase(string path) { - var connection = SQLiteLoader.LoadConnection(path, 0); + var connection = SQLiteLoader.LoadConnection(path); using (var command = connection.CreateCommand()) { command.CommandText = "CREATE TABLE TestTable (ID INTEGER PRIMARY KEY, Name TEXT)"; - command.ExecuteNonQuery(); + await command.ExecuteNonQueryAsync().ConfigureAwait(false); command.CommandText = "INSERT INTO TestTable (ID, Name) VALUES (1, 'Test1'), (2, 'Test2'), (3, 'Test3')"; - command.ExecuteNonQuery(); + await command.ExecuteNonQueryAsync().ConfigureAwait(false); } return connection; @@ -50,47 +53,49 @@ namespace Duplicati.UnitTest [Test] [Category("SQLite")] - public void TestEmptyTransaction() + public async Task TestEmptyTransaction() { using var tf = new TempFile(); - using var connection = CreateDummyDatabase(tf); + using var connection = await CreateDummyDatabase(tf) + .ConfigureAwait(false); using var t1 = connection.BeginTransaction(); - t1.Commit(); // No exception should be thrown + await t1.CommitAsync().ConfigureAwait(false); // No exception should be thrown using var t2 = connection.BeginTransaction(); - t2.Rollback(); // No exception should be thrown + await t2.RollbackAsync().ConfigureAwait(false); // No exception should be thrown } [Test] [Category("SQLite")] - public void TestListExpansion() + public async Task TestListExpansion() { using var tf = new TempFile(); - using var connection = CreateDummyDatabase(tf); + using var connection = await CreateDummyDatabase(tf).ConfigureAwait(false); + using var rtr = new ReusableTransaction(connection); using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) { command.ExpandInClauseParameter("@List", new long[] { 1, 2, 3 }); - Assert.AreEqual(3, command.ExecuteScalarInt64()); + Assert.AreEqual(3, await command.ExecuteScalarInt64Async(CancellationToken.None).ConfigureAwait(false)); } using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) { command.ExpandInClauseParameter("@List", new long[] { 1, 2 }); - Assert.AreEqual(2, command.ExecuteScalarInt64()); + Assert.AreEqual(2, await command.ExecuteScalarInt64Async(CancellationToken.None).ConfigureAwait(false)); } using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) { command.ExpandInClauseParameter("@List", new long[] { 1 }); - Assert.AreEqual(1, command.ExecuteScalarInt64()); + Assert.AreEqual(1, await command.ExecuteScalarInt64Async(CancellationToken.None).ConfigureAwait(false)); } using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) { command.ExpandInClauseParameter("@List", new long[0]); - Assert.AreEqual(0, command.ExecuteScalarInt64()); + Assert.AreEqual(0, await command.ExecuteScalarInt64Async(CancellationToken.None).ConfigureAwait(false)); } var list = new List(); @@ -99,9 +104,10 @@ namespace Duplicati.UnitTest using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) { - using var tmplist = new TemporaryDbValueList(connection, null, list); + using var tmplist = await TemporaryDbValueList.CreateAsync(connection, rtr, list, CancellationToken.None) + .ConfigureAwait(false); command.ExpandInClauseParameter("@List", list); - Assert.AreEqual(3, command.ExecuteScalarInt64()); + Assert.AreEqual(3, await command.ExecuteScalarInt64Async(CancellationToken.None).ConfigureAwait(false)); } using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)")) @@ -109,9 +115,10 @@ namespace Duplicati.UnitTest list.Remove(1); list.Remove(2); - using var tmplist = new TemporaryDbValueList(connection, null, list); + using var tmplist = await TemporaryDbValueList.CreateAsync(connection, rtr, list, CancellationToken.None) + .ConfigureAwait(false); command.ExpandInClauseParameter("@List", list); - Assert.AreEqual(1, command.ExecuteScalarInt64()); + Assert.AreEqual(1, await command.ExecuteScalarInt64Async(CancellationToken.None).ConfigureAwait(false)); } } } diff --git a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs index 58c492d7b..875b21a29 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.Text.Json; using System.Text.Json.Serialization; @@ -179,7 +179,7 @@ public class BackupGet : IEndpointV1 if (!File.Exists(bk.DBPath)) return new List>(); - using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(bk.DBPath, 0)) + using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(bk.DBPath)) using (var cmd = con.CreateCommand()) return LogData.DumpTable(cmd, "LogData", "ID", offset, pagesize); } @@ -189,7 +189,7 @@ public class BackupGet : IEndpointV1 if (!File.Exists(bk.DBPath)) return new List>(); - using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(bk.DBPath, 0)) + using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(bk.DBPath)) using (var cmd = con.CreateCommand()) { var dt = LogData.DumpTable(cmd, "RemoteOperation", "ID", offset, pagesize); diff --git a/ReleaseBuilder/Build/Command.Compile.Post.cs b/ReleaseBuilder/Build/Command.Compile.Post.cs index 0383b4b75..f7301369e 100644 --- a/ReleaseBuilder/Build/Command.Compile.Post.cs +++ b/ReleaseBuilder/Build/Command.Compile.Post.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.Text.RegularExpressions; @@ -56,7 +56,6 @@ public static partial class Command case OSType.Linux: await ReplaceLibMonoUnix(baseDir, buildDir, target.Arch); - await ReplaceSQLiteInterop(baseDir, buildDir, target.Arch); break; default: @@ -280,28 +279,6 @@ public static partial class Command return Task.CompletedTask; } - /// - /// Replaces the library SQLiteInterop.dll with a version that is built against GLIBC_2.33 for ARM7 - /// - /// The base directory - /// The build directory - /// The architecture to build for - /// An awaitable task - static Task ReplaceSQLiteInterop(string baseDir, string buildDir, ArchType arch) - { - if (arch != ArchType.Arm7) - return Task.CompletedTask; - - var sourceFile = Path.Combine(baseDir, "ReleaseBuilder", "Resources", "linux-arm-binary", "SQLite.Interop.dll"); - var targetFile = Path.Combine(buildDir, "SQLite.Interop.dll"); - if (!File.Exists(targetFile)) - throw new Exception($"Expected file \"{targetFile}\" not found, has build changed?"); - - File.Copy(sourceFile, targetFile, overwrite: true); - - return Task.CompletedTask; - } - /// /// Signs all .exe and .dll files with Authenticode /// diff --git a/ReleaseBuilder/Build/Command.Compile.Verify.cs b/ReleaseBuilder/Build/Command.Compile.Verify.cs index ed33a0928..0784d41f1 100644 --- a/ReleaseBuilder/Build/Command.Compile.Verify.cs +++ b/ReleaseBuilder/Build/Command.Compile.Verify.cs @@ -1,22 +1,22 @@ // Copyright (C) 2025, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System.Reflection; using System.Text; @@ -54,7 +54,7 @@ public static partial class Command "Google.Apis.Auth.dll", "Google.Apis.Core.dll", "SQLiteHelper.dll", - "SQLite.Interop.dll", + "Microsoft.Data.Sqlite.dll", "Microsoft.IdentityModel.Abstractions.dll", "System.Reactive.dll" }; @@ -248,7 +248,7 @@ public static partial class Command { // Using v3.3 for assembly, but 3.7 in nuget { "AWSSDK.Core", new Version(3, 3, 0, 0) }, - + // Using the Framework version, not the package version { "Microsoft.CSharp", new Version(8, 0, 0, 0) }, { "System.Memory", new Version(8, 0, 0, 0) }, @@ -256,7 +256,10 @@ public static partial class Command { "System.Security.Principal.Windows", new Version(8, 0, 0, 0) }, { "System.Security.Cryptography.Algorithms", new Version(8, 0, 0, 0) }, { "System.Security.Cryptography.Cng", new Version(8, 0, 0, 0) }, - + + // The assembly version also has a revision number, but the nuget version does not. + { "SQLitePCLRaw.core", new Version(2, 1, 10, 2445) }, + // Using v9.0 for assembly, but 9.0.2 in nuget { "System.IO.Pipelines", new Version(9, 0, 0, 0) } };